From 0e877570896f960829a0709557fb68f55cb654de Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Tue, 7 Feb 2023 16:58:44 -0800 Subject: [PATCH 001/169] Go 1.20 --- .github/workflows/bench.yml | 2 +- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- .github/workflows/test-monitor-flaky.yml | 2 +- .github/workflows/test-monitor-regular-skipped.yml | 2 +- .github/workflows/tools.yml | 2 +- cmd/Dockerfile | 4 ++-- cmd/testclient/go.mod | 2 +- crypto/Dockerfile | 2 +- crypto/go.mod | 2 +- go.mod | 2 +- insecure/go.mod | 2 +- integration/benchmark/cmd/manual/Dockerfile | 2 +- integration/go.mod | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e78d7a18c85..ef5b88d7f55 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -36,7 +36,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.19" + go-version: "1.20" cache: true - name: Build relic diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index eb28e840078..9079fb06a98 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Checkout repo uses: actions/checkout@v2 - name: Build relic diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0bca5d4aba..a5e6300c8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: - 'v[0-9]+.[0-9]+' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index 3a5b47e2c2f..f6637edf0ae 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -5,7 +5,7 @@ on: branches: - '**/*flaky-test-debug*' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" #concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-flaky.yml b/.github/workflows/test-monitor-flaky.yml index 8a951583285..e34642e6d8c 100644 --- a/.github/workflows/test-monitor-flaky.yml +++ b/.github/workflows/test-monitor-flaky.yml @@ -13,7 +13,7 @@ on: env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-regular-skipped.yml b/.github/workflows/test-monitor-regular-skipped.yml index 8eb48c1129e..9276b28db18 100644 --- a/.github/workflows/test-monitor-regular-skipped.yml +++ b/.github/workflows/test-monitor-regular-skipped.yml @@ -15,7 +15,7 @@ env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: skipped_tests BIGQUERY_TABLE2: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 8a057d9dfb5..852247cbed7 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -24,7 +24,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@v1 with: diff --git a/cmd/Dockerfile b/cmd/Dockerfile index 473effbef9b..a1c500ef760 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -3,7 +3,7 @@ #################################### ## (1) Setup the build environment -FROM golang:1.19-bullseye AS build-setup +FROM golang:1.20-bullseye AS build-setup RUN apt-get update RUN apt-get -y install cmake zip @@ -67,7 +67,7 @@ RUN --mount=type=ssh \ RUN chmod a+x /app/app ## (4) Add the statically linked debug binary to a distroless image configured for debugging -FROM golang:1.19-bullseye as debug +FROM golang:1.20-bullseye as debug RUN go install github.com/go-delve/delve/cmd/dlv@latest diff --git a/cmd/testclient/go.mod b/cmd/testclient/go.mod index 0a02e69ad42..dbe66a78fb5 100644 --- a/cmd/testclient/go.mod +++ b/cmd/testclient/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/cmd/testclient -go 1.19 +go 1.20 require ( github.com/onflow/flow-go-sdk v0.4.1 diff --git a/crypto/Dockerfile b/crypto/Dockerfile index 37a0b373171..d75e9543de4 100644 --- a/crypto/Dockerfile +++ b/crypto/Dockerfile @@ -1,6 +1,6 @@ # gcr.io/dl-flow/golang-cmake -FROM golang:1.19-buster +FROM golang:1.20-buster RUN apt-get update RUN apt-get -y install cmake zip RUN go install github.com/axw/gocov/gocov@latest diff --git a/crypto/go.mod b/crypto/go.mod index c7fe54f9ff5..9895e1c35db 100644 --- a/crypto/go.mod +++ b/crypto/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/crypto -go 1.19 +go 1.20 require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 diff --git a/go.mod b/go.mod index e09791a0877..a2ecb856638 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go -go 1.19 +go 1.20 require ( cloud.google.com/go/compute v1.12.1 // indirect diff --git a/insecure/go.mod b/insecure/go.mod index 4c3c3c54bea..6fb01cd0df0 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/insecure -go 1.19 +go 1.20 require ( github.com/golang/protobuf v1.5.2 diff --git a/integration/benchmark/cmd/manual/Dockerfile b/integration/benchmark/cmd/manual/Dockerfile index 1ad38985a43..58f2b71d42b 100644 --- a/integration/benchmark/cmd/manual/Dockerfile +++ b/integration/benchmark/cmd/manual/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:experimental # NOTE: Must be run in the context of the repo's root directory -FROM golang:1.19-buster AS build-setup +FROM golang:1.20-buster AS build-setup RUN apt-get update RUN apt-get -y install cmake zip diff --git a/integration/go.mod b/integration/go.mod index b16d6bd4699..7cde83ef919 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/integration -go 1.19 +go 1.20 require ( cloud.google.com/go/bigquery v1.43.0 From 026312d540e62693dfbe0abc2bc8918bec6ec086 Mon Sep 17 00:00:00 2001 From: Kay-Zee Date: Wed, 8 Feb 2023 13:13:38 -0800 Subject: [PATCH 002/169] Update golangci-lint --- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e6300c8de..67f79da4561 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index f6637edf0ae..8058a656f29 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -36,7 +36,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 From a0e34347d8c8359011ce0ac82021bd5befca4723 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 13:12:57 -0600 Subject: [PATCH 003/169] replace math/rand in production code - remove depriacted functions in go1.20 --- cmd/bootstrap/cmd/clusters.go | 17 +- cmd/bootstrap/cmd/constraints.go | 2 +- cmd/bootstrap/cmd/dkg.go | 2 +- cmd/bootstrap/cmd/finalize.go | 23 +- cmd/bootstrap/cmd/finalize_test.go | 240 +----------------- cmd/bootstrap/cmd/machine_account_test.go | 1 + cmd/bootstrap/cmd/rootblock.go | 11 - cmd/bootstrap/cmd/rootblock_test.go | 12 +- cmd/bootstrap/cmd/seal.go | 2 +- cmd/bootstrap/utils/file.go | 2 +- cmd/execution_builder.go | 1 + cmd/observer/node_builder/observer_builder.go | 1 + cmd/scaffold.go | 4 - .../leader/leader_selection_test.go | 2 +- .../signature/block_signer_decoder_test.go | 3 +- .../signature/randombeacon_inspector_test.go | 13 +- .../timeout_collector_test.go | 1 - .../hotstuff/validator/validator_test.go | 3 - .../combined_vote_processor_v2_test.go | 5 +- .../combined_vote_processor_v3_test.go | 5 +- consensus/integration/network_test.go | 6 +- consensus/integration/nodes_test.go | 2 +- engine/access/relay/example_test.go | 8 +- engine/access/rest_api_test.go | 1 - engine/access/rpc/backend/backend.go | 5 +- engine/access/rpc/backend/backend_test.go | 4 +- .../rpc/backend/backend_transactions.go | 5 +- engine/access/state_stream/api_test.go | 5 +- engine/collection/compliance/core_test.go | 3 - .../message_hub/message_hub_test.go | 2 - engine/collection/synchronization/engine.go | 28 +- .../collection/synchronization/engine_test.go | 1 - engine/common/follower/engine.go | 5 +- engine/common/follower/engine_test.go | 2 +- engine/common/requester/engine.go | 24 +- engine/common/requester/engine_test.go | 5 - engine/common/rpc/convert/convert_test.go | 2 +- .../common/splitter/network/example_test.go | 8 +- engine/common/synchronization/engine.go | 29 ++- engine/common/synchronization/engine_test.go | 1 - engine/consensus/approvals/request_tracker.go | 49 +++- .../verifying_assignment_collector.go | 10 +- engine/consensus/compliance/core.go | 5 +- engine/consensus/compliance/core_test.go | 5 +- .../consensus/message_hub/message_hub_test.go | 2 - engine/execution/computation/manager.go | 14 +- engine/execution/ingestion/engine_test.go | 6 +- engine/execution/provider/engine.go | 14 +- engine/execution/provider/engine_test.go | 2 - engine/protocol/api_test.go | 4 +- engine/testutil/nodes.go | 8 +- engine/verification/requester/requester.go | 7 +- engine/verification/verifier/engine_test.go | 3 +- fvm/environment/unsafe_random_generator.go | 32 ++- fvm/fvm_bench_test.go | 3 - fvm/fvm_blockcontext_test.go | 2 +- fvm/fvm_signature_test.go | 2 +- .../wintermute/attackOrchestrator_test.go | 7 +- integration/dkg/dkg_emulator_test.go | 2 - integration/dkg/dkg_whiteboard_test.go | 2 - integration/testnet/network.go | 3 +- .../tests/access/consensus_follower_test.go | 4 +- integration/tests/consensus/inclusion_test.go | 1 - integration/tests/consensus/sealing_test.go | 1 - integration/tests/lib/util.go | 2 +- ledger/common/bitutils/utils_test.go | 7 +- ledger/common/hash/hash_test.go | 7 +- ledger/common/testutils/testutils.go | 11 +- ledger/complete/ledger_benchmark_test.go | 11 - ledger/complete/ledger_test.go | 2 - .../complete/mtrie/flattener/encoding_test.go | 3 +- ledger/complete/mtrie/forest_test.go | 2 +- ledger/complete/mtrie/trie/trie_test.go | 7 +- ledger/complete/mtrie/trieCache_test.go | 2 +- ledger/complete/wal/checkpoint_v6_test.go | 2 +- ledger/complete/wal/triequeue_test.go | 2 +- ledger/partial/ptrie/partialTrie_test.go | 2 +- model/encodable/keys_test.go | 3 +- model/flow/address_test.go | 5 - model/flow/identifier.go | 21 +- model/flow/identifierList.go | 10 +- model/flow/identifierList_test.go | 3 +- model/flow/identifier_test.go | 11 +- model/flow/identity.go | 59 +++-- model/flow/identity_test.go | 28 +- model/verification/chunkDataPackRequest.go | 17 +- module/builder/collection/builder_test.go | 2 - module/chunks/chunkVerifier_test.go | 3 - module/chunks/chunk_assigner_test.go | 2 +- module/dkg/controller.go | 35 ++- module/dkg/controller_test.go | 24 +- module/epochs/qc_voter_test.go | 2 +- .../execution_data/store_test.go | 7 +- module/finalizer/collection/finalizer_test.go | 3 - .../herocache/backdata/heropool/pool.go | 21 +- module/mempool/herocache/dns_cache.go | 3 +- module/mempool/herocache/transactions.go | 12 +- module/mempool/mock/back_data.go | 13 +- module/mempool/queue/heroQueue.go | 18 +- module/mempool/queue/heroQueue_test.go | 1 - module/mempool/queue/heroStore.go | 7 +- .../stdmap/backDataHeapBenchmark_test.go | 14 +- module/mempool/stdmap/backend.go | 8 +- module/mempool/stdmap/eject.go | 119 +-------- module/mempool/stdmap/eject_test.go | 230 ----------------- module/signature/aggregation_test.go | 30 ++- module/signature/signer_indices_test.go | 21 +- .../execution_data_requester_test.go | 1 - .../jobs/execution_data_reader_test.go | 2 - module/trace/trace_test.go | 2 +- network/cache/rcvcache.go | 3 +- .../p2p/cache/node_blocklist_wrapper_test.go | 6 +- network/p2p/connection/connector.go | 13 +- network/p2p/connection/peerManager.go | 8 +- network/p2p/network.go | 7 +- network/p2p/unicast/manager.go | 9 +- network/queue/messageQueue_test.go | 2 +- network/stub/network.go | 6 +- network/test/epochtransition_test.go | 2 +- state/cluster/badger/mutator_test.go | 2 - state/cluster/badger/snapshot_test.go | 3 - state/protocol/badger/mutator_test.go | 2 +- state/protocol/badger/snapshot_test.go | 11 +- state/protocol/badger/state_test.go | 5 +- state/protocol/badger/validity_test.go | 8 +- state/protocol/seed/prg_test.go | 15 +- storage/badger/cleaner.go | 28 +- storage/badger/dkg_state_test.go | 3 - storage/badger/operation/common_test.go | 4 +- storage/cleaner.go | 2 +- storage/merkle/proof_test.go | 3 +- storage/merkle/tree_test.go | 14 +- storage/mock/cleaner.go | 13 +- utils/math/math.go | 16 -- utils/rand/rand.go | 169 ++++++++++++ utils/unittest/chain_suite.go | 7 +- utils/unittest/fixtures.go | 9 + utils/unittest/network/fixtures.go | 5 +- 138 files changed, 783 insertions(+), 1073 deletions(-) delete mode 100644 module/mempool/stdmap/eject_test.go delete mode 100644 utils/math/math.go create mode 100644 utils/rand/rand.go diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 8f6faa10505..cf91214349f 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -13,16 +13,21 @@ import ( // Construct cluster assignment with internal and partner nodes uniformly // distributed across clusters. This function will produce the same cluster // assignments for the same partner and internal lists, and the same seed. -func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo, seed int64) (flow.AssignmentList, flow.ClusterList) { +func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (flow.AssignmentList, flow.ClusterList) { partners := model.ToIdentityList(partnerNodes).Filter(filter.HasRole(flow.RoleCollection)) internals := model.ToIdentityList(internalNodes).Filter(filter.HasRole(flow.RoleCollection)) - // deterministically shuffle both collector lists based on the input seed - // by using a different seed each spork, we will have different clusters - // even with the same collectors - partners = partners.DeterministicShuffle(seed) - internals = internals.DeterministicShuffle(seed) + // we will have different clusters even with the same collectors + var err error + partners, err = partners.Shuffle() + if err != nil { + log.Fatal().Err(err).Msg("could not shuffle partners") + } + internals, err = internals.Shuffle() + if err != nil { + log.Fatal().Err(err).Msg("could not shuffle internals") + } nClusters := flagCollectionClusters identifierLists := make([]flow.IdentifierList, nClusters) diff --git a/cmd/bootstrap/cmd/constraints.go b/cmd/bootstrap/cmd/constraints.go index b7c17b07b4a..56c09b380ce 100644 --- a/cmd/bootstrap/cmd/constraints.go +++ b/cmd/bootstrap/cmd/constraints.go @@ -37,7 +37,7 @@ func checkConstraints(partnerNodes, internalNodes []model.NodeInfo) { // check collection committee Byzantine threshold for each cluster // for checking Byzantine constraints, the seed doesn't matter - _, clusters := constructClusterAssignment(partnerNodes, internalNodes, 0) + _, clusters := constructClusterAssignment(partnerNodes, internalNodes) partnerCOLCount := uint(0) internalCOLCount := uint(0) for _, cluster := range clusters { diff --git a/cmd/bootstrap/cmd/dkg.go b/cmd/bootstrap/cmd/dkg.go index b190b1a7c2c..5f9c5df8bd3 100644 --- a/cmd/bootstrap/cmd/dkg.go +++ b/cmd/bootstrap/cmd/dkg.go @@ -20,7 +20,7 @@ func runDKG(nodes []model.NodeInfo) dkg.DKGData { var dkgData dkg.DKGData var err error if flagFastKG { - dkgData, err = bootstrapDKG.RunFastKG(n, flagBootstrapRandomSeed) + dkgData, err = bootstrapDKG.RunFastKG(n, GenerateRandomSeed(crypto.SeedMinLenDKG)) } else { dkgData, err = bootstrapDKG.RunDKG(n, GenerateRandomSeeds(n, crypto.SeedMinLenDKG)) } diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 5d1eb74106a..a688e21928f 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -1,7 +1,7 @@ package cmd import ( - "encoding/binary" + "crypto/rand" "encoding/hex" "encoding/json" "fmt" @@ -48,9 +48,6 @@ var ( flagNumViewsInStakingAuction uint64 flagNumViewsInDKGPhase uint64 flagEpochCommitSafetyThreshold uint64 - - // this flag is used to seed the DKG, clustering and cluster QC generation - flagBootstrapRandomSeed []byte ) // PartnerWeights is the format of the JSON file specifying partner node weights. @@ -101,7 +98,6 @@ func addFinalizeCmdFlags() { finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views") finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views") finalizeCmd.Flags().Uint64Var(&flagEpochCommitSafetyThreshold, "epoch-commit-safety-threshold", 500, "defines epoch commitment deadline") - finalizeCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation") finalizeCmd.Flags().UintVar(&flagProtocolVersion, "protocol-version", flow.DefaultProtocolVersion, "major software version used for the duration of this spork") cmd.MarkFlagRequired(finalizeCmd, "root-block") @@ -143,14 +139,6 @@ func finalize(cmd *cobra.Command, args []string) { log.Fatal().Err(err).Msg("invalid or unsafe epoch commit threshold config") } - if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength { - log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid") - return - } - - log.Info().Str("seed", hex.EncodeToString(flagBootstrapRandomSeed)).Msg("deterministic bootstrapping random seed") - log.Info().Msg("") - log.Info().Msg("collecting partner network and staking keys") partnerNodes := readPartnerNodeInfos() log.Info().Msg("") @@ -195,8 +183,7 @@ func finalize(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("computing collection node clusters") - clusterAssignmentSeed := binary.BigEndian.Uint64(flagBootstrapRandomSeed) - assignments, clusters := constructClusterAssignment(partnerNodes, internalNodes, int64(clusterAssignmentSeed)) + assignments, clusters := constructClusterAssignment(partnerNodes, internalNodes) log.Info().Msg("") log.Info().Msg("constructing root blocks for collection node clusters") @@ -211,7 +198,6 @@ func finalize(cmd *cobra.Command, args []string) { if flagRootCommit == "0000000000000000000000000000000000000000000000000000000000000000" { generateEmptyExecutionState( block.Header.ChainID, - flagBootstrapRandomSeed, assignments, clusterQCs, dkgData, @@ -587,7 +573,6 @@ func loadRootProtocolSnapshot(path string) (*inmem.Snapshot, error) { // given configuration. Sets the flagRootCommit variable for future reads. func generateEmptyExecutionState( chainID flow.ChainID, - randomSource []byte, assignments flow.AssignmentList, clusterQCs []*flow.QuorumCertificate, dkgData dkg.DKGData, @@ -606,6 +591,10 @@ func generateEmptyExecutionState( log.Fatal().Err(err).Msg("invalid genesis token supply") } + randomSource := make([]byte, flow.EpochSetupRandomSourceLength) + if _, err = rand.Read(randomSource); err != nil { + log.Fatal().Err(err).Msg("failed to generate a random source") + } cdcRandomSource, err := cadence.NewString(hex.EncodeToString(randomSource)) if err != nil { log.Fatal().Err(err).Msg("invalid random source") diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 033e29b6609..6890788da39 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -2,23 +2,19 @@ package cmd import ( "encoding/hex" - "os" "path/filepath" "regexp" "strings" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" utils "github.com/onflow/flow-go/cmd/bootstrap/utils" model "github.com/onflow/flow-go/model/bootstrap" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) -const finalizeHappyPathLogs = "^deterministic bootstrapping random seed" + - "collecting partner network and staking keys" + +const finalizeHappyPathLogs = "collecting partner network and staking keys" + `read \d+ partner node configuration files` + `read \d+ weights for partner nodes` + "generating internal private networking and staking keys" + @@ -52,7 +48,6 @@ const finalizeHappyPathLogs = "^deterministic bootstrapping random seed" + var finalizeHappyPathRegex = regexp.MustCompile(finalizeHappyPathLogs) func TestFinalize_HappyPath(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootCommit := unittest.StateCommitmentFixture() rootParent := unittest.StateCommitmentFixture() chainName := "main" @@ -73,9 +68,6 @@ func TestFinalize_HappyPath(t *testing.T) { flagRootParent = hex.EncodeToString(rootParent[:]) flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - // rootBlock will generate DKG and place it into bootDir/public-root-information rootBlock(nil, nil) @@ -101,233 +93,3 @@ func TestFinalize_HappyPath(t *testing.T) { assert.FileExists(t, snapshotPath) }) } - -func TestFinalize_Deterministic(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(1000) - epochCounter := uint64(0) - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - // rootBlock will generate DKG and place it into model.PathRootDKGData - rootBlock(nil, nil) - - flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) - flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) - flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - snapshotPath := filepath.Join(bootDir, model.PathRootProtocolStateSnapshot) - assert.FileExists(t, snapshotPath) - - // read snapshot - _, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // delete snapshot file - err = os.Remove(snapshotPath) - require.NoError(t, err) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - assert.FileExists(t, snapshotPath) - - // read snapshot - _, err = utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // ATTENTION: we can't use next statement because QC generation is not deterministic - // assert.Equal(t, firstSnapshot, secondSnapshot) - // Meaning we don't have a guarantee that with same input arguments we will get same QC. - // This doesn't mean that QC is invalid, but it will result in different structures, - // different QC => different service events => different result => different seal - // We need to use a different mechanism for comparing. - // ToDo: Revisit if this test case is valid at all. - }) -} - -func TestFinalize_SameSeedDifferentStateCommits(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(1000) - epochCounter := uint64(0) - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - // rootBlock will generate DKG and place it into bootDir/public-root-information - rootBlock(nil, nil) - - flagRootBlock = filepath.Join(bootDir, model.PathRootBlockData) - flagDKGDataPath = filepath.Join(bootDir, model.PathRootDKGData) - flagRootBlockVotesDir = filepath.Join(bootDir, model.DirnameRootBlockVotes) - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - snapshotPath := filepath.Join(bootDir, model.PathRootProtocolStateSnapshot) - assert.FileExists(t, snapshotPath) - - // read snapshot - snapshot1, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // delete snapshot file - err = os.Remove(snapshotPath) - require.NoError(t, err) - - // change input state commitments - rootCommit2 := unittest.StateCommitmentFixture() - rootParent2 := unittest.StateCommitmentFixture() - flagRootCommit = hex.EncodeToString(rootCommit2[:]) - flagRootParent = hex.EncodeToString(rootParent2[:]) - - finalize(nil, nil) - require.Regexp(t, finalizeHappyPathRegex, hook.logs.String()) - hook.logs.Reset() - - // check if root protocol snapshot exists - assert.FileExists(t, snapshotPath) - - // read snapshot - snapshot2, err := utils.ReadRootProtocolSnapshot(bootDir) - require.NoError(t, err) - - // current epochs - currentEpoch1 := snapshot1.Epochs().Current() - currentEpoch2 := snapshot2.Epochs().Current() - - // check dkg - dkg1, err := currentEpoch1.DKG() - require.NoError(t, err) - dkg2, err := currentEpoch2.DKG() - require.NoError(t, err) - assert.Equal(t, dkg1, dkg2) - - // check clustering - clustering1, err := currentEpoch1.Clustering() - require.NoError(t, err) - clustering2, err := currentEpoch2.Clustering() - require.NoError(t, err) - assert.Equal(t, clustering1, clustering2) - - // verify random sources are same - randomSource1, err := currentEpoch1.RandomSource() - require.NoError(t, err) - randomSource2, err := currentEpoch2.RandomSource() - require.NoError(t, err) - assert.Equal(t, randomSource1, randomSource2) - assert.Equal(t, randomSource1, deterministicSeed) - assert.Equal(t, flow.EpochSetupRandomSourceLength, len(randomSource1)) - }) -} - -func TestFinalize_InvalidRandomSeedLength(t *testing.T) { - rootCommit := unittest.StateCommitmentFixture() - rootParent := unittest.StateCommitmentFixture() - chainName := "main" - rootHeight := uint64(12332) - epochCounter := uint64(2) - - // set random seed with smaller length - deterministicSeed, err := hex.DecodeString("a12354a343234aa44bbb43") - require.NoError(t, err) - - // invalid length execution logs - expectedLogs := regexp.MustCompile("random seed provided length is not valid") - - utils.RunWithSporkBootstrapDir(t, func(bootDir, partnerDir, partnerWeights, internalPrivDir, configPath string) { - - flagOutdir = bootDir - - flagConfig = configPath - flagPartnerNodeInfoDir = partnerDir - flagPartnerWeights = partnerWeights - flagInternalNodePrivInfoDir = internalPrivDir - - flagFastKG = true - - flagRootCommit = hex.EncodeToString(rootCommit[:]) - flagRootParent = hex.EncodeToString(rootParent[:]) - flagRootChain = chainName - flagRootHeight = rootHeight - flagEpochCounter = epochCounter - flagNumViewsInEpoch = 100_000 - flagNumViewsInStakingAuction = 50_000 - flagNumViewsInDKGPhase = 2_000 - flagEpochCommitSafetyThreshold = 1_000 - - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - - hook := zeroLoggerHook{logs: &strings.Builder{}} - log = log.Hook(hook) - - finalize(nil, nil) - assert.Regexp(t, expectedLogs, hook.logs.String()) - hook.logs.Reset() - }) -} diff --git a/cmd/bootstrap/cmd/machine_account_test.go b/cmd/bootstrap/cmd/machine_account_test.go index 5fab682e561..7a1627ca3ac 100644 --- a/cmd/bootstrap/cmd/machine_account_test.go +++ b/cmd/bootstrap/cmd/machine_account_test.go @@ -31,6 +31,7 @@ func TestMachineAccountHappyPath(t *testing.T) { flagRole = "consensus" flagAddress = "189.123.123.42:3869" addr, err := flow.Mainnet.Chain().AddressAtIndex(uint64(rand.Intn(1_000_000))) + t.Logf("address is %s", addr) require.NoError(t, err) flagMachineAccountAddress = addr.HexWithPrefix() diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index d9acfff8037..f1275551657 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -1,7 +1,6 @@ package cmd import ( - "encoding/hex" "time" "github.com/spf13/cobra" @@ -60,8 +59,6 @@ func addRootBlockCmdFlags() { cmd.MarkFlagRequired(rootBlockCmd, "root-parent") cmd.MarkFlagRequired(rootBlockCmd, "root-height") - rootBlockCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(flow.EpochSetupRandomSourceLength), "The seed used to for DKG, Clustering and Cluster QC generation") - // optional parameters to influence various aspects of identity generation rootBlockCmd.Flags().BoolVar(&flagFastKG, "fast-kg", false, "use fast (centralized) random beacon key generation instead of DKG") } @@ -78,14 +75,6 @@ func rootBlock(cmd *cobra.Command, args []string) { } } - if len(flagBootstrapRandomSeed) != flow.EpochSetupRandomSourceLength { - log.Error().Int("expected", flow.EpochSetupRandomSourceLength).Int("actual", len(flagBootstrapRandomSeed)).Msg("random seed provided length is not valid") - return - } - - log.Info().Str("seed", hex.EncodeToString(flagBootstrapRandomSeed)).Msg("deterministic bootstrapping random seed") - log.Info().Msg("") - log.Info().Msg("collecting partner network and staking keys") partnerNodes := readPartnerNodeInfos() log.Info().Msg("") diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index 0883037115f..61b11379e8e 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -13,12 +13,10 @@ import ( "github.com/onflow/flow-go/cmd/bootstrap/utils" model "github.com/onflow/flow-go/model/bootstrap" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) -const rootBlockHappyPathLogs = "^deterministic bootstrapping random seed" + - "collecting partner network and staking keys" + +const rootBlockHappyPathLogs = "collecting partner network and staking keys" + `read \d+ partner node configuration files` + `read \d+ weights for partner nodes` + "generating internal private networking and staking keys" + @@ -42,7 +40,6 @@ const rootBlockHappyPathLogs = "^deterministic bootstrapping random seed" + var rootBlockHappyPathRegex = regexp.MustCompile(rootBlockHappyPathLogs) func TestRootBlock_HappyPath(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootParent := unittest.StateCommitmentFixture() chainName := "main" rootHeight := uint64(12332) @@ -62,9 +59,6 @@ func TestRootBlock_HappyPath(t *testing.T) { flagRootChain = chainName flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - hook := zeroLoggerHook{logs: &strings.Builder{}} log = log.Hook(hook) @@ -79,7 +73,6 @@ func TestRootBlock_HappyPath(t *testing.T) { } func TestRootBlock_Deterministic(t *testing.T) { - deterministicSeed := GenerateRandomSeed(flow.EpochSetupRandomSourceLength) rootParent := unittest.StateCommitmentFixture() chainName := "main" rootHeight := uint64(1000) @@ -99,9 +92,6 @@ func TestRootBlock_Deterministic(t *testing.T) { flagRootChain = chainName flagRootHeight = rootHeight - // set deterministic bootstrapping seed - flagBootstrapRandomSeed = deterministicSeed - hook := zeroLoggerHook{logs: &strings.Builder{}} log = log.Hook(hook) diff --git a/cmd/bootstrap/cmd/seal.go b/cmd/bootstrap/cmd/seal.go index 91533377a0e..1a34c394e13 100644 --- a/cmd/bootstrap/cmd/seal.go +++ b/cmd/bootstrap/cmd/seal.go @@ -41,7 +41,7 @@ func constructRootResultAndSeal( DKGPhase3FinalView: firstView + flagNumViewsInStakingAuction + flagNumViewsInDKGPhase*3 - 1, Participants: participants.Sort(order.Canonical), Assignments: assignments, - RandomSource: flagBootstrapRandomSeed, + RandomSource: GenerateRandomSeed(flow.EpochSetupRandomSourceLength), } qcsWithSignerIDs := make([]*flow.QuorumCertificateWithSignerIDs, 0, len(clusterQCs)) diff --git a/cmd/bootstrap/utils/file.go b/cmd/bootstrap/utils/file.go index b1c0585ba0e..fc5f35c7122 100644 --- a/cmd/bootstrap/utils/file.go +++ b/cmd/bootstrap/utils/file.go @@ -35,7 +35,7 @@ func ReadRootProtocolSnapshot(bootDir string) (*inmem.Snapshot, error) { func ReadRootBlock(rootBlockDataPath string) (*flow.Block, error) { bytes, err := io.ReadFile(rootBlockDataPath) if err != nil { - return nil, fmt.Errorf("could not read root block: %w", err) + return nil, fmt.Errorf("could not read root block file: %w", err) } var encodable flow.Block diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index 3f12f25edca..828e946ad81 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -498,6 +498,7 @@ func (exeNode *ExecutionNode) LoadProviderEngine( chunkDataPackRequestQueueMetrics = metrics.ChunkDataPackRequestQueueMetricsFactory(node.MetricsRegisterer) } chdpReqQueue := queue.NewHeroStore(exeNode.exeConf.chunkDataPackRequestsCacheSize, node.Logger, chunkDataPackRequestQueueMetrics) + exeNode.providerEngine, err = exeprovider.New( node.Logger, node.Tracer, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 1b1da34653e..6f480bf90c0 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -937,6 +937,7 @@ func (builder *ObserverServiceBuilder) enqueuePublicNetworkInit() { if builder.HeroCacheMetricsEnable { heroCacheCollector = metrics.NetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer) } + receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize, builder.Logger, heroCacheCollector) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 8877440b422..e0f1f30db9e 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "os" "path/filepath" "runtime" @@ -1702,9 +1701,6 @@ func (fnb *FlowNodeBuilder) Build() (Node, error) { func (fnb *FlowNodeBuilder) onStart() error { - // seed random generator - rand.Seed(time.Now().UnixNano()) - // init nodeinfo by reading the private bootstrap file if not already set if fnb.NodeID == flow.ZeroID { if err := fnb.initNodeInfo(); err != nil { diff --git a/consensus/hotstuff/committees/leader/leader_selection_test.go b/consensus/hotstuff/committees/leader/leader_selection_test.go index 7d580c76a6a..d5560cd0f40 100644 --- a/consensus/hotstuff/committees/leader/leader_selection_test.go +++ b/consensus/hotstuff/committees/leader/leader_selection_test.go @@ -203,7 +203,7 @@ func TestViewOutOfRange(t *testing.T) { _, err = leaders.LeaderForView(before) assert.Error(t, err) - before = rand.Uint64() % firstView // random view before first view + before = uint64(rand.Intn(int(firstView))) // random view before first view _, err = leaders.LeaderForView(before) assert.Error(t, err) }) diff --git a/consensus/hotstuff/signature/block_signer_decoder_test.go b/consensus/hotstuff/signature/block_signer_decoder_test.go index 4325b50c7b7..972b639884e 100644 --- a/consensus/hotstuff/signature/block_signer_decoder_test.go +++ b/consensus/hotstuff/signature/block_signer_decoder_test.go @@ -112,7 +112,8 @@ func (s *blockSignerDecoderSuite) Test_EpochTransition() { blockView := s.block.Header.View parentView := s.block.Header.ParentView epoch1Committee := s.allConsensus - epoch2Committee := s.allConsensus.SamplePct(.8) + epoch2Committee, err := s.allConsensus.SamplePct(.8) + require.NoError(s.T(), err) *s.committee = *hotstuff.NewDynamicCommittee(s.T()) s.committee.On("IdentitiesByEpoch", parentView).Return(epoch1Committee, nil).Maybe() diff --git a/consensus/hotstuff/signature/randombeacon_inspector_test.go b/consensus/hotstuff/signature/randombeacon_inspector_test.go index 5784577f668..04016d97fe7 100644 --- a/consensus/hotstuff/signature/randombeacon_inspector_test.go +++ b/consensus/hotstuff/signature/randombeacon_inspector_test.go @@ -2,10 +2,9 @@ package signature import ( "errors" - mrand "math/rand" + "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,6 +23,7 @@ func TestRandomBeaconInspector(t *testing.T) { type randomBeaconSuite struct { suite.Suite + rng *rand.Rand n int threshold int kmac hash.Hasher @@ -39,9 +39,10 @@ func (rs *randomBeaconSuite) SetupTest() { rs.threshold = signature.RandomBeaconThreshold(rs.n) // generate threshold keys - mrand.Seed(time.Now().UnixNano()) + rs.rng = unittest.GetPRG(rs.T()) + seed := make([]byte, crypto.SeedMinLenDKG) - _, err := mrand.Read(seed) + _, err := rs.rng.Read(seed) require.NoError(rs.T(), err) rs.skShares, rs.pkShares, rs.pkGroup, err = crypto.BLSThresholdKeyGen(rs.n, rs.threshold, seed) require.NoError(rs.T(), err) @@ -57,7 +58,7 @@ func (rs *randomBeaconSuite) SetupTest() { for i := 0; i < rs.n; i++ { rs.signers = append(rs.signers, i) } - mrand.Shuffle(rs.n, func(i, j int) { + rs.rng.Shuffle(rs.n, func(i, j int) { rs.signers[i], rs.signers[j] = rs.signers[j], rs.signers[i] }) } @@ -166,7 +167,7 @@ func (rs *randomBeaconSuite) TestInvalidSignerIndex() { func (rs *randomBeaconSuite) TestInvalidSignature() { follower, err := NewRandomBeaconInspector(rs.pkGroup, rs.pkShares, rs.threshold, rs.thresholdSignatureMessage) require.NoError(rs.T(), err) - index := mrand.Intn(rs.n) // random signer + index := rs.rng.Intn(rs.n) // random signer share, err := rs.skShares[index].Sign(rs.thresholdSignatureMessage, rs.kmac) require.NoError(rs.T(), err) diff --git a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go index 691209cb179..d3472fcbcd8 100644 --- a/consensus/hotstuff/timeoutcollector/timeout_collector_test.go +++ b/consensus/hotstuff/timeoutcollector/timeout_collector_test.go @@ -174,7 +174,6 @@ func (s *TimeoutCollectorTestSuite) TestAddTimeout_TONotifications() { expectedHighestQC := timeouts[len(timeouts)-1].NewestQC // shuffle timeouts in random order - rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(timeouts), func(i, j int) { timeouts[i], timeouts[j] = timeouts[j], timeouts[i] }) diff --git a/consensus/hotstuff/validator/validator_test.go b/consensus/hotstuff/validator/validator_test.go index 8dbf03736d1..9c8f052d7cf 100644 --- a/consensus/hotstuff/validator/validator_test.go +++ b/consensus/hotstuff/validator/validator_test.go @@ -5,7 +5,6 @@ import ( "fmt" "math/rand" "testing" - "time" "github.com/onflow/flow-go/module/signature" @@ -46,7 +45,6 @@ type ProposalSuite struct { func (ps *ProposalSuite) SetupTest() { // the leader is a random node for now - rand.Seed(time.Now().UnixNano()) ps.finalized = uint64(rand.Uint32() + 1) ps.participants = unittest.IdentityListFixture(8, unittest.WithRole(flow.RoleConsensus)) ps.leader = ps.participants[0] @@ -753,7 +751,6 @@ func (s *TCSuite) SetupTest() { s.indices, err = signature.EncodeSignersToIndices(s.participants.NodeIDs(), s.signers.NodeIDs()) require.NoError(s.T(), err) - rand.Seed(time.Now().UnixNano()) view := uint64(int(rand.Uint32()) + len(s.participants)) highQCViews := make([]uint64, 0, len(s.signers)) diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go index ef1fa25df85..fe574e4f283 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -597,7 +596,7 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -745,7 +744,7 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go index 01497d59ff5..a4fe0e03dde 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sync" "testing" - "time" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -647,7 +646,7 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -880,7 +879,7 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) diff --git a/consensus/integration/network_test.go b/consensus/integration/network_test.go index 181e3e79adc..ec263ff93a0 100644 --- a/consensus/integration/network_test.go +++ b/consensus/integration/network_test.go @@ -158,7 +158,11 @@ func (n *Network) publish(event interface{}, channel channels.Channel, targetIDs // Engines attached to the same channel on other nodes. The targeted nodes are selected based on the selector. // In this test helper implementation, multicast uses submit method under the hood. func (n *Network) multicast(event interface{}, channel channels.Channel, num uint, targetIDs ...flow.Identifier) error { - targetIDs = flow.Sample(num, targetIDs...) + var err error + targetIDs, err = flow.Sample(num, targetIDs...) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } return n.submit(event, channel, targetIDs...) } diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index b3f90233c4f..183388df135 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -415,7 +415,7 @@ func createNode( notifier.AddConsumer(logConsumer) cleaner := &storagemock.Cleaner{} - cleaner.On("RunGC") + cleaner.On("RunGC").Return(nil) require.Equal(t, participant.nodeInfo.NodeID, localID) privateKeys, err := participant.nodeInfo.PrivateKeys() diff --git a/engine/access/relay/example_test.go b/engine/access/relay/example_test.go index 6574dce4567..3d343535547 100644 --- a/engine/access/relay/example_test.go +++ b/engine/access/relay/example_test.go @@ -1,8 +1,8 @@ package relay_test import ( + "encoding/hex" "fmt" - "math/rand" "github.com/rs/zerolog" @@ -21,10 +21,10 @@ func Example() { logger := zerolog.Nop() splitterNet := splitterNetwork.NewNetwork(net, logger) - // generate a random origin ID + // generate an origin ID var id flow.Identifier - rand.Seed(0) - rand.Read(id[:]) + bytes, _ := hex.DecodeString("0194fdc2fa2ffcc041d3ff12045b73c86e4ff95ff662a5eee82abdf44a2d0b75") + copy(id[:], bytes) // create engines engineProcessFunc := func(engineName string) testnet.EngineProcessFunc { diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 69bde45c23b..28cec8fd5c6 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -285,7 +285,6 @@ func (suite *RestAPITestSuite) TestGetBlock() { defer cancel() // replace one ID with a block ID for which the storage returns a not found error - rand.Seed(time.Now().Unix()) invalidBlockIndex := rand.Intn(len(testBlocks)) invalidID := unittest.IdentifierFixture() suite.blocks.On("ByID", invalidID).Return(nil, storage.ErrNotFound).Once() diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index f59ab0dffe4..b34cc4e34cf 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -331,7 +331,10 @@ func executionNodesForBlockID( } // randomly choose upto maxExecutionNodesCnt identities - executionIdentitiesRandom := subsetENs.Sample(maxExecutionNodesCnt) + executionIdentitiesRandom, err := subsetENs.Sample(maxExecutionNodesCnt) + if err != nil { + return nil, fmt.Errorf("sampling failed: %w", err) + } if len(executionIdentitiesRandom) == 0 { return nil, fmt.Errorf("no matching execution node found for block ID %v", blockID) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 5c3445faaa0..c741e9b639d 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -3,9 +3,7 @@ package backend import ( "context" "fmt" - "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -57,7 +55,7 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - rand.Seed(time.Now().UnixNano()) + suite.log = zerolog.New(zerolog.NewConsoleWriter()) suite.state = new(protocol.State) suite.snapshot = new(protocol.Snapshot) diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 796c3cba5c2..7cff5273971 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -130,7 +130,10 @@ func (b *backendTransactions) chooseCollectionNodes(tx *flow.TransactionBody, sa } // select a random subset of collection nodes from the cluster to be tried in order - targetNodes := txCluster.Sample(sampleSize) + targetNodes, err := txCluster.Sample(sampleSize) + if err != nil { + return nil, fmt.Errorf("sampling failed: %w", err) + } // collect the addresses of all the chosen collection nodes var targetAddrs = make([]string, len(targetNodes)) diff --git a/engine/access/state_stream/api_test.go b/engine/access/state_stream/api_test.go index 55268439910..ad52c7d7d78 100644 --- a/engine/access/state_stream/api_test.go +++ b/engine/access/state_stream/api_test.go @@ -3,9 +3,8 @@ package state_stream import ( "bytes" "context" - "math/rand" + "crypto/rand" "testing" - "time" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -35,7 +34,7 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - rand.Seed(time.Now().UnixNano()) + suite.headers = storagemock.NewHeaders(suite.T()) suite.seals = storagemock.NewSeals(suite.T()) suite.results = storagemock.NewExecutionResults(suite.T()) diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index c39b5f578c0..39e3683ca3f 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -2,9 +2,7 @@ package compliance import ( "errors" - "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -66,7 +64,6 @@ type CommonSuite struct { func (cs *CommonSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) block := unittest.ClusterBlockFixture() cs.head = &block diff --git a/engine/collection/message_hub/message_hub_test.go b/engine/collection/message_hub/message_hub_test.go index 16aa4e729a7..52ca114ec44 100644 --- a/engine/collection/message_hub/message_hub_test.go +++ b/engine/collection/message_hub/message_hub_test.go @@ -2,7 +2,6 @@ package message_hub import ( "context" - "math/rand" "sync" "testing" "time" @@ -69,7 +68,6 @@ type MessageHubSuite struct { func (s *MessageHubSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) // initialize the paramaters s.cluster = unittest.IdentityListFixture(3, diff --git a/engine/collection/synchronization/engine.go b/engine/collection/synchronization/engine.go index 02d75c392a6..9be4b7ba026 100644 --- a/engine/collection/synchronization/engine.go +++ b/engine/collection/synchronization/engine.go @@ -5,7 +5,6 @@ package synchronization import ( "errors" "fmt" - "math/rand" "time" "github.com/hashicorp/go-multierror" @@ -27,6 +26,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/cluster" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/rand" ) // defaultSyncResponseQueueCapacity maximum capacity of sync responses queue @@ -361,9 +361,15 @@ func (e *Engine) pollHeight() { return } + nonce, err := rand.Uint64() + if err != nil { + e.log.Error().Err(err).Msg("nonce generation failed") + return + } + // send the request for synchronization req := &messages.SyncRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, Height: head.Height, } err = e.con.Multicast(req, synccore.DefaultPollNodes, e.participants.NodeIDs()...) @@ -379,12 +385,17 @@ func (e *Engine) sendRequests(ranges []chainsync.Range, batches []chainsync.Batc var errs *multierror.Error for _, ran := range ranges { + nonce, err := rand.Uint64() + if err != nil { + e.log.Error().Err(err).Msg("nonce generation failed") + return + } req := &messages.RangeRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, FromHeight: ran.From, ToHeight: ran.To, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit range request: %w", err)) continue @@ -399,11 +410,16 @@ func (e *Engine) sendRequests(ranges []chainsync.Range, batches []chainsync.Batc } for _, batch := range batches { + nonce, err := rand.Uint64() + if err != nil { + e.log.Error().Err(err).Msg("nonce generation failed") + return + } req := &messages.BatchRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, BlockIDs: batch.BlockIDs, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, e.participants.NodeIDs()...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit batch request: %w", err)) continue diff --git a/engine/collection/synchronization/engine_test.go b/engine/collection/synchronization/engine_test.go index 06799cc6ddf..775372c12cc 100644 --- a/engine/collection/synchronization/engine_test.go +++ b/engine/collection/synchronization/engine_test.go @@ -58,7 +58,6 @@ type SyncSuite struct { func (ss *SyncSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection)) diff --git a/engine/common/follower/engine.go b/engine/common/follower/engine.go index b261b4fcd24..66f13c34652 100644 --- a/engine/common/follower/engine.go +++ b/engine/common/follower/engine.go @@ -348,7 +348,10 @@ func (e *Engine) processBlockProposal(originID flow.Identifier, proposal *messag // good moment to potentially kick-off a garbage collection of the DB // NOTE: this is only effectively run every 1000th calls, which corresponds // to every 1000th successfully processed block - e.cleaner.RunGC() + err = e.cleaner.RunGC() + if err != nil { + return fmt.Errorf("run GC failed: %w", err) + } return nil } diff --git a/engine/common/follower/engine_test.go b/engine/common/follower/engine_test.go index 36a687e8c3b..1f7e690f020 100644 --- a/engine/common/follower/engine_test.go +++ b/engine/common/follower/engine_test.go @@ -69,7 +69,7 @@ func (s *Suite) SetupTest() { s.me.On("NodeID").Return(nodeID).Maybe() s.net.On("Register", mock.Anything, mock.Anything).Return(s.con, nil) - s.cleaner.On("RunGC").Return().Maybe() + s.cleaner.On("RunGC").Return(nil).Maybe() s.state.On("Final").Return(s.snapshot).Maybe() s.cache.On("PruneByView", mock.Anything).Return().Maybe() s.cache.On("Size", mock.Anything).Return(uint(0)).Maybe() diff --git a/engine/common/requester/engine.go b/engine/common/requester/engine.go index f83a2d03780..8df7f3855f3 100644 --- a/engine/common/requester/engine.go +++ b/engine/common/requester/engine.go @@ -3,7 +3,6 @@ package requester import ( "fmt" "math" - "math/rand" "time" "github.com/rs/zerolog" @@ -20,6 +19,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) // HandleFunc is a function provided to the requester engine to handle an entity @@ -51,7 +51,6 @@ type Engine struct { items map[flow.Identifier]*Item requests map[uint64]*messages.EntityRequest forcedDispatchOngoing *atomic.Bool // to ensure only trigger dispatching logic once at any time - rng *rand.Rand } // New creates a new requester engine, operating on the provided network channel, and requesting entities from a node @@ -117,7 +116,6 @@ func New(log zerolog.Logger, metrics module.EngineMetrics, net network.Network, items: make(map[flow.Identifier]*Item), // holds all pending items requests: make(map[uint64]*messages.EntityRequest), // holds all sent requests forcedDispatchOngoing: atomic.NewBool(false), - rng: rand.New(rand.NewSource(time.Now().UnixNano())), } // register the engine with the network layer and store the conduit @@ -319,7 +317,12 @@ func (e *Engine) dispatchRequest() (bool, error) { for k := range e.items { rndItems = append(rndItems, e.items[k].EntityID) } - e.rng.Shuffle(len(rndItems), func(i, j int) { rndItems[i], rndItems[j] = rndItems[j], rndItems[i] }) + err = rand.Shuffle(uint(len(rndItems)), func(i, j uint) { + rndItems[i], rndItems[j] = rndItems[j], rndItems[i] + }) + if err != nil { + return false, fmt.Errorf("shuffle failed: %w", err) + } // go through each item and decide if it should be requested again now := time.Now().UTC() @@ -364,7 +367,11 @@ func (e *Engine) dispatchRequest() (bool, error) { if len(providers) == 0 { return false, fmt.Errorf("no valid providers available") } - providerID = providers.Sample(1)[0].NodeID + id, err := providers.Sample(1) + if err != nil { + return false, fmt.Errorf("sampling failed: %w", err) + } + providerID = id[0].NodeID } // add item to list and set retry parameters @@ -396,9 +403,14 @@ func (e *Engine) dispatchRequest() (bool, error) { return false, nil } + nonce, err := rand.Uint64() + if err != nil { + return false, fmt.Errorf("nonce generation failed %w", err) + } + // create a batch request, send it and store it for reference req := &messages.EntityRequest{ - Nonce: e.rng.Uint64(), + Nonce: nonce, EntityIDs: entityIDs, } diff --git a/engine/common/requester/engine_test.go b/engine/common/requester/engine_test.go index a2a259d44dc..553386c85d6 100644 --- a/engine/common/requester/engine_test.go +++ b/engine/common/requester/engine_test.go @@ -29,7 +29,6 @@ func TestEntityByID(t *testing.T) { request := Engine{ unit: engine.NewUnit(), items: make(map[flow.Identifier]*Item), - rng: rand.New(rand.NewSource(0)), } now := time.Now().UTC() @@ -136,7 +135,6 @@ func TestDispatchRequestVarious(t *testing.T) { items: items, requests: make(map[uint64]*messages.EntityRequest), selector: filter.HasNodeID(targetID), - rng: rand.New(rand.NewSource(0)), } dispatched, err := request.dispatchRequest() require.NoError(t, err) @@ -213,7 +211,6 @@ func TestDispatchRequestBatchSize(t *testing.T) { items: items, requests: make(map[uint64]*messages.EntityRequest), selector: filter.Any, - rng: rand.New(rand.NewSource(0)), } dispatched, err := request.dispatchRequest() require.NoError(t, err) @@ -293,7 +290,6 @@ func TestOnEntityResponseValid(t *testing.T) { close(done) } }, - rng: rand.New(rand.NewSource(0)), } request.items[iwanted1.EntityID] = iwanted1 @@ -377,7 +373,6 @@ func TestOnEntityIntegrityCheck(t *testing.T) { selector: filter.HasNodeID(targetID), create: func() flow.Entity { return &flow.Collection{} }, handle: func(flow.Identifier, flow.Entity) { close(called) }, - rng: rand.New(rand.NewSource(0)), } request.items[iwanted.EntityID] = iwanted diff --git a/engine/common/rpc/convert/convert_test.go b/engine/common/rpc/convert/convert_test.go index a98f828d0f6..ec0c3dc930c 100644 --- a/engine/common/rpc/convert/convert_test.go +++ b/engine/common/rpc/convert/convert_test.go @@ -2,7 +2,7 @@ package convert_test import ( "bytes" - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/assert" diff --git a/engine/common/splitter/network/example_test.go b/engine/common/splitter/network/example_test.go index b94f9e8a70e..fb11d960a83 100644 --- a/engine/common/splitter/network/example_test.go +++ b/engine/common/splitter/network/example_test.go @@ -1,8 +1,8 @@ package network_test import ( + "encoding/hex" "fmt" - "math/rand" "github.com/rs/zerolog" @@ -20,10 +20,10 @@ func Example() { logger := zerolog.Nop() splitterNet := splitterNetwork.NewNetwork(net, logger) - // generate a random origin ID + // generate an origin ID var id flow.Identifier - rand.Seed(0) - rand.Read(id[:]) + bytes, _ := hex.DecodeString("0194fdc2fa2ffcc041d3ff12045b73c86e4ff95ff662a5eee82abdf44a2d0b75") + copy(id[:], bytes) // create engines engineProcessFunc := func(engineID int) testnet.EngineProcessFunc { diff --git a/engine/common/synchronization/engine.go b/engine/common/synchronization/engine.go index e31880e30e0..7a1e5fabfc2 100644 --- a/engine/common/synchronization/engine.go +++ b/engine/common/synchronization/engine.go @@ -4,7 +4,6 @@ package synchronization import ( "fmt" - "math/rand" "time" "github.com/hashicorp/go-multierror" @@ -23,6 +22,7 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/rand" ) // defaultSyncResponseQueueCapacity maximum capacity of sync responses queue @@ -357,16 +357,22 @@ func (e *Engine) pollHeight() { head := e.finalizedHeader.Get() participants := e.participantsProvider.Identifiers() + nonce, err := rand.Uint64() + if err != nil { + e.log.Warn().Err(err).Msg("nonce generation failed") + return + } + // send the request for synchronization req := &messages.SyncRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, Height: head.Height, } e.log.Debug(). Uint64("height", req.Height). Uint64("range_nonce", req.Nonce). Msg("sending sync request") - err := e.con.Multicast(req, synccore.DefaultPollNodes, participants...) + err = e.con.Multicast(req, synccore.DefaultPollNodes, participants...) if err != nil { e.log.Warn().Err(err).Msg("sending sync request to poll heights failed") return @@ -378,9 +384,15 @@ func (e *Engine) pollHeight() { func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsync.Range, batches []chainsync.Batch) { var errs *multierror.Error + nonce, err := rand.Uint64() + if err != nil { + e.log.Error().Err(err).Msg("nonce generation failed") + return + } + for _, ran := range ranges { req := &messages.RangeRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, FromHeight: ran.From, ToHeight: ran.To, } @@ -399,11 +411,16 @@ func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsy } for _, batch := range batches { + nonce, err := rand.Uint64() + if err != nil { + e.log.Error().Err(err).Msg("nonce generation failed") + return + } req := &messages.BatchRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, BlockIDs: batch.BlockIDs, } - err := e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) + err = e.con.Multicast(req, synccore.DefaultBlockRequestNodes, participants...) if err != nil { errs = multierror.Append(errs, fmt.Errorf("could not submit batch request: %w", err)) continue diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index c38e101484f..5e6cac8bce6 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -59,7 +59,6 @@ type SyncSuite struct { func (ss *SyncSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus)) diff --git a/engine/consensus/approvals/request_tracker.go b/engine/consensus/approvals/request_tracker.go index 02520d10ee7..36c7a208078 100644 --- a/engine/consensus/approvals/request_tracker.go +++ b/engine/consensus/approvals/request_tracker.go @@ -1,8 +1,9 @@ package approvals import ( + "crypto/rand" + "encoding/binary" "fmt" - "math/rand" "sync" "time" @@ -28,30 +29,45 @@ type RequestTrackerItem struct { // NewRequestTrackerItem instantiates a new RequestTrackerItem where the // NextTimeout is evaluated to the current time plus a random blackout period // contained between min and max. -func NewRequestTrackerItem(blackoutPeriodMin, blackoutPeriodMax int) RequestTrackerItem { +func NewRequestTrackerItem(blackoutPeriodMin, blackoutPeriodMax int) (RequestTrackerItem, error) { item := RequestTrackerItem{ blackoutPeriodMin: blackoutPeriodMin, blackoutPeriodMax: blackoutPeriodMax, } - item.NextTimeout = randBlackout(blackoutPeriodMin, blackoutPeriodMax) - return item + var err error + item.NextTimeout, err = randBlackout(blackoutPeriodMin, blackoutPeriodMax) + if err != nil { + return RequestTrackerItem{}, err + } + + return item, err } // Update creates a _new_ RequestTrackerItem with incremented request number and updated NextTimeout. -func (i RequestTrackerItem) Update() RequestTrackerItem { +func (i RequestTrackerItem) Update() (RequestTrackerItem, error) { i.Requests++ - i.NextTimeout = randBlackout(i.blackoutPeriodMin, i.blackoutPeriodMax) - return i + var err error + i.NextTimeout, err = randBlackout(i.blackoutPeriodMin, i.blackoutPeriodMax) + if err != nil { + return RequestTrackerItem{}, err + } + return i, err } func (i RequestTrackerItem) IsBlackout() bool { return time.Now().Before(i.NextTimeout) } -func randBlackout(min int, max int) time.Time { - blackoutSeconds := rand.Intn(max-min+1) + min +func randBlackout(min int, max int) (time.Time, error) { + buff := make([]byte, 8) + if _, err := rand.Read(buff); err != nil { + return time.Now(), fmt.Errorf("failed to generate randomness") + } + rand := binary.LittleEndian.Uint64(buff) + + blackoutSeconds := rand%uint64(max-min+1) + uint64(min) blackout := time.Now().Add(time.Duration(blackoutSeconds) * time.Second) - return blackout + return blackout, nil } /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -93,10 +109,14 @@ func (rt *RequestTracker) TryUpdate(result *flow.ExecutionResult, incorporatedBl rt.lock.Lock() defer rt.lock.Unlock() item, ok := rt.index[resultID][incorporatedBlockID][chunkIndex] + var err error if !ok { - item = NewRequestTrackerItem(rt.blackoutPeriodMin, rt.blackoutPeriodMax) - err := rt.set(resultID, result.BlockID, incorporatedBlockID, chunkIndex, item) + item, err = NewRequestTrackerItem(rt.blackoutPeriodMin, rt.blackoutPeriodMax) + if err != nil { + return item, false, fmt.Errorf("could not create tracker item: %w", err) + } + err = rt.set(resultID, result.BlockID, incorporatedBlockID, chunkIndex, item) if err != nil { return item, false, fmt.Errorf("could not set created tracker item: %w", err) } @@ -104,7 +124,10 @@ func (rt *RequestTracker) TryUpdate(result *flow.ExecutionResult, incorporatedBl canUpdate := !item.IsBlackout() if canUpdate { - item = item.Update() + item, err = item.Update() + if err != nil { + return item, false, fmt.Errorf("could not update tracker item: %w", err) + } rt.index[resultID][incorporatedBlockID][chunkIndex] = item } diff --git a/engine/consensus/approvals/verifying_assignment_collector.go b/engine/consensus/approvals/verifying_assignment_collector.go index 118627db3bc..5a4c8b588de 100644 --- a/engine/consensus/approvals/verifying_assignment_collector.go +++ b/engine/consensus/approvals/verifying_assignment_collector.go @@ -2,7 +2,6 @@ package approvals import ( "fmt" - "math/rand" "sync" "github.com/rs/zerolog" @@ -15,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/mempool" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/utils/rand" ) // **Emergency-sealing parameters** @@ -360,9 +360,15 @@ func (ac *VerifyingAssignmentCollector) RequestMissingApprovals(observation cons ) } + nonce, err := rand.Uint64() + if err != nil { + log.Error().Err(err). + Msgf("nonce generation falied") + } + // prepare the request req := &messages.ApprovalRequest{ - Nonce: rand.Uint64(), + Nonce: nonce, ResultID: ac.ResultID(), ChunkIndex: chunkIndex, } diff --git a/engine/consensus/compliance/core.go b/engine/consensus/compliance/core.go index 8f6a11c0eb3..994b45c56df 100644 --- a/engine/consensus/compliance/core.go +++ b/engine/consensus/compliance/core.go @@ -242,7 +242,10 @@ func (c *Core) OnBlockProposal(originID flow.Identifier, proposal *messages.Bloc // good moment to potentially kick-off a garbage collection of the DB // NOTE: this is only effectively run every 1000th calls, which corresponds // to every 1000th successfully processed block - c.cleaner.RunGC() + err = c.cleaner.RunGC() + if err != nil { + return fmt.Errorf("run GC failed: %w", err) + } return nil } diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index 186ab4040b6..162f7665290 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -2,9 +2,7 @@ package compliance import ( "errors" - "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -82,7 +80,6 @@ type CommonSuite struct { func (cs *CommonSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) // initialize the paramaters cs.participants = unittest.IdentityListFixture(3, @@ -113,7 +110,7 @@ func (cs *CommonSuite) SetupTest() { // set up storage cleaner cs.cleaner = &storage.Cleaner{} - cs.cleaner.On("RunGC").Return() + cs.cleaner.On("RunGC").Return(nil) // set up header storage mock cs.headers = &storage.Headers{} diff --git a/engine/consensus/message_hub/message_hub_test.go b/engine/consensus/message_hub/message_hub_test.go index 97351ba649b..62f1765ead6 100644 --- a/engine/consensus/message_hub/message_hub_test.go +++ b/engine/consensus/message_hub/message_hub_test.go @@ -2,7 +2,6 @@ package message_hub import ( "context" - "math/rand" "sync" "testing" "time" @@ -66,7 +65,6 @@ type MessageHubSuite struct { func (s *MessageHubSuite) SetupTest() { // seed the RNG - rand.Seed(time.Now().UnixNano()) // initialize the paramaters s.participants = unittest.IdentityListFixture(3, diff --git a/engine/execution/computation/manager.go b/engine/execution/computation/manager.go index bcd42e98d5f..e7f296e540d 100644 --- a/engine/execution/computation/manager.go +++ b/engine/execution/computation/manager.go @@ -4,9 +4,7 @@ import ( "context" "encoding/hex" "fmt" - "math/rand" "strings" - "sync" "time" jsoncdc "github.com/onflow/cadence/encoding/json" @@ -26,6 +24,7 @@ import ( "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/debug" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -75,8 +74,6 @@ type Manager struct { derivedChainData *derived.DerivedChainData scriptLogThreshold time.Duration scriptExecutionTimeLimit time.Duration - rngLock *sync.Mutex - rng *rand.Rand } func New( @@ -145,8 +142,6 @@ func New( derivedChainData: derivedChainData, scriptLogThreshold: params.ScriptLogThreshold, scriptExecutionTimeLimit: params.ScriptExecutionTimeLimit, - rngLock: &sync.Mutex{}, - rng: rand.New(rand.NewSource(time.Now().UnixNano())), } return &e, nil @@ -171,9 +166,10 @@ func (e *Manager) ExecuteScript( // scripts might not be unique so we use this extra tracker to follow their logs // TODO: this is a temporary measure, we could remove this in the future if e.log.Debug().Enabled() { - e.rngLock.Lock() - trackerID := e.rng.Uint32() - e.rngLock.Unlock() + trackerID, err := rand.Uint32() + if err != nil { + return nil, fmt.Errorf("failed to generate tracker id: %w", err) + } trackedLogger := e.log.With().Hex("script_hex", code).Uint32("trackerID", trackerID).Logger() trackedLogger.Debug().Msg("script is sent for execution") diff --git a/engine/execution/ingestion/engine_test.go b/engine/execution/ingestion/engine_test.go index 75e8f8c0a14..50b28ef142f 100644 --- a/engine/execution/ingestion/engine_test.go +++ b/engine/execution/ingestion/engine_test.go @@ -96,8 +96,7 @@ func runWithEngine(t *testing.T, f func(testingContext)) { // generates signing identity including staking key for signing seed := make([]byte, crypto.KeyGenSeedMinLenBLSBLS12381) - n, err := rand.Read(seed) - require.Equal(t, n, crypto.KeyGenSeedMinLenBLSBLS12381) + _, err := rand.Read(seed) require.NoError(t, err) sk, err := crypto.GeneratePrivateKey(crypto.BLSBLS12381, seed) require.NoError(t, err) @@ -1386,8 +1385,7 @@ func newIngestionEngine(t *testing.T, ps *mocks.ProtocolState, es *mocks.Executi // generates signing identity including staking key for signing seed := make([]byte, crypto.KeyGenSeedMinLenBLSBLS12381) - n, err := rand.Read(seed) - require.Equal(t, n, crypto.KeyGenSeedMinLenBLSBLS12381) + _, err = rand.Read(seed) require.NoError(t, err) sk, err := crypto.GeneratePrivateKey(crypto.BLSBLS12381, seed) require.NoError(t, err) diff --git a/engine/execution/provider/engine.go b/engine/execution/provider/engine.go index bea81dc26b5..8217df4187c 100644 --- a/engine/execution/provider/engine.go +++ b/engine/execution/provider/engine.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "time" "github.com/rs/zerolog" @@ -25,6 +24,7 @@ import ( "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) type ProviderEngine interface { @@ -315,12 +315,20 @@ func (e *Engine) deliverChunkDataResponse(chunkDataPack *flow.ChunkDataPack, req // sends requested chunk data pack to the requester deliveryStartTime := time.Now() + nonce, err := rand.Uint64() + if err != nil { + lg.Error(). + Err(err). + Msg("could not generate nonce") + return + } + response := &messages.ChunkDataResponse{ ChunkDataPack: *chunkDataPack, - Nonce: rand.Uint64(), + Nonce: nonce, } - err := e.chunksConduit.Unicast(response, requesterId) + err = e.chunksConduit.Unicast(response, requesterId) if err != nil { lg.Warn(). Err(err). diff --git a/engine/execution/provider/engine_test.go b/engine/execution/provider/engine_test.go index 1411061b123..9346bfe02df 100644 --- a/engine/execution/provider/engine_test.go +++ b/engine/execution/provider/engine_test.go @@ -98,7 +98,6 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - e, err := New( unittest.Logger(), trace.NewNoopTracer(), @@ -157,7 +156,6 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) - e, err := New( unittest.Logger(), trace.NewNoopTracer(), diff --git a/engine/protocol/api_test.go b/engine/protocol/api_test.go index e2b7234eb42..4025f612513 100644 --- a/engine/protocol/api_test.go +++ b/engine/protocol/api_test.go @@ -2,9 +2,7 @@ package protocol import ( "context" - "math/rand" "testing" - "time" "github.com/stretchr/testify/suite" @@ -37,7 +35,7 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - rand.Seed(time.Now().UnixNano()) + suite.snapshot = new(protocol.Snapshot) suite.state = new(protocol.State) diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 673cc38e7af..48fdf096e86 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -274,13 +274,15 @@ func CollectionNode(t *testing.T, ctx irrecoverable.SignalerContext, hub *stub.H coll, err := collections.ByID(collID) return coll, err } + + store := queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()) providerEngine, err := provider.New( node.Log, node.Metrics, node.Net, node.Me, node.State, - queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()), + store, uint(1000), channels.ProvideCollections, selector, @@ -582,6 +584,8 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit ) require.NoError(t, err) + store := queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()) + pusherEngine, err := executionprovider.New( node.Log, node.Tracer, @@ -590,7 +594,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit execState, metricsCollector, checkAuthorizedAtBlock, - queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()), + store, executionprovider.DefaultChunkDataPackRequestWorker, executionprovider.DefaultChunkDataPackQueryTimeout, executionprovider.DefaultChunkDataPackDeliveryTimeout, diff --git a/engine/verification/requester/requester.go b/engine/verification/requester/requester.go index 10f91780c72..2285da61025 100644 --- a/engine/verification/requester/requester.go +++ b/engine/verification/requester/requester.go @@ -331,8 +331,11 @@ func (e *Engine) requestChunkDataPack(request *verification.ChunkDataPackRequest } // publishes the chunk data request to the network - targetIDs := request.SampleTargets(int(e.requestTargets)) - err := e.con.Publish(req, targetIDs...) + targetIDs, err := request.SampleTargets(int(e.requestTargets)) + if err != nil { + return fmt.Errorf("target sampling failed: %w", err) + } + err = e.con.Publish(req, targetIDs...) if err != nil { return fmt.Errorf("could not publish chunk data pack request for chunk (id=%s): %w", request.ChunkID, err) } diff --git a/engine/verification/verifier/engine_test.go b/engine/verification/verifier/engine_test.go index 90df4264a7e..e70a1b6557e 100644 --- a/engine/verification/verifier/engine_test.go +++ b/engine/verification/verifier/engine_test.go @@ -78,8 +78,7 @@ func (suite *VerifierEngineTestSuite) SetupTest() { // // generates signing and verification keys seed := make([]byte, crypto.KeyGenSeedMinLenBLSBLS12381) - n, err := rand.Read(seed) - require.Equal(suite.T(), n, crypto.KeyGenSeedMinLenBLSBLS12381) + _, err := rand.Read(seed) require.NoError(suite.T(), err) // creates private key of verification node diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index 61efcbacbd0..764b235806b 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -1,10 +1,14 @@ package environment import ( + "crypto/sha256" "encoding/binary" - "math/rand" + "hash" "sync" + "golang.org/x/crypto/hkdf" + + "github.com/onflow/flow-go/crypto/random" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/state" "github.com/onflow/flow-go/fvm/tracing" @@ -13,8 +17,7 @@ import ( ) type UnsafeRandomGenerator interface { - // UnsafeRandom returns a random uint64, where the process of random number - // derivation is not cryptographically secure. + // UnsafeRandom returns a random uint64 UnsafeRandom() (uint64, error) } @@ -23,7 +26,7 @@ type unsafeRandomGenerator struct { blockHeader *flow.Header - rng *rand.Rand + rng random.Rand seedOnce sync.Once } @@ -76,14 +79,21 @@ func (gen *unsafeRandomGenerator) seed() { // header ID. The random number generator will be used by the // UnsafeRandom function. id := gen.blockHeader.ID() - source := rand.NewSource(int64(binary.BigEndian.Uint64(id[:]))) - gen.rng = rand.New(source) + // extract the entropy from `id` and expand it into the required seed + hkdf := hkdf.New(func() hash.Hash { return sha256.New() }, id[:], nil, nil) + seed := make([]byte, random.Chacha20SeedLen) + hkdf.Read(seed) + // initialize a fresh CSPRNG with the seed (crypto-secure PRG) + source, err := random.NewChacha20PRG(seed, []byte{}) + if err != nil { + return + } + gen.rng = source }) } -// UnsafeRandom returns a random uint64, where the process of random number -// derivation is not cryptographically secure. -// this is not thread safe, due to gen.rng.Read(buf). +// UnsafeRandom returns a random uint64 using the underlying PRG (currently using a crypto-secure one). +// this is not thread safe, due to the gen.rng instance currently used. // Its also not thread safe because each thread needs to be deterministically seeded with a different seed. // This is Ok because a single transaction has a single UnsafeRandomGenerator and is run in a single thread. func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { @@ -95,9 +105,7 @@ func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { return 0, errors.NewOperationNotSupportedError("UnsafeRandom") } - // TODO (ramtin) return errors this assumption that this always succeeds - // might not be true buf := make([]byte, 8) - _, _ = gen.rng.Read(buf) // Always succeeds, no need to check error + gen.rng.Read(buf) return binary.LittleEndian.Uint64(buf), nil } diff --git a/fvm/fvm_bench_test.go b/fvm/fvm_bench_test.go index 8c9519fde66..f1f0582ed2d 100644 --- a/fvm/fvm_bench_test.go +++ b/fvm/fvm_bench_test.go @@ -5,10 +5,8 @@ import ( "encoding/json" "fmt" "io" - "math/rand" "strings" "testing" - "time" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -365,7 +363,6 @@ var _ io.Writer = &logExtractor{} // BenchmarkRuntimeEmptyTransaction simulates executing blocks with `transactionsPerBlock` // where each transaction is an empty transaction func BenchmarkRuntimeTransaction(b *testing.B) { - rand.Seed(time.Now().UnixNano()) transactionsPerBlock := 10 diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index 93771fb1f52..f8e46ea920c 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -1661,7 +1661,7 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { num, err := strconv.ParseUint(tx.Logs[0], 10, 64) require.NoError(t, err) - require.Equal(t, uint64(0xde226d5af92d269), num) + require.Equal(t, uint64(0x7515f254adc6f8af), num) }) } diff --git a/fvm/fvm_signature_test.go b/fvm/fvm_signature_test.go index d69857efd43..cdba75c04b5 100644 --- a/fvm/fvm_signature_test.go +++ b/fvm/fvm_signature_test.go @@ -1,8 +1,8 @@ package fvm_test import ( + "crypto/rand" "fmt" - "math/rand" "testing" "github.com/onflow/cadence" diff --git a/insecure/wintermute/attackOrchestrator_test.go b/insecure/wintermute/attackOrchestrator_test.go index ce2b6f41459..fdd7768c229 100644 --- a/insecure/wintermute/attackOrchestrator_test.go +++ b/insecure/wintermute/attackOrchestrator_test.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/utils/rand" "github.com/onflow/flow-go/utils/unittest" ) @@ -557,7 +558,7 @@ func TestPassingThroughMiscellaneousEvents(t *testing.T) { // creates a block event fixture that is out of the context of // the wintermute attack. miscellaneousEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds.Sample(1)[0], + CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, @@ -631,7 +632,7 @@ func TestPassingThrough_ResultApproval(t *testing.T) { require.NotEqual(t, wintermuteOrchestrator.state.originalResult.ID(), approval.ID()) require.NotEqual(t, wintermuteOrchestrator.state.corruptedResult.ID(), approval.ID()) approvalEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds.Sample(1)[0], + CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, @@ -703,7 +704,7 @@ func TestWintermute_ResultApproval(t *testing.T) { // generates a result approval event for one of the chunks of the original result. approvalEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds.Sample(1)[0], + CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, diff --git a/integration/dkg/dkg_emulator_test.go b/integration/dkg/dkg_emulator_test.go index c68e5e9e617..2131e6c696b 100644 --- a/integration/dkg/dkg_emulator_test.go +++ b/integration/dkg/dkg_emulator_test.go @@ -168,8 +168,6 @@ func (s *DKGSuite) runTest(goodNodes int, emulatorProblems bool) { // shuffle the signatures and indices before constructing the group // signature (since it only uses the first half signatures) - seed := time.Now().UnixNano() - rand.Seed(seed) rand.Shuffle(len(signatures), func(i, j int) { signatures[i], signatures[j] = signatures[j], signatures[i] indices[i], indices[j] = indices[j], indices[i] diff --git a/integration/dkg/dkg_whiteboard_test.go b/integration/dkg/dkg_whiteboard_test.go index f3166e8a57d..67cbf0285fe 100644 --- a/integration/dkg/dkg_whiteboard_test.go +++ b/integration/dkg/dkg_whiteboard_test.go @@ -298,8 +298,6 @@ func TestWithWhiteboard(t *testing.T) { // shuffle the signatures and indices before constructing the group // signature (since it only uses the first half signatures) - seed := time.Now().UnixNano() - rand.Seed(seed) rand.Shuffle(len(signatures), func(i, j int) { signatures[i], signatures[j] = signatures[j], signatures[i] indices[i], indices[j] = indices[j], indices[i] diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 3e3b7e6ce9c..774d1c56c1b 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -2,6 +2,7 @@ package testnet import ( "context" + crand "crypto/rand" "encoding/hex" "fmt" "math/rand" @@ -1336,7 +1337,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl } randomSource := make([]byte, flow.EpochSetupRandomSourceLength) - _, err = rand.Read(randomSource) + _, err = crand.Read(randomSource) if err != nil { return nil, err } diff --git a/integration/tests/access/consensus_follower_test.go b/integration/tests/access/consensus_follower_test.go index e713d0c892c..ab71a4503f0 100644 --- a/integration/tests/access/consensus_follower_test.go +++ b/integration/tests/access/consensus_follower_test.go @@ -177,8 +177,8 @@ func (suite *ConsensusFollowerSuite) buildNetworkConfig() { // TODO: Move this to unittest and resolve the circular dependency issue func UnstakedNetworkingKey() (crypto.PrivateKey, error) { seed := make([]byte, crypto.KeyGenSeedMinLenECDSASecp256k1) - n, err := rand.Read(seed) - if err != nil || n != crypto.KeyGenSeedMinLenECDSASecp256k1 { + _, err := rand.Read(seed) + if err != nil { return nil, err } return utils.GeneratePublicNetworkingKey(unittest.SeedFixture(n)) diff --git a/integration/tests/consensus/inclusion_test.go b/integration/tests/consensus/inclusion_test.go index a5cd974a42e..572cfa6c13a 100644 --- a/integration/tests/consensus/inclusion_test.go +++ b/integration/tests/consensus/inclusion_test.go @@ -47,7 +47,6 @@ func (is *InclusionSuite) SetupTest() { is.log.Info().Msgf("================> SetupTest") // seed random generator - rand.Seed(time.Now().UnixNano()) // to collect node confiis... var nodeConfigs []testnet.NodeConfig diff --git a/integration/tests/consensus/sealing_test.go b/integration/tests/consensus/sealing_test.go index fdf1b67a288..ddb62ae96aa 100644 --- a/integration/tests/consensus/sealing_test.go +++ b/integration/tests/consensus/sealing_test.go @@ -67,7 +67,6 @@ func (ss *SealingSuite) SetupTest() { ss.log.Info().Msgf("================> SetupTest") // seed random generator - rand.Seed(time.Now().UnixNano()) // to collect node confiss... var nodeConfigs []testnet.NodeConfig diff --git a/integration/tests/lib/util.go b/integration/tests/lib/util.go index af5a3e4f37d..6d0a14ca540 100644 --- a/integration/tests/lib/util.go +++ b/integration/tests/lib/util.go @@ -2,8 +2,8 @@ package lib import ( "context" + "crypto/rand" "fmt" - "math/rand" "testing" "time" diff --git a/ledger/common/bitutils/utils_test.go b/ledger/common/bitutils/utils_test.go index f6d3e0d2383..8671711fdf3 100644 --- a/ledger/common/bitutils/utils_test.go +++ b/ledger/common/bitutils/utils_test.go @@ -1,6 +1,7 @@ package bitutils import ( + crand "crypto/rand" "math/big" "math/bits" "math/rand" @@ -38,7 +39,7 @@ func Test_PaddedByteSliceLength(t *testing.T) { func TestBitTools(t *testing.T) { seed := time.Now().UnixNano() t.Logf("rand seed is %d", seed) - rand.Seed(seed) + r := rand.NewSource(seed) const maxBits = 131 * 8 // upper bound of indices to test @@ -71,7 +72,7 @@ func TestBitTools(t *testing.T) { t.Run("testing WriteBit", func(t *testing.T) { b.SetInt64(0) bytes := MakeBitVector(maxBits) - rand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works // build a random big bit by bit for idx := 0; idx < maxBits; idx++ { @@ -91,7 +92,7 @@ func TestBitTools(t *testing.T) { t.Run("testing ClearBit and SetBit", func(t *testing.T) { b.SetInt64(0) bytes := MakeBitVector(maxBits) - rand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works // build a random big bit by bit for idx := 0; idx < maxBits; idx++ { diff --git a/ledger/common/hash/hash_test.go b/ledger/common/hash/hash_test.go index f1fab40a634..9713340a3a9 100644 --- a/ledger/common/hash/hash_test.go +++ b/ledger/common/hash/hash_test.go @@ -1,9 +1,8 @@ package hash_test import ( - "math/rand" + "crypto/rand" "testing" - "time" "golang.org/x/crypto/sha3" @@ -15,10 +14,6 @@ import ( ) func TestHash(t *testing.T) { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) - t.Run("lengthSanity", func(t *testing.T) { assert.Equal(t, 32, hash.HashLen) }) diff --git a/ledger/common/testutils/testutils.go b/ledger/common/testutils/testutils.go index cdb1803414f..8543abbc0de 100644 --- a/ledger/common/testutils/testutils.go +++ b/ledger/common/testutils/testutils.go @@ -1,6 +1,7 @@ package testutils import ( + crand "crypto/rand" "encoding/binary" "encoding/hex" "fmt" @@ -151,7 +152,7 @@ func RandomPaths(n int) []l.Path { i := 0 for i < n { var path l.Path - rand.Read(path[:]) + crand.Read(path[:]) // deduplicate if _, found := alreadySelectPaths[path]; !found { paths = append(paths, path) @@ -166,11 +167,11 @@ func RandomPaths(n int) []l.Path { func RandomPayload(minByteSize int, maxByteSize int) *l.Payload { keyByteSize := minByteSize + rand.Intn(maxByteSize-minByteSize) keydata := make([]byte, keyByteSize) - rand.Read(keydata) + crand.Read(keydata) key := l.Key{KeyParts: []l.KeyPart{{Type: 0, Value: keydata}}} valueByteSize := minByteSize + rand.Intn(maxByteSize-minByteSize) valuedata := make([]byte, valueByteSize) - rand.Read(valuedata) + crand.Read(valuedata) value := l.Value(valuedata) return l.NewPayload(key, value) } @@ -196,7 +197,7 @@ func RandomValues(n int, minByteSize, maxByteSize int) []l.Value { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } value := make([]byte, byteSize) - rand.Read(value) + crand.Read(value) values = append(values, value) } return values @@ -218,7 +219,7 @@ func RandomUniqueKeys(n, m, minByteSize, maxByteSize int) []l.Key { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } keyPartData := make([]byte, byteSize) - rand.Read(keyPartData) + crand.Read(keyPartData) keyParts = append(keyParts, l.NewKeyPart(uint16(j), keyPartData)) } key := l.NewKey(keyParts) diff --git a/ledger/complete/ledger_benchmark_test.go b/ledger/complete/ledger_benchmark_test.go index ddc78095cc8..6c0855be914 100644 --- a/ledger/complete/ledger_benchmark_test.go +++ b/ledger/complete/ledger_benchmark_test.go @@ -2,7 +2,6 @@ package complete_test import ( "math" - "math/rand" "testing" "time" @@ -40,8 +39,6 @@ func benchmarkStorage(steps int, b *testing.B) { checkpointsToKeep = 1 ) - rand.Seed(time.Now().UnixNano()) - dir := b.TempDir() diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, steps+1, pathfinder.PathByteSize, wal.SegmentSize) @@ -155,8 +152,6 @@ func BenchmarkTrieUpdate(b *testing.B) { checkpointsToKeep = 1 ) - rand.Seed(1) - dir := b.TempDir() diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) @@ -209,8 +204,6 @@ func BenchmarkTrieRead(b *testing.B) { checkpointsToKeep = 1 ) - rand.Seed(1) - dir := b.TempDir() diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) @@ -272,8 +265,6 @@ func BenchmarkLedgerGetOneValue(b *testing.B) { checkpointsToKeep = 1 ) - rand.Seed(1) - dir := b.TempDir() diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) @@ -352,8 +343,6 @@ func BenchmarkTrieProve(b *testing.B) { checkpointsToKeep = 1 ) - rand.Seed(1) - dir := b.TempDir() diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, metrics.NewNoopCollector(), dir, capacity, pathfinder.PathByteSize, wal.SegmentSize) diff --git a/ledger/complete/ledger_test.go b/ledger/complete/ledger_test.go index 1f791b2eaa8..a723d2a58f1 100644 --- a/ledger/complete/ledger_test.go +++ b/ledger/complete/ledger_test.go @@ -7,7 +7,6 @@ import ( "math" "math/rand" "testing" - "time" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -591,7 +590,6 @@ func TestLedgerFunctionality(t *testing.T) { checkpointsToKeep = 1 ) - rand.Seed(time.Now().UnixNano()) // You can manually increase this for more coverage experimentRep := 2 metricsCollector := &metrics.NoopCollector{} diff --git a/ledger/complete/mtrie/flattener/encoding_test.go b/ledger/complete/mtrie/flattener/encoding_test.go index b7e8ad07901..1876f2199ac 100644 --- a/ledger/complete/mtrie/flattener/encoding_test.go +++ b/ledger/complete/mtrie/flattener/encoding_test.go @@ -2,6 +2,7 @@ package flattener_test import ( "bytes" + crand "crypto/rand" "errors" "fmt" "math/rand" @@ -160,7 +161,7 @@ func TestRandomLeafNodeEncodingDecoding(t *testing.T) { height := rand.Intn(257) var hashValue hash.Hash - rand.Read(hashValue[:]) + crand.Read(hashValue[:]) n := node.NewNode(height, nil, nil, paths[i], payloads[i], hashValue) diff --git a/ledger/complete/mtrie/forest_test.go b/ledger/complete/mtrie/forest_test.go index 4248be54940..0f0377faf57 100644 --- a/ledger/complete/mtrie/forest_test.go +++ b/ledger/complete/mtrie/forest_test.go @@ -750,7 +750,7 @@ func TestRandomUpdateReadProofValueSizes(t *testing.T) { rep := 10 maxNumPathsPerStep := 10 seed := time.Now().UnixNano() - rand.Seed(seed) + t.Log(seed) forest, err := NewForest(5, &metrics.NoopCollector{}, nil) diff --git a/ledger/complete/mtrie/trie/trie_test.go b/ledger/complete/mtrie/trie/trie_test.go index f88d67770f8..780c63c1410 100644 --- a/ledger/complete/mtrie/trie/trie_test.go +++ b/ledger/complete/mtrie/trie/trie_test.go @@ -5,10 +5,8 @@ import ( "encoding/binary" "encoding/hex" "math" - "math/rand" "sort" "testing" - "time" "github.com/stretchr/testify/require" "gotest.tools/assert" @@ -354,9 +352,7 @@ func deduplicateWrites(paths []ledger.Path, payloads []ledger.Payload) ([]ledger } func TestSplitByPath(t *testing.T) { - seed := time.Now().UnixNano() - t.Logf("rand seed is %d", seed) - rand.Seed(seed) + rand := unittest.GetPRG(t) const pathsNumber = 100 const redundantPaths = 10 @@ -490,6 +486,7 @@ func Test_DifferentiateEmptyVsLeaf(t *testing.T) { } func Test_Pruning(t *testing.T) { + rand := unittest.GetPRG(t) emptyTrie := trie.NewEmptyMTrie() path1 := testutils.PathByUint16(1 << 12) // 000100... diff --git a/ledger/complete/mtrie/trieCache_test.go b/ledger/complete/mtrie/trieCache_test.go index df01688d627..bc5130ddd60 100644 --- a/ledger/complete/mtrie/trieCache_test.go +++ b/ledger/complete/mtrie/trieCache_test.go @@ -6,7 +6,7 @@ package mtrie // test across boundry import ( - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/require" diff --git a/ledger/complete/wal/checkpoint_v6_test.go b/ledger/complete/wal/checkpoint_v6_test.go index 1e579b258d7..ce3dc406f43 100644 --- a/ledger/complete/wal/checkpoint_v6_test.go +++ b/ledger/complete/wal/checkpoint_v6_test.go @@ -3,10 +3,10 @@ package wal import ( "bufio" "bytes" + "crypto/rand" "errors" "fmt" "io" - "math/rand" "os" "path" "path/filepath" diff --git a/ledger/complete/wal/triequeue_test.go b/ledger/complete/wal/triequeue_test.go index 54dd2e1ef6c..4f93006c3ec 100644 --- a/ledger/complete/wal/triequeue_test.go +++ b/ledger/complete/wal/triequeue_test.go @@ -1,7 +1,7 @@ package wal import ( - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/require" diff --git a/ledger/partial/ptrie/partialTrie_test.go b/ledger/partial/ptrie/partialTrie_test.go index c452175c9e3..e035c5c4ff9 100644 --- a/ledger/partial/ptrie/partialTrie_test.go +++ b/ledger/partial/ptrie/partialTrie_test.go @@ -376,7 +376,7 @@ func TestRandomProofs(t *testing.T) { // generate some random paths and payloads seed := time.Now().UnixNano() - rand.Seed(seed) + t.Logf("rand seed is %x", seed) numberOfPaths := rand.Intn(256) + 1 paths := testutils.RandomPaths(numberOfPaths) diff --git a/model/encodable/keys_test.go b/model/encodable/keys_test.go index ccdf63cd044..5b396fb6f99 100644 --- a/model/encodable/keys_test.go +++ b/model/encodable/keys_test.go @@ -252,8 +252,7 @@ func TestEncodableRandomBeaconPrivKeyMsgPack(t *testing.T) { func generateRandomSeed(t *testing.T) []byte { seed := make([]byte, 48) - n, err := rand.Read(seed) + _, err := rand.Read(seed) require.Nil(t, err) - require.Equal(t, n, 48) return seed } diff --git a/model/flow/address_test.go b/model/flow/address_test.go index edfb10eda24..b3eabde4859 100644 --- a/model/flow/address_test.go +++ b/model/flow/address_test.go @@ -5,7 +5,6 @@ import ( "math/bits" "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -137,7 +136,6 @@ const invalidCodeWord = uint64(0xab2ae42382900010) func testAddressGeneration(t *testing.T) { // seed random generator - rand.Seed(time.Now().UnixNano()) // loops in each test const loop = 50 @@ -230,7 +228,6 @@ func testAddressGeneration(t *testing.T) { func testAddressesIntersection(t *testing.T) { // seed random generator - rand.Seed(time.Now().UnixNano()) // loops in each test const loop = 25 @@ -299,7 +296,6 @@ func testAddressesIntersection(t *testing.T) { func testIndexFromAddress(t *testing.T) { // seed random generator - rand.Seed(time.Now().UnixNano()) // loops in each test const loop = 50 @@ -340,7 +336,6 @@ func testIndexFromAddress(t *testing.T) { func TestUint48(t *testing.T) { // seed random generator - rand.Seed(time.Now().UnixNano()) const loop = 50 // test consistensy of putUint48 and uint48 diff --git a/model/flow/identifier.go b/model/flow/identifier.go index 62ad2a64735..e205e74a716 100644 --- a/model/flow/identifier.go +++ b/model/flow/identifier.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "encoding/hex" "fmt" - "math/rand" "reflect" "github.com/ipfs/go-cid" @@ -16,6 +15,7 @@ import ( "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/model/fingerprint" "github.com/onflow/flow-go/storage/merkle" + "github.com/onflow/flow-go/utils/rand" ) const IdentifierLen = 32 @@ -179,21 +179,24 @@ func CheckConcatSum(sum Identifier, fps ...Identifier) bool { return sum == computed } -// Sample returns random sample of length 'size' of the ids -// [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). -func Sample(size uint, ids ...Identifier) []Identifier { +// Sample returns non-deterministic random sample of length 'size' of the ids +func Sample(size uint, ids ...Identifier) ([]Identifier, error) { n := uint(len(ids)) dup := make([]Identifier, 0, n) dup = append(dup, ids...) // if sample size is greater than total size, return all the elements if n <= size { - return dup + return dup, nil } - for i := uint(0); i < size; i++ { - j := uint(rand.Intn(int(n - i))) - dup[i], dup[j+i] = dup[j+i], dup[i] + swap := func(i, j uint) { + dup[i], dup[j] = dup[j], dup[i] } - return dup[:size] + + err := rand.Samples(n, size, swap) + if err != nil { + return nil, fmt.Errorf("generating randoms failed: %w", err) + } + return dup[:size], nil } func CidToId(c cid.Cid) (Identifier, error) { diff --git a/model/flow/identifierList.go b/model/flow/identifierList.go index 33ce2447707..eda69b35909 100644 --- a/model/flow/identifierList.go +++ b/model/flow/identifierList.go @@ -2,7 +2,6 @@ package flow import ( "bytes" - "math/rand" "sort" "github.com/rs/zerolog/log" @@ -92,15 +91,8 @@ func (il IdentifierList) Union(other IdentifierList) IdentifierList { return union } -// DeterministicSample returns deterministic random sample from the `IdentifierList` using the given seed -func (il IdentifierList) DeterministicSample(size uint, seed int64) IdentifierList { - rand.Seed(seed) - return il.Sample(size) -} - // Sample returns random sample of length 'size' of the ids -// [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher–Yates_shuffle). -func (il IdentifierList) Sample(size uint) IdentifierList { +func (il IdentifierList) Sample(size uint) (IdentifierList, error) { return Sample(size, il...) } diff --git a/model/flow/identifierList_test.go b/model/flow/identifierList_test.go index b878938a5e3..7e18b6ee921 100644 --- a/model/flow/identifierList_test.go +++ b/model/flow/identifierList_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "sort" "testing" - "time" "github.com/stretchr/testify/require" @@ -21,7 +20,7 @@ func TestIdentifierListSort(t *testing.T) { var ids flow.IdentifierList = unittest.IdentifierListFixture(count) // shuffles array before sorting to enforce some pseudo-randomness - rand.Seed(time.Now().UnixNano()) + rand.Shuffle(ids.Len(), ids.Swap) sort.Sort(ids) diff --git a/model/flow/identifier_test.go b/model/flow/identifier_test.go index a4362e95f37..3a6d3c33aa8 100644 --- a/model/flow/identifier_test.go +++ b/model/flow/identifier_test.go @@ -1,10 +1,10 @@ package flow_test import ( + "crypto/rand" "encoding/binary" "encoding/json" "fmt" - "math/rand" "testing" blocks "github.com/ipfs/go-block-format" @@ -66,20 +66,23 @@ func TestIdentifierSample(t *testing.T) { t.Run("Sample creates a random sample", func(t *testing.T) { sampleSize := uint(5) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Len(t, sample, int(sampleSize)) require.NotEqual(t, sample, ids[:sampleSize]) }) t.Run("sample size greater than total size results in the original list", func(t *testing.T) { sampleSize := uint(len(ids) + 1) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Equal(t, sample, ids) }) t.Run("sample size of zero results in an empty list", func(t *testing.T) { sampleSize := uint(0) - sample := flow.Sample(sampleSize, ids...) + sample, err := flow.Sample(sampleSize, ids...) + require.NoError(t, err) require.Empty(t, sample) }) } diff --git a/model/flow/identity.go b/model/flow/identity.go index cc4970fba8d..a3246241b81 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "math" - "math/rand" "regexp" "strconv" @@ -18,6 +17,7 @@ import ( "github.com/vmihailenco/msgpack" "github.com/onflow/flow-go/crypto" + "github.com/onflow/flow-go/utils/rand" ) // DefaultInitialWeight is the default initial weight for a node identity. @@ -443,40 +443,39 @@ func (il IdentityList) ByNetworkingKey(key crypto.PublicKey) (*Identity, bool) { return nil, false } -// Sample returns simple random sample from the `IdentityList` -func (il IdentityList) Sample(size uint) IdentityList { - return il.sample(size, rand.Intn) -} - -// DeterministicSample returns deterministic random sample from the `IdentityList` using the given seed -func (il IdentityList) DeterministicSample(size uint, seed int64) IdentityList { - rng := rand.New(rand.NewSource(seed)) - return il.sample(size, rng.Intn) -} - -func (il IdentityList) sample(size uint, intn func(int) int) IdentityList { +// Sample returns non-deterministic random sample from the `IdentityList` +func (il IdentityList) Sample(size uint) (IdentityList, error) { n := uint(len(il)) - if size > n { - size = n + dup := make([]*Identity, 0, n) + dup = append(dup, il...) + // if sample size is greater than total size, return all the elements + if n <= size { + return dup, nil } - - dup := il.Copy() - for i := uint(0); i < size; i++ { - j := uint(intn(int(n - i))) - dup[i], dup[j+i] = dup[j+i], dup[i] + swap := func(i, j uint) { + dup[i], dup[j] = dup[j], dup[i] + } + err := rand.Samples(n, size, swap) + if err != nil { + return nil, fmt.Errorf("failed to generate randomness: %w", err) } - return dup[:size] + return dup[:size], nil } -// DeterministicShuffle randomly and deterministically shuffles the identity +// Shuffle non-deterministically randomly shuffles the identity // list, returning the shuffled list without modifying the receiver. -func (il IdentityList) DeterministicShuffle(seed int64) IdentityList { - dup := il.Copy() - rng := rand.New(rand.NewSource(seed)) - rng.Shuffle(len(il), func(i, j int) { +func (il IdentityList) Shuffle() (IdentityList, error) { + n := uint(len(il)) + dup := make([]*Identity, 0, n) + dup = append(dup, il...) + swap := func(i, j uint) { dup[i], dup[j] = dup[j], dup[i] - }) - return dup + } + err := rand.Shuffle(n, swap) + if err != nil { + return nil, fmt.Errorf("failed to generate randomness: %w", err) + } + return dup, nil } // SamplePct returns a random sample from the receiver identity list. The @@ -484,9 +483,9 @@ func (il IdentityList) DeterministicShuffle(seed int64) IdentityList { // if `pct>0`, so this will always select at least one identity. // // NOTE: The input must be between 0-1. -func (il IdentityList) SamplePct(pct float64) IdentityList { +func (il IdentityList) SamplePct(pct float64) (IdentityList, error) { if pct <= 0 { - return IdentityList{} + return IdentityList{}, nil } count := float64(il.Count()) * pct diff --git a/model/flow/identity_test.go b/model/flow/identity_test.go index 9c1a137d8ab..891a854aca6 100644 --- a/model/flow/identity_test.go +++ b/model/flow/identity_test.go @@ -2,10 +2,8 @@ package flow_test import ( "encoding/json" - "math/rand" "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -198,28 +196,35 @@ func TestIdentityList_Union(t *testing.T) { func TestSample(t *testing.T) { t.Run("Sample max", func(t *testing.T) { il := unittest.IdentityListFixture(10) - require.Equal(t, uint(10), il.Sample(10).Count()) + sam, err := il.Sample(10) + require.NoError(t, err) + require.Equal(t, uint(10), sam.Count()) }) t.Run("Sample oversized", func(t *testing.T) { il := unittest.IdentityListFixture(10) - require.Equal(t, uint(10), il.Sample(11).Count()) + sam, err := il.Sample(11) + require.NoError(t, err) + require.Equal(t, uint(10), sam.Count()) }) } func TestShuffle(t *testing.T) { t.Run("should be shuffled", func(t *testing.T) { il := unittest.IdentityListFixture(15) // ~1/billion chance of shuffling to input state - shuffled := il.DeterministicShuffle(rand.Int63()) + shuffled, err := il.Shuffle() + require.NoError(t, err) assert.Equal(t, len(il), len(shuffled)) assert.ElementsMatch(t, il, shuffled) }) - t.Run("should be deterministic", func(t *testing.T) { + t.Run("should not be deterministic", func(t *testing.T) { il := unittest.IdentityListFixture(10) - seed := rand.Int63() - shuffled1 := il.DeterministicShuffle(seed) - shuffled2 := il.DeterministicShuffle(seed) - assert.Equal(t, shuffled1, shuffled2) + shuffled1, err := il.Shuffle() + require.NoError(t, err) + shuffled2, err := il.Shuffle() + require.NoError(t, err) + assert.NotEqual(t, shuffled1, shuffled2) + assert.ElementsMatch(t, shuffled1, shuffled2) }) } @@ -238,7 +243,8 @@ func TestIdentity_ID(t *testing.T) { func TestIdentity_Sort(t *testing.T) { il := unittest.IdentityListFixture(20) - random := il.DeterministicShuffle(time.Now().UnixNano()) + random, err := il.Shuffle() + require.NoError(t, err) assert.False(t, random.Sorted(order.Canonical)) canonical := il.Sort(order.Canonical) diff --git a/model/verification/chunkDataPackRequest.go b/model/verification/chunkDataPackRequest.go index 0c0cd4cd92a..9f2bf42c52c 100644 --- a/model/verification/chunkDataPackRequest.go +++ b/model/verification/chunkDataPackRequest.go @@ -1,6 +1,8 @@ package verification import ( + "fmt" + "github.com/onflow/flow-go/model/chunks" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -23,10 +25,14 @@ type ChunkDataPackRequestInfo struct { // SampleTargets returns identifier of execution nodes that can be asked for the chunk data pack, based on // the agreeing and disagreeing execution nodes of the chunk data pack request. -func (c ChunkDataPackRequestInfo) SampleTargets(count int) flow.IdentifierList { +func (c ChunkDataPackRequestInfo) SampleTargets(count int) (flow.IdentifierList, error) { // if there are enough receipts produced the same result (agrees), we sample from them. if len(c.Agrees) >= count { - return c.Targets.Filter(filter.HasNodeID(c.Agrees...)).Sample(uint(count)).NodeIDs() + sample, err := c.Targets.Filter(filter.HasNodeID(c.Agrees...)).Sample(uint(count)) + if err != nil { + return nil, fmt.Errorf("sampling target failed: %w", err) + } + return sample.NodeIDs(), nil } // since there is at least one agree, then usually, we just need `count - 1` extra nodes as backup. @@ -35,8 +41,11 @@ func (c ChunkDataPackRequestInfo) SampleTargets(count int) flow.IdentifierList { // fetch from the one produced the same result (the only agree) need := uint(count - len(c.Agrees)) - nonResponders := c.Targets.Filter(filter.Not(filter.HasNodeID(c.Disagrees...))).Sample(need).NodeIDs() - return append(c.Agrees, nonResponders...) + nonResponders, err := c.Targets.Filter(filter.Not(filter.HasNodeID(c.Disagrees...))).Sample(need) + if err != nil { + return nil, fmt.Errorf("sampling target failed: %w", err) + } + return append(c.Agrees, nonResponders.NodeIDs()...), nil } type ChunkDataPackRequestInfoList []*ChunkDataPackRequestInfo diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 1bdcff76392..7bde32540dd 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -61,7 +60,6 @@ func (suite *BuilderSuite) SetupTest() { var err error // seed the RNG - rand.Seed(time.Now().UnixNano()) suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID diff --git a/module/chunks/chunkVerifier_test.go b/module/chunks/chunkVerifier_test.go index 98fe3afca61..b4818bac2dc 100644 --- a/module/chunks/chunkVerifier_test.go +++ b/module/chunks/chunkVerifier_test.go @@ -2,9 +2,7 @@ package chunks_test import ( "fmt" - "math/rand" "testing" - "time" "github.com/onflow/cadence/runtime" "github.com/rs/zerolog" @@ -69,7 +67,6 @@ type ChunkVerifierTestSuite struct { // SetupTest is executed prior to each individual test in this test suite func (s *ChunkVerifierTestSuite) SetupSuite() { // seed the RNG - rand.Seed(time.Now().UnixNano()) vm := new(vmMock) systemOkVm := new(vmSystemOkMock) diff --git a/module/chunks/chunk_assigner_test.go b/module/chunks/chunk_assigner_test.go index 1c65c91d817..13475bcd4b7 100644 --- a/module/chunks/chunk_assigner_test.go +++ b/module/chunks/chunk_assigner_test.go @@ -1,7 +1,7 @@ package chunks import ( - "math/rand" + "crypto/rand" "testing" "github.com/stretchr/testify/mock" diff --git a/module/dkg/controller.go b/module/dkg/controller.go index 5c9adf4994a..ae4b54ecb38 100644 --- a/module/dkg/controller.go +++ b/module/dkg/controller.go @@ -3,7 +3,6 @@ package dkg import ( "fmt" "math" - "math/rand" "sync" "time" @@ -12,6 +11,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -304,7 +304,10 @@ func (c *Controller) doBackgroundWork() { isFirstMessage := false c.once.Do(func() { isFirstMessage = true - delay := c.preHandleFirstBroadcastDelay() + delay, err := c.preHandleFirstBroadcastDelay() + if err != nil { + c.log.Err(err).Msg("pre handle first broadcast delay failed") + } c.log.Info().Msgf("sleeping for %s before processing first phase 1 broadcast message", delay) time.Sleep(delay) }) @@ -337,12 +340,15 @@ func (c *Controller) start() error { // before starting the DKG, sleep for a random delay to avoid synchronizing // this expensive operation across all consensus nodes - delay := c.preStartDelay() + delay, err := c.preStartDelay() + if err != nil { + return fmt.Errorf("pre start delay failed: %w", err) + } c.log.Debug().Msgf("sleeping for %s before starting DKG", delay) time.Sleep(delay) c.dkgLock.Lock() - err := c.dkg.Start(c.seed) + err = c.dkg.Start(c.seed) c.dkgLock.Unlock() if err != nil { return fmt.Errorf("Error starting DKG: %w", err) @@ -421,18 +427,16 @@ func (c *Controller) phase3() error { // preStartDelay returns a duration to delay prior to starting the DKG process. // This prevents synchronization of the DKG starting (an expensive operation) // across the network, which can impact finalization. -func (c *Controller) preStartDelay() time.Duration { - delay := computePreprocessingDelay(c.config.BaseStartDelay, c.dkg.Size()) - return delay +func (c *Controller) preStartDelay() (time.Duration, error) { + return computePreprocessingDelay(c.config.BaseStartDelay, c.dkg.Size()) } // preHandleFirstBroadcastDelay returns a duration to delay prior to handling // the first broadcast message. This delay is used only during phase 1 of the DKG. // This prevents synchronization of processing verification vectors (an // expensive operation) across the network, which can impact finalization. -func (c *Controller) preHandleFirstBroadcastDelay() time.Duration { - delay := computePreprocessingDelay(c.config.BaseHandleFirstBroadcastDelay, c.dkg.Size()) - return delay +func (c *Controller) preHandleFirstBroadcastDelay() (time.Duration, error) { + return computePreprocessingDelay(c.config.BaseHandleFirstBroadcastDelay, c.dkg.Size()) } // computePreprocessingDelay computes a random delay to introduce before an @@ -441,15 +445,18 @@ func (c *Controller) preHandleFirstBroadcastDelay() time.Duration { // The maximum delay is m=b*n^2 where: // * b is a configurable base delay // * n is the size of the DKG committee -func computePreprocessingDelay(baseDelay time.Duration, dkgSize int) time.Duration { +func computePreprocessingDelay(baseDelay time.Duration, dkgSize int) (time.Duration, error) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) if maxDelay <= 0 { - return 0 + return 0, nil } // select delay from [0,m) - delay := time.Duration(rand.Int63n(maxDelay.Nanoseconds())) - return delay + r, err := rand.Uint64n(uint64(maxDelay.Nanoseconds())) + if err != nil { + return time.Duration(0), fmt.Errorf("delay generation failed %w", err) + } + return time.Duration(r), nil } // computePreprocessingDelayMax computes the maximum dely for computePreprocessingDelay. diff --git a/module/dkg/controller_test.go b/module/dkg/controller_test.go index 03f10adf1c1..e8f8d253537 100644 --- a/module/dkg/controller_test.go +++ b/module/dkg/controller_test.go @@ -333,20 +333,26 @@ func checkArtifacts(t *testing.T, nodes []*node, totalNodes int) { func TestDelay(t *testing.T) { t.Run("should return 0 delay for <=0 inputs", func(t *testing.T) { - delay := computePreprocessingDelay(0, 100) + delay, err := computePreprocessingDelay(0, 100) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(time.Hour, 0) + delay, err = computePreprocessingDelay(time.Hour, 0) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(time.Millisecond, -1) + delay, err = computePreprocessingDelay(time.Millisecond, -1) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) - delay = computePreprocessingDelay(-time.Millisecond, 100) + delay, err = computePreprocessingDelay(-time.Millisecond, 100) + require.NoError(t, err) assert.Equal(t, delay, time.Duration(0)) }) // NOTE: this is a probabilistic test. It will (extremely infrequently) fail. t.Run("should return different values for same inputs", func(t *testing.T) { - d1 := computePreprocessingDelay(time.Hour, 100) - d2 := computePreprocessingDelay(time.Hour, 100) + d1, err := computePreprocessingDelay(time.Hour, 100) + require.NoError(t, err) + d2, err := computePreprocessingDelay(time.Hour, 100) + require.NoError(t, err) assert.NotEqual(t, d1, d2) }) @@ -360,7 +366,8 @@ func TestDelay(t *testing.T) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) assert.Equal(t, expectedMaxDelay, maxDelay) - delay := computePreprocessingDelay(baseDelay, dkgSize) + delay, err := computePreprocessingDelay(baseDelay, dkgSize) + require.NoError(t, err) assert.LessOrEqual(t, minDelay, delay) assert.GreaterOrEqual(t, expectedMaxDelay, delay) }) @@ -375,7 +382,8 @@ func TestDelay(t *testing.T) { maxDelay := computePreprocessingDelayMax(baseDelay, dkgSize) assert.Equal(t, expectedMaxDelay, maxDelay) - delay := computePreprocessingDelay(baseDelay, dkgSize) + delay, err := computePreprocessingDelay(baseDelay, dkgSize) + require.NoError(t, err) assert.LessOrEqual(t, minDelay, delay) assert.GreaterOrEqual(t, expectedMaxDelay, delay) }) diff --git a/module/epochs/qc_voter_test.go b/module/epochs/qc_voter_test.go index 71a2fdd3b97..47a54483200 100644 --- a/module/epochs/qc_voter_test.go +++ b/module/epochs/qc_voter_test.go @@ -69,7 +69,7 @@ func (suite *Suite) SetupTest() { suite.counter = rand.Uint64() suite.nodes = unittest.IdentityListFixture(4, unittest.WithRole(flow.RoleCollection)) - suite.me = suite.nodes.Sample(1)[0] + suite.me = suite.nodes[rand.Intn(len(suite.nodes))] suite.local.On("NodeID").Return(func() flow.Identifier { return suite.me.NodeID }) diff --git a/module/executiondatasync/execution_data/store_test.go b/module/executiondatasync/execution_data/store_test.go index 39d00d93044..711a8d24ed5 100644 --- a/module/executiondatasync/execution_data/store_test.go +++ b/module/executiondatasync/execution_data/store_test.go @@ -3,9 +3,10 @@ package execution_data_test import ( "bytes" "context" + "crypto/rand" "fmt" "io" - "math/rand" + mrand "math/rand" "testing" "github.com/ipfs/go-cid" @@ -134,7 +135,7 @@ type corruptedTailSerializer struct { func newCorruptedTailSerializer(numChunks int) *corruptedTailSerializer { return &corruptedTailSerializer{ - corruptedChunk: rand.Intn(numChunks) + 1, + corruptedChunk: mrand.Intn(numChunks) + 1, } } @@ -197,7 +198,7 @@ func TestGetIncompleteData(t *testing.T) { cids := getAllKeys(t, blobstore) t.Logf("%d blobs in blob tree", len(cids)) - cidToDelete := cids[rand.Intn(len(cids))] + cidToDelete := cids[mrand.Intn(len(cids))] require.NoError(t, blobstore.DeleteBlob(context.Background(), cidToDelete)) _, err = eds.GetExecutionData(context.Background(), rootID) diff --git a/module/finalizer/collection/finalizer_test.go b/module/finalizer/collection/finalizer_test.go index 921e8cc6c57..c3c837f8738 100644 --- a/module/finalizer/collection/finalizer_test.go +++ b/module/finalizer/collection/finalizer_test.go @@ -1,9 +1,7 @@ package collection_test import ( - "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -27,7 +25,6 @@ func TestFinalizer(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { // seed the RNG - rand.Seed(time.Now().UnixNano()) // reference block on the main consensus chain refBlock := unittest.BlockHeaderFixture() diff --git a/module/mempool/herocache/backdata/heropool/pool.go b/module/mempool/herocache/backdata/heropool/pool.go index 33bfa34163b..c22f5db8e99 100644 --- a/module/mempool/herocache/backdata/heropool/pool.go +++ b/module/mempool/herocache/backdata/heropool/pool.go @@ -1,9 +1,8 @@ package heropool import ( - "math/rand" - "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/rand" ) type EjectionMode string @@ -94,8 +93,10 @@ func (p *Pool) initFreeEntities() { // If the pool has an available slot (either empty or by ejection), then the second boolean returned value (ejectionOccurred) // determines whether an ejection happened to make one slot free or not. Ejection happens if there is no available // slot, and there is an ejection mode set. -func (p *Pool) Add(entityId flow.Identifier, entity flow.Entity, owner uint64) (i EIndex, slotAvailable bool, ejectionOccurred bool) { - entityIndex, slotAvailable, ejectionHappened := p.sliceIndexForEntity() +func (p *Pool) Add(entityId flow.Identifier, entity flow.Entity, owner uint64) ( + entityIndex EIndex, slotAvailable bool, ejectionOccurred bool) { + entityIndex, slotAvailable, ejectionOccurred = p.sliceIndexForEntity() + if slotAvailable { p.poolEntities[entityIndex].entity = entity p.poolEntities[entityIndex].id = entityId @@ -120,7 +121,7 @@ func (p *Pool) Add(entityId flow.Identifier, entity flow.Entity, owner uint64) ( p.size++ } - return entityIndex, slotAvailable, ejectionHappened + return entityIndex, slotAvailable, ejectionOccurred } // Get returns entity corresponding to the entity index from the underlying list. @@ -160,7 +161,7 @@ func (p Pool) Head() (flow.Entity, bool) { // If the pool has an available slot (either empty or by ejection), then the second boolean returned value // (ejectionOccurred) determines whether an ejection happened to make one slot free or not. // Ejection happens if there is no available slot, and there is an ejection mode set. -func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectionOccurred bool) { +func (p *Pool) sliceIndexForEntity() (EIndex, bool, bool) { if p.free.head.isUndefined() { // the free list is empty, so we are out of space, and we need to eject. switch p.ejectionMode { @@ -174,7 +175,13 @@ func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectionO return p.claimFreeHead(), true, true case RandomEjection: // we only eject randomly when the pool is full and random ejection is on. - randomIndex := EIndex(rand.Uint32() % p.size) + random, err := rand.Uint32n(p.size) + if err != nil { + // TODO: to check with Yahya + // randomness failed and no ejection has happened + return 0, false, false + } + randomIndex := EIndex(random) p.invalidateEntityAtIndex(randomIndex) return p.claimFreeHead(), true, true } diff --git a/module/mempool/herocache/dns_cache.go b/module/mempool/herocache/dns_cache.go index db4c9a9b67b..9af171c39ae 100644 --- a/module/mempool/herocache/dns_cache.go +++ b/module/mempool/herocache/dns_cache.go @@ -19,7 +19,8 @@ type DNSCache struct { txtCache *stdmap.Backend } -func NewDNSCache(sizeLimit uint32, logger zerolog.Logger, ipCollector module.HeroCacheMetrics, txtCollector module.HeroCacheMetrics) *DNSCache { +func NewDNSCache(sizeLimit uint32, logger zerolog.Logger, ipCollector module.HeroCacheMetrics, txtCollector module.HeroCacheMetrics, +) *DNSCache { return &DNSCache{ txtCache: stdmap.NewBackend( stdmap.WithBackData( diff --git a/module/mempool/herocache/transactions.go b/module/mempool/herocache/transactions.go index a052728de52..e8784c6a851 100644 --- a/module/mempool/herocache/transactions.go +++ b/module/mempool/herocache/transactions.go @@ -18,14 +18,16 @@ type Transactions struct { // NewTransactions implements a transactions mempool based on hero cache. func NewTransactions(limit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *Transactions { + cache := herocache.NewCache(limit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + logger.With().Str("mempool", "transactions").Logger(), + collector) + t := &Transactions{ c: stdmap.NewBackend( stdmap.WithBackData( - herocache.NewCache(limit, - herocache.DefaultOversizeFactor, - heropool.LRUEjection, - logger.With().Str("mempool", "transactions").Logger(), - collector))), + cache)), } return t diff --git a/module/mempool/mock/back_data.go b/module/mempool/mock/back_data.go index d66eab22e62..a31b7d381a9 100644 --- a/module/mempool/mock/back_data.go +++ b/module/mempool/mock/back_data.go @@ -90,8 +90,17 @@ func (_m *BackData) ByID(entityID flow.Identifier) (flow.Entity, bool) { } // Clear provides a mock function with given fields: -func (_m *BackData) Clear() { - _m.Called() +func (_m *BackData) Clear() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 } // Entities provides a mock function with given fields: diff --git a/module/mempool/queue/heroQueue.go b/module/mempool/queue/heroQueue.go index ec1269147b8..52274fd4a81 100644 --- a/module/mempool/queue/heroQueue.go +++ b/module/mempool/queue/heroQueue.go @@ -19,14 +19,18 @@ type HeroQueue struct { sizeLimit uint } -func NewHeroQueue(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *HeroQueue { +func NewHeroQueue(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *HeroQueue { + + cache := herocache.NewCache( + sizeLimit, + herocache.DefaultOversizeFactor, + heropool.NoEjection, + logger.With().Str("mempool", "hero-queue").Logger(), + collector) + return &HeroQueue{ - cache: herocache.NewCache( - sizeLimit, - herocache.DefaultOversizeFactor, - heropool.NoEjection, - logger.With().Str("mempool", "hero-queue").Logger(), - collector), + cache: cache, sizeLimit: uint(sizeLimit), } } diff --git a/module/mempool/queue/heroQueue_test.go b/module/mempool/queue/heroQueue_test.go index 75396a9b1ed..494dba7ae78 100644 --- a/module/mempool/queue/heroQueue_test.go +++ b/module/mempool/queue/heroQueue_test.go @@ -59,7 +59,6 @@ func TestHeroQueue_Sequential(t *testing.T) { func TestHeroQueue_Concurrent(t *testing.T) { sizeLimit := 100 q := queue.NewHeroQueue(uint32(sizeLimit), unittest.Logger(), metrics.NewNoopCollector()) - // initially queue must be zero require.Zero(t, q.Size()) diff --git a/module/mempool/queue/heroStore.go b/module/mempool/queue/heroStore.go index 8606b9a3010..150e9b17ae4 100644 --- a/module/mempool/queue/heroStore.go +++ b/module/mempool/queue/heroStore.go @@ -14,9 +14,12 @@ type HeroStore struct { q *HeroQueue } -func NewHeroStore(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *HeroStore { +func NewHeroStore(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *HeroStore { + queue := NewHeroQueue(sizeLimit, logger, collector) + return &HeroStore{ - q: NewHeroQueue(sizeLimit, logger, collector), + q: queue, } } diff --git a/module/mempool/stdmap/backDataHeapBenchmark_test.go b/module/mempool/stdmap/backDataHeapBenchmark_test.go index 1a3fdbc7e17..4b9d7fc7c35 100644 --- a/module/mempool/stdmap/backDataHeapBenchmark_test.go +++ b/module/mempool/stdmap/backDataHeapBenchmark_test.go @@ -46,14 +46,16 @@ func BenchmarkArrayBackDataLRU(b *testing.B) { defer debug.SetGCPercent(debug.SetGCPercent(-1)) // disable GC limit := uint(50_000) + cache := herocache.NewCache( + uint32(limit), + 8, + heropool.LRUEjection, + unittest.Logger(), + metrics.NewNoopCollector()) + backData := stdmap.NewBackend( stdmap.WithBackData( - herocache.NewCache( - uint32(limit), - 8, - heropool.LRUEjection, - unittest.Logger(), - metrics.NewNoopCollector())), + cache), stdmap.WithLimit(limit)) entities := unittest.EntityListFixture(uint(100_000_000)) diff --git a/module/mempool/stdmap/backend.go b/module/mempool/stdmap/backend.go index cb0dca2640d..fb42e5297d5 100644 --- a/module/mempool/stdmap/backend.go +++ b/module/mempool/stdmap/backend.go @@ -23,12 +23,12 @@ type Backend struct { } // NewBackend creates a new memory pool backend. -// This is using EjectTrueRandomFast() +// This is using EjectRandomFast() func NewBackend(options ...OptionFunc) *Backend { b := Backend{ backData: backdata.NewMapBackData(), guaranteedCapacity: uint(math.MaxUint32), - batchEject: EjectTrueRandomFast, + batchEject: EjectRandomFast, eject: nil, ejectionCallbacks: nil, } @@ -185,14 +185,14 @@ func (b *Backend) reduce() { //defer binstat.Leave(bs) // we keep reducing the cache size until we are at limit again - // this was a loop, but the loop is now in EjectTrueRandomFast() + // this was a loop, but the loop is now in EjectRandomFast() // the ejections are batched, so this call to eject() may not actually // do anything until the batch threshold is reached (currently 128) if b.backData.Size() > b.guaranteedCapacity { // get the key from the eject function // we don't do anything if there is an error if b.batchEject != nil { - _ = b.batchEject(b) + _, _ = b.batchEject(b) } else { _, _, _ = b.eject(b) } diff --git a/module/mempool/stdmap/eject.go b/module/mempool/stdmap/eject.go index 3ed2d59683a..2e52d7320bd 100644 --- a/module/mempool/stdmap/eject.go +++ b/module/mempool/stdmap/eject.go @@ -3,12 +3,11 @@ package stdmap import ( - "math" - "math/rand" + "fmt" "sort" - "sync" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/rand" ) // this is the threshold for how much over the guaranteed capacity the @@ -31,49 +30,33 @@ const overCapacityThreshold = 128 // concurrency (specifically, it locks the mempool during ejection). // - The implementation should be non-blocking (though, it is allowed to // take a bit of time; the mempool will just be locked during this time). -type BatchEjectFunc func(b *Backend) bool +type BatchEjectFunc func(b *Backend) (bool, error) type EjectFunc func(b *Backend) (flow.Identifier, flow.Entity, bool) -// EjectTrueRandom relies on a random generator to pick a random entity to eject from the -// entity set. It will, on average, iterate through half the entities of the set. However, -// it provides us with a truly evenly distributed random selection. -func EjectTrueRandom(b *Backend) (flow.Identifier, flow.Entity, bool) { - var entity flow.Entity - var entityID flow.Identifier - - bFound := false - i := 0 - n := rand.Intn(int(b.backData.Size())) - for entityID, entity = range b.backData.All() { - if i == n { - bFound = true - break - } - i++ - } - return entityID, entity, bFound -} - -// EjectTrueRandomFast checks if the map size is beyond the +// EjectRandomFast checks if the map size is beyond the // threshold size, and will iterate through them and eject unneeded // entries if that is the case. Return values are unused -func EjectTrueRandomFast(b *Backend) bool { +func EjectRandomFast(b *Backend) (bool, error) { currentSize := b.backData.Size() if b.guaranteedCapacity >= currentSize { - return false + return false, nil } // At this point, we know that currentSize > b.guaranteedCapacity. As // currentSize fits into an int, b.guaranteedCapacity must also fit. overcapacity := currentSize - b.guaranteedCapacity if overcapacity <= overCapacityThreshold { - return false + return false, nil } // Randomly select indices of elements to remove: mapIndices := make([]int, 0, overcapacity) for i := overcapacity; i > 0; i-- { - mapIndices = append(mapIndices, rand.Intn(int(currentSize))) + rand, err := rand.Uintn(currentSize) + if err != nil { + return false, fmt.Errorf("random generation failed: %w", err) + } + mapIndices = append(mapIndices, int(rand)) } sort.Ints(mapIndices) // inplace @@ -99,13 +82,13 @@ func EjectTrueRandomFast(b *Backend) bool { } if idx == int(overcapacity) { - return true + return true, nil } next2Remove = mapIndices[idx] } i++ } - return true + return true, nil } // EjectPanic simply panics, crashing the program. Useful when cache is not expected @@ -113,77 +96,3 @@ func EjectTrueRandomFast(b *Backend) bool { func EjectPanic(b *Backend) (flow.Identifier, flow.Entity, bool) { panic("unexpected: mempool size over the limit") } - -// LRUEjector provides a swift FIFO ejection functionality -type LRUEjector struct { - sync.Mutex - table map[flow.Identifier]uint64 // keeps sequence number of entities it tracks - seqNum uint64 // keeps the most recent sequence number -} - -func NewLRUEjector() *LRUEjector { - return &LRUEjector{ - table: make(map[flow.Identifier]uint64), - seqNum: 0, - } -} - -// Track should be called every time a new entity is added to the mempool. -// It tracks the entity for later ejection. -func (q *LRUEjector) Track(entityID flow.Identifier) { - q.Lock() - defer q.Unlock() - - if _, ok := q.table[entityID]; ok { - // skips adding duplicate item - return - } - - // TODO current table structure provides O(1) track and untrack features - // however, the Eject functionality is asymptotically O(n). - // With proper resource cleanups by the mempools, the Eject is supposed - // as a very infrequent operation. However, further optimizations on - // Eject efficiency is needed. - q.table[entityID] = q.seqNum - q.seqNum++ -} - -// Untrack simply removes the tracker of the ejector off the entityID -func (q *LRUEjector) Untrack(entityID flow.Identifier) { - q.Lock() - defer q.Unlock() - - delete(q.table, entityID) -} - -// Eject implements EjectFunc for LRUEjector. It finds the entity with the lowest sequence number (i.e., -// the oldest entity). It also untracks. This is using a linear search -func (q *LRUEjector) Eject(b *Backend) (flow.Identifier, flow.Entity, bool) { - q.Lock() - defer q.Unlock() - - // finds the oldest entity - oldestSQ := uint64(math.MaxUint64) - var oldestID flow.Identifier - for _, id := range b.backData.Identifiers() { - if sq, ok := q.table[id]; ok { - if sq < oldestSQ { - oldestID = id - oldestSQ = sq - } - - } - } - - // TODO: don't do a lookup if it isn't necessary - oldestEntity, ok := b.backData.ByID(oldestID) - - if !ok { - oldestID, oldestEntity, ok = EjectTrueRandom(b) - } - - // untracks the oldest id as it is supposed to be ejected - delete(q.table, oldestID) - - return oldestID, oldestEntity, ok -} diff --git a/module/mempool/stdmap/eject_test.go b/module/mempool/stdmap/eject_test.go deleted file mode 100644 index cee1974e840..00000000000 --- a/module/mempool/stdmap/eject_test.go +++ /dev/null @@ -1,230 +0,0 @@ -package stdmap - -import ( - crand "crypto/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" -) - -// TestLRUEjector_Track evaluates that tracking a new item adds the item to the ejector table. -func TestLRUEjector_Track(t *testing.T) { - ejector := NewLRUEjector() - // ejector's table should be empty - assert.Len(t, ejector.table, 0) - - // sequence number of ejector should initially be zero - assert.Equal(t, ejector.seqNum, uint64(0)) - - // creates adds an item to the ejector - item := flow.Identifier{0x00} - ejector.Track(item) - - // size of ejector's table should be one - // which indicates that ejector is tracking the item - assert.Len(t, ejector.table, 1) - - // item should reside in the ejector's table - _, ok := ejector.table[item] - assert.True(t, ok) - - // sequence number of ejector should be increased by one - assert.Equal(t, ejector.seqNum, uint64(1)) -} - -// TestLRUEjector_Track_Duplicate evaluates that tracking a duplicate item -// does not change the internal state of the ejector. -func TestLRUEjector_Track_Duplicate(t *testing.T) { - ejector := NewLRUEjector() - - // creates adds an item to the ejector - item := flow.Identifier{0x00} - ejector.Track(item) - - // size of ejector's table should be one - // which indicates that ejector is tracking the item - assert.Len(t, ejector.table, 1) - - // item should reside in the ejector's table - _, ok := ejector.table[item] - assert.True(t, ok) - - // sequence number of ejector should be increased by one - assert.Equal(t, ejector.seqNum, uint64(1)) - - // adds the duplicate item - ejector.Track(item) - - // internal state of the ejector should be unchaged - assert.Len(t, ejector.table, 1) - assert.Equal(t, ejector.seqNum, uint64(1)) - _, ok = ejector.table[item] - assert.True(t, ok) -} - -// TestLRUEjector_Track_Many evaluates that tracking many items -// changes the state of ejector properly, i.e., items reside on the -// memory, and sequence number changed accordingly. -func TestLRUEjector_Track_Many(t *testing.T) { - ejector := NewLRUEjector() - - // creates and tracks 100 items - size := 100 - items := flow.IdentifierList{} - for i := 0; i < size; i++ { - var id flow.Identifier - _, _ = crand.Read(id[:]) - ejector.Track(id) - items = append(items, id) - } - - // size of ejector's table should be 100 - assert.Len(t, ejector.table, size) - - // all items should reside in the ejector's table - for _, id := range items { - _, ok := ejector.table[id] - require.True(t, ok) - } - - // sequence number of ejector should be increased by size - assert.Equal(t, ejector.seqNum, uint64(size)) -} - -// TestLRUEjector_Untrack_One evaluates that untracking an existing item -// removes it from the ejector state and changes the state accordingly. -func TestLRUEjector_Untrack_One(t *testing.T) { - ejector := NewLRUEjector() - - // creates adds an item to the ejector - item := flow.Identifier{0x00} - ejector.Track(item) - - // size of ejector's table should be one - // which indicates that ejector is tracking the item - assert.Len(t, ejector.table, 1) - - // item should reside in the ejector's table - _, ok := ejector.table[item] - assert.True(t, ok) - - // sequence number of ejector should be increased by one - assert.Equal(t, ejector.seqNum, uint64(1)) - - // untracks the item - ejector.Untrack(item) - - // internal state of the ejector should be changed - assert.Len(t, ejector.table, 0) - - // sequence number should not be changed - assert.Equal(t, ejector.seqNum, uint64(1)) - - // item should no longer reside on internal state of ejector - _, ok = ejector.table[item] - assert.False(t, ok) -} - -// TestLRUEjector_Untrack_Duplicate evaluates that untracking an item twice -// removes it from the ejector state only once and changes the state safely. -func TestLRUEjector_Untrack_Duplicate(t *testing.T) { - ejector := NewLRUEjector() - - // creates and adds two items to the ejector - item1 := flow.Identifier{0x00} - item2 := flow.Identifier{0x01} - ejector.Track(item1) - ejector.Track(item2) - - // size of ejector's table should be two - // which indicates that ejector is tracking the items - assert.Len(t, ejector.table, 2) - - // items should reside in the ejector's table - _, ok := ejector.table[item1] - assert.True(t, ok) - _, ok = ejector.table[item2] - assert.True(t, ok) - - // sequence number of ejector should be increased by two - assert.Equal(t, ejector.seqNum, uint64(2)) - - // untracks the item twice - ejector.Untrack(item1) - ejector.Untrack(item1) - - // internal state of the ejector should be changed - assert.Len(t, ejector.table, 1) - - // sequence number should not be changed - assert.Equal(t, ejector.seqNum, uint64(2)) - - // double untracking should only affect the untracked item1 - _, ok = ejector.table[item1] - assert.False(t, ok) - - // item 2 should still reside in the memory - _, ok = ejector.table[item2] - assert.True(t, ok) -} - -// TestLRUEjector_UntrackEject evaluates that untracking the next ejectable item -// properly changes the next ejectable item in the ejector. -func TestLRUEjector_UntrackEject(t *testing.T) { - ejector := NewLRUEjector() - - // creates and tracks 100 items - size := 100 - backEnd := NewBackend() - - items := make([]flow.Identifier, size) - - for i := 0; i < size; i++ { - mockEntity := unittest.MockEntityFixture() - require.True(t, backEnd.Add(mockEntity)) - - id := mockEntity.ID() - ejector.Track(id) - items[i] = id - } - - // untracks the oldest item - ejector.Untrack(items[0]) - - // next ejectable item should be the second oldest item - id, _, _ := ejector.Eject(backEnd) - assert.Equal(t, id, items[1]) -} - -// TestLRUEjector_EjectAll adds many item to the ejector and then ejects them -// all one by one and evaluates an LRU ejection behavior. -func TestLRUEjector_EjectAll(t *testing.T) { - ejector := NewLRUEjector() - - // creates and tracks 100 items - size := 100 - backEnd := NewBackend() - - items := make([]flow.Identifier, size) - - for i := 0; i < size; i++ { - mockEntity := unittest.MockEntityFixture() - require.True(t, backEnd.Add(mockEntity)) - - id := mockEntity.ID() - ejector.Track(id) - items[i] = id - } - - require.Equal(t, uint(size), backEnd.Size()) - - // ejects one by one - for i := 0; i < size; i++ { - id, _, _ := ejector.Eject(backEnd) - require.Equal(t, id, items[i]) - } -} diff --git a/module/signature/aggregation_test.go b/module/signature/aggregation_test.go index 243b8f06551..de40002f00f 100644 --- a/module/signature/aggregation_test.go +++ b/module/signature/aggregation_test.go @@ -4,7 +4,6 @@ package signature import ( - "crypto/rand" mrand "math/rand" "sort" "testing" @@ -16,7 +15,14 @@ import ( "github.com/onflow/flow-go/crypto" ) -func createAggregationData(t *testing.T, signersNumber int) (*SignatureAggregatorSameMessage, []crypto.Signature) { +func getPRG(t *testing.T) *mrand.Rand { + random := time.Now().UnixNano() + t.Logf("rng seed is %d", random) + rng := mrand.New(mrand.NewSource(random)) + return rng +} + +func createAggregationData(t *testing.T, rand *mrand.Rand, signersNumber int) (*SignatureAggregatorSameMessage, []crypto.Signature) { // create message and tag msgLen := 100 msg := make([]byte, msgLen) @@ -43,7 +49,7 @@ func createAggregationData(t *testing.T, signersNumber int) (*SignatureAggregato } func TestAggregatorSameMessage(t *testing.T) { - + rand := getPRG(t) signersNum := 20 // constructor edge cases @@ -67,7 +73,7 @@ func TestAggregatorSameMessage(t *testing.T) { // Happy paths t.Run("happy path", func(t *testing.T) { - aggregator, sigs := createAggregationData(t, signersNum) + aggregator, sigs := createAggregationData(t, rand, signersNum) // only add half of the signatures subSet := signersNum / 2 for i, sig := range sigs[subSet:] { @@ -127,7 +133,7 @@ func TestAggregatorSameMessage(t *testing.T) { // Unhappy paths t.Run("invalid inputs", func(t *testing.T) { - aggregator, sigs := createAggregationData(t, signersNum) + aggregator, sigs := createAggregationData(t, rand, signersNum) // loop through invalid inputs for _, index := range []int{-1, signersNum} { ok, err := aggregator.Verify(index, sigs[0]) @@ -156,7 +162,7 @@ func TestAggregatorSameMessage(t *testing.T) { }) t.Run("duplicate signature", func(t *testing.T) { - aggregator, sigs := createAggregationData(t, signersNum) + aggregator, sigs := createAggregationData(t, rand, signersNum) for i, sig := range sigs { err := aggregator.TrustedAdd(i, sig) require.NoError(t, err) @@ -182,12 +188,12 @@ func TestAggregatorSameMessage(t *testing.T) { // 2. The signature was deserialized successfully, but the aggregate signature doesn't verify to the aggregate public key. In // this case, the aggregation step succeeds. But the post-check fails. t.Run("invalid signature", func(t *testing.T) { - _, s := createAggregationData(t, 1) + _, s := createAggregationData(t, rand, 1) invalidStructureSig := (crypto.Signature)([]byte{0, 0}) mismatchingSig := s[0] for _, invalidSig := range []crypto.Signature{invalidStructureSig, mismatchingSig} { - aggregator, sigs := createAggregationData(t, signersNum) + aggregator, sigs := createAggregationData(t, rand, signersNum) ok, err := aggregator.VerifyAndAdd(0, sigs[0]) // first, add a valid signature require.NoError(t, err) assert.True(t, ok) @@ -221,9 +227,7 @@ func TestAggregatorSameMessage(t *testing.T) { } func TestKeyAggregator(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) signersNum := 20 // create keys @@ -305,8 +309,8 @@ func TestKeyAggregator(t *testing.T) { rounds := 30 for i := 0; i < rounds; i++ { go func() { // test module concurrency - low := mrand.Intn(signersNum - 1) - high := low + 1 + mrand.Intn(signersNum-1-low) + low := rand.Intn(signersNum - 1) + high := low + 1 + rand.Intn(signersNum-1-low) var key, expectedKey crypto.PublicKey var err error key, err = aggregator.KeyAggregate(indices[low:high]) diff --git a/module/signature/signer_indices_test.go b/module/signature/signer_indices_test.go index c34daea4f37..0bd7aaee34e 100644 --- a/module/signature/signer_indices_test.go +++ b/module/signature/signer_indices_test.go @@ -112,7 +112,7 @@ func Test_EncodeSignerToIndicesAndSigType(t *testing.T) { // create committee committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := committeeIdentities.NodeIDs() - stakingSigners, beaconSigners := sampleSigners(committee, numStakingSigners, numRandomBeaconSigners) + stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) // encode prefixed, sigTypes, err := signature.EncodeSignerToIndicesAndSigType(committee, stakingSigners, beaconSigners) @@ -150,7 +150,7 @@ func Test_DecodeSigTypeToStakingAndBeaconSigners(t *testing.T) { // create committee committeeIdentities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := committeeIdentities.NodeIDs() - stakingSigners, beaconSigners := sampleSigners(committee, numStakingSigners, numRandomBeaconSigners) + stakingSigners, beaconSigners := sampleSigners(t, committee, numStakingSigners, numRandomBeaconSigners) // encode signerIndices, sigTypes, err := signature.EncodeSignerToIndicesAndSigType(committee, stakingSigners, beaconSigners) @@ -276,7 +276,8 @@ func Test_EncodeSignersToIndices(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := identities.NodeIDs() - signers := committee.Sample(uint(numSigners)) + signers, err := committee.Sample(uint(numSigners)) + require.NoError(t, err) // encode prefixed, err := signature.EncodeSignersToIndices(committee, signers) @@ -305,7 +306,8 @@ func Test_DecodeSignerIndicesToIdentifiers(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) committee := identities.NodeIDs() - signers := committee.Sample(uint(numSigners)) + signers, err := committee.Sample(uint(numSigners)) + require.NoError(t, err) sort.Sort(signers) // encode @@ -340,7 +342,8 @@ func Test_DecodeSignerIndicesToIdentities(t *testing.T) { // create committee identities := unittest.IdentityListFixture(committeeSize, unittest.WithRole(flow.RoleConsensus)).Sort(order.Canonical) - signers := identities.Sample(uint(numSigners)) + signers, err := identities.Sample(uint(numSigners)) + require.NoError(t, err) // encode signerIndices, err := signature.EncodeSignersToIndices(identities.NodeIDs(), signers.NodeIDs()) @@ -356,6 +359,7 @@ func Test_DecodeSignerIndicesToIdentities(t *testing.T) { // sampleSigners takes `committee` and samples to _disjoint_ subsets // (`stakingSigners` and `randomBeaconSigners`) with the specified cardinality func sampleSigners( + t *rapid.T, committee flow.IdentifierList, numStakingSigners int, numRandomBeaconSigners int, @@ -364,9 +368,12 @@ func sampleSigners( panic(fmt.Sprintf("Cannot sample %d nodes out of a committee is size %d", numStakingSigners+numRandomBeaconSigners, len(committee))) } - stakingSigners = committee.Sample(uint(numStakingSigners)) + var err error + stakingSigners, err = committee.Sample(uint(numStakingSigners)) + require.NoError(t, err) remaining := committee.Filter(id.Not(id.In(stakingSigners...))) - randomBeaconSigners = remaining.Sample(uint(numRandomBeaconSigners)) + randomBeaconSigners, err = remaining.Sample(uint(numRandomBeaconSigners)) + require.NoError(t, err) return } diff --git a/module/state_synchronization/requester/execution_data_requester_test.go b/module/state_synchronization/requester/execution_data_requester_test.go index e2e01cb7929..ed9a3ebc268 100644 --- a/module/state_synchronization/requester/execution_data_requester_test.go +++ b/module/state_synchronization/requester/execution_data_requester_test.go @@ -51,7 +51,6 @@ type ExecutionDataRequesterSuite struct { func TestExecutionDataRequesterSuite(t *testing.T) { t.Parallel() - rand.Seed(time.Now().UnixMilli()) suite.Run(t, new(ExecutionDataRequesterSuite)) } diff --git a/module/state_synchronization/requester/jobs/execution_data_reader_test.go b/module/state_synchronization/requester/jobs/execution_data_reader_test.go index 35547851c53..dbc6981a539 100644 --- a/module/state_synchronization/requester/jobs/execution_data_reader_test.go +++ b/module/state_synchronization/requester/jobs/execution_data_reader_test.go @@ -3,7 +3,6 @@ package jobs import ( "context" "errors" - "math/rand" "testing" "time" @@ -42,7 +41,6 @@ type ExecutionDataReaderSuite struct { func TestExecutionDataReaderSuite(t *testing.T) { t.Parallel() - rand.Seed(time.Now().UnixMilli()) suite.Run(t, new(ExecutionDataReaderSuite)) } diff --git a/module/trace/trace_test.go b/module/trace/trace_test.go index c98a632d4a9..f1011589930 100644 --- a/module/trace/trace_test.go +++ b/module/trace/trace_test.go @@ -2,7 +2,7 @@ package trace import ( "context" - "math/rand" + "crypto/rand" "testing" "github.com/rs/zerolog" diff --git a/network/cache/rcvcache.go b/network/cache/rcvcache.go index be685ae670d..bdab2ad894a 100644 --- a/network/cache/rcvcache.go +++ b/network/cache/rcvcache.go @@ -29,7 +29,8 @@ func (r receiveCacheEntry) Checksum() flow.Identifier { } // NewHeroReceiveCache returns a new HeroCache-based receive cache. -func NewHeroReceiveCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *ReceiveCache { +func NewHeroReceiveCache(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, +) *ReceiveCache { backData := herocache.NewCache(sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, // receive cache must be LRU. diff --git a/network/p2p/cache/node_blocklist_wrapper_test.go b/network/p2p/cache/node_blocklist_wrapper_test.go index b42b9eac15f..57a7f8c6c9b 100644 --- a/network/p2p/cache/node_blocklist_wrapper_test.go +++ b/network/p2p/cache/node_blocklist_wrapper_test.go @@ -137,7 +137,8 @@ func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { blocklistLookup := blocklist.Lookup() honestIdentities := unittest.IdentityListFixture(8) combinedIdentities := honestIdentities.Union(blocklist) - combinedIdentities = combinedIdentities.DeterministicShuffle(1234) + combinedIdentities, err = combinedIdentities.Shuffle() + require.NoError(s.T(), err) numIdentities := len(combinedIdentities) s.provider.On("Identities", mock.Anything).Return(combinedIdentities) @@ -164,7 +165,8 @@ func (s *NodeBlocklistWrapperTestSuite) TestDenylistedNode() { blocklistLookup := blocklist.Lookup() honestIdentities := unittest.IdentityListFixture(8) combinedIdentities := honestIdentities.Union(blocklist) - combinedIdentities = combinedIdentities.DeterministicShuffle(1234) + combinedIdentities, err = combinedIdentities.Shuffle() + require.NoError(s.T(), err) numIdentities := len(combinedIdentities) s.provider.On("Identities", mock.Anything).Return(combinedIdentities) diff --git a/network/p2p/connection/connector.go b/network/p2p/connection/connector.go index 5c25921a520..781bddc71c7 100644 --- a/network/p2p/connection/connector.go +++ b/network/p2p/connection/connector.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "math/rand" + mrand "math/rand" "time" "github.com/hashicorp/go-multierror" @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/network/internal/p2putils" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -165,11 +166,17 @@ func (l *Libp2pConnector) pruneAllConnectionsExcept(peerIDs peer.IDSlice) { // defaultLibp2pBackoffConnector creates a default libp2p backoff connector similar to the one created by libp2p.pubsub // (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) func defaultLibp2pBackoffConnector(host host.Host) (*discoveryBackoff.BackoffConnector, error) { - rngSrc := rand.NewSource(rand.Int63()) + r, err := rand.Uint64() + if err != nil { + return nil, fmt.Errorf("failed to create backoff connector: %w", err) + } + // math/rand is used as // NewExponentialBackoff is based on a math/rand parameter which is used as a jitter is based on a math/rand parameter. + // the random source is used as a jitter in NewExponentialBackoff + rng := mrand.New(mrand.NewSource(int64(r))) minBackoff, maxBackoff := time.Second*10, time.Hour cacheSize := 100 dialTimeout := time.Minute * 2 - backoff := discoveryBackoff.NewExponentialBackoff(minBackoff, maxBackoff, discoveryBackoff.FullJitter, time.Second, 5.0, 0, rand.New(rngSrc)) + backoff := discoveryBackoff.NewExponentialBackoff(minBackoff, maxBackoff, discoveryBackoff.FullJitter, time.Second, 5.0, 0, rng) backoffConnector, err := discoveryBackoff.NewBackoffConnector(host, cacheSize, dialTimeout, backoff) if err != nil { return nil, fmt.Errorf("failed to create backoff connector: %w", err) diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index ded4a58c746..7ee014e3dc6 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -3,7 +3,7 @@ package connection import ( "context" "fmt" - mrand "math/rand" + "sync" "time" @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/logging" + "github.com/onflow/flow-go/utils/rand" ) // DefaultPeerUpdateInterval is default duration for which the peer manager waits in between attempts to update peer connections @@ -85,7 +86,10 @@ func (pm *PeerManager) updateLoop(ctx irrecoverable.SignalerContext) { func (pm *PeerManager) periodicLoop(ctx irrecoverable.SignalerContext) { // add a random delay to initial launch to avoid synchronizing this // potentially expensive operation across the network - delay := time.Duration(mrand.Int63n(pm.peerUpdateInterval.Nanoseconds())) + r, _ := rand.Uint64n(uint64(pm.peerUpdateInterval.Nanoseconds())) + // ignore the error here, if randomness fails `r` would be zero and there will be no delay + // for the current node + delay := time.Duration(r) ticker := time.NewTicker(pm.peerUpdateInterval) defer ticker.Stop() diff --git a/network/p2p/network.go b/network/p2p/network.go index b5bf83c8c11..6941f537753 100644 --- a/network/p2p/network.go +++ b/network/p2p/network.go @@ -418,13 +418,16 @@ func (n *Network) PublishOnChannel(channel channels.Channel, message interface{} // MulticastOnChannel unreliably sends the specified event over the channel to randomly selected 'num' number of recipients // selected from the specified targetIDs. func (n *Network) MulticastOnChannel(channel channels.Channel, message interface{}, num uint, targetIDs ...flow.Identifier) error { - selectedIDs := flow.IdentifierList(targetIDs).Filter(n.removeSelfFilter()).Sample(num) + selectedIDs, err := flow.IdentifierList(targetIDs).Filter(n.removeSelfFilter()).Sample(num) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } if len(selectedIDs) == 0 { return network.EmptyTargetList } - err := n.sendOnChannel(channel, message, selectedIDs) + err = n.sendOnChannel(channel, message, selectedIDs) // publishes the message to the selected targets if err != nil { diff --git a/network/p2p/unicast/manager.go b/network/p2p/unicast/manager.go index 020ce4d390b..7ecb88e2eef 100644 --- a/network/p2p/unicast/manager.go +++ b/network/p2p/unicast/manager.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "math/rand" "strings" "time" @@ -17,6 +16,7 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/rand" ) // MaxConnectAttemptSleepDuration is the maximum number of milliseconds to wait between attempts for a 1-1 direct connection @@ -143,8 +143,11 @@ func (m *Manager) rawStreamWithProtocol(ctx context.Context, if retries > 0 { // choose a random interval between 0 to 5 // (to ensure that this node and the target node don't attempt to reconnect at the same time) - r := rand.Intn(MaxConnectAttemptSleepDuration) - time.Sleep(time.Duration(r) * time.Millisecond) + r, err := rand.Uintn(uint(MaxConnectAttemptSleepDuration)) + if err != nil { + return s, dialAddr, fmt.Errorf("failed to generate randomness: %w", err) + } + time.Sleep(time.Duration(int64(r)) * time.Millisecond) } err := m.streamFactory.Connect(ctx, peer.AddrInfo{ID: peerID}) diff --git a/network/queue/messageQueue_test.go b/network/queue/messageQueue_test.go index 159ce7506cb..5fd7cf86839 100644 --- a/network/queue/messageQueue_test.go +++ b/network/queue/messageQueue_test.go @@ -217,7 +217,7 @@ func createMessages(messageCnt int, priorityFunc queue.MessagePriorityFunc) map[ } func randomPriority(_ interface{}) (queue.Priority, error) { - rand.Seed(time.Now().UnixNano()) + p := rand.Intn(int(queue.HighPriority-queue.LowPriority+1)) + int(queue.LowPriority) return queue.Priority(p), nil } diff --git a/network/stub/network.go b/network/stub/network.go index ef99b3e39aa..56f96cbad9b 100644 --- a/network/stub/network.go +++ b/network/stub/network.go @@ -157,7 +157,11 @@ func (n *Network) PublishOnChannel(channel channels.Channel, event interface{}, // Engines attached to the same channel on other nodes. The targeted nodes are selected based on the selector. // In this test helper implementation, multicast uses submit method under the hood. func (n *Network) MulticastOnChannel(channel channels.Channel, event interface{}, num uint, targetIDs ...flow.Identifier) error { - targetIDs = flow.Sample(num, targetIDs...) + var err error + targetIDs, err = flow.Sample(num, targetIDs...) + if err != nil { + return fmt.Errorf("sampling failed: %w", err) + } return n.submit(channel, event, targetIDs...) } diff --git a/network/test/epochtransition_test.go b/network/test/epochtransition_test.go index 8b7c0a655bd..881ba5f17a3 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/epochtransition_test.go @@ -135,7 +135,7 @@ func (suite *MutableIdentityTableSuite) signalIdentityChanged() { func (suite *MutableIdentityTableSuite) SetupTest() { suite.testNodes = newTestNodeList() suite.removedTestNodes = newTestNodeList() - rand.Seed(time.Now().UnixNano()) + nodeCount := 10 suite.logger = zerolog.New(os.Stderr).Level(zerolog.ErrorLevel) log.SetAllLoggers(log.LevelError) diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index a81f4ebb248..4d8ccbc256a 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -7,7 +7,6 @@ import ( "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -52,7 +51,6 @@ func (suite *MutatorSuite) SetupTest() { var err error // seed the RNG - rand.Seed(time.Now().UnixNano()) suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID diff --git a/state/cluster/badger/snapshot_test.go b/state/cluster/badger/snapshot_test.go index d2dc9aee15b..efb42150774 100644 --- a/state/cluster/badger/snapshot_test.go +++ b/state/cluster/badger/snapshot_test.go @@ -2,10 +2,8 @@ package badger import ( "math" - "math/rand" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -44,7 +42,6 @@ func (suite *SnapshotSuite) SetupTest() { var err error // seed the RNG - rand.Seed(time.Now().UnixNano()) suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index d836fb30675..d0d95e121be 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -40,7 +40,7 @@ import ( ) func init() { - rand.Seed(time.Now().UnixNano()) + } var participants = unittest.IdentityListFixture(5, unittest.WithAllRoles()) diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index dd0d24f9e7f..07169cb76e2 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -7,7 +7,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -27,7 +26,7 @@ import ( ) func init() { - rand.Seed(time.Now().UnixNano()) + } func TestHead(t *testing.T) { @@ -148,16 +147,18 @@ func TestIdentities(t *testing.T) { }) t.Run("single identity", func(t *testing.T) { - expected := identities.Sample(1)[0] + expected := identities[rand.Intn(len(identities))] actual, err := state.Final().Identity(expected.NodeID) require.Nil(t, err) assert.Equal(t, expected, actual) }) t.Run("filtered", func(t *testing.T) { + sample, err := identities.SamplePct(0.1) + require.NoError(t, err) filters := []flow.IdentityFilter{ filter.HasRole(flow.RoleCollection), - filter.HasNodeID(identities.SamplePct(0.1).NodeIDs()...), + filter.HasNodeID(sample.NodeIDs()...), filter.HasWeight(true), } @@ -1108,7 +1109,7 @@ func TestSnapshot_CrossEpochIdentities(t *testing.T) { // 1 identity added at epoch 2 that was not present in epoch 1 addedAtEpoch2 := unittest.IdentityFixture() // 1 identity removed in epoch 2 that was present in epoch 1 - removedAtEpoch2 := epoch1Identities.Sample(1)[0] + removedAtEpoch2 := epoch1Identities[rand.Intn(len(epoch1Identities))] // epoch 2 has partial overlap with epoch 1 epoch2Identities := append( epoch1Identities.Filter(filter.Not(filter.HasNodeID(removedAtEpoch2.NodeID))), diff --git a/state/protocol/badger/state_test.go b/state/protocol/badger/state_test.go index 7619f6a612d..a24a996cc8f 100644 --- a/state/protocol/badger/state_test.go +++ b/state/protocol/badger/state_test.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "testing" - "time" "github.com/dgraph-io/badger/v2" @@ -304,7 +303,9 @@ func TestBootstrap_InvalidIdentities(t *testing.T) { root := unittest.RootSnapshotFixture(participants) // randomly shuffle the identities so they are not canonically ordered encodable := root.Encodable() - encodable.Identities = participants.DeterministicShuffle(time.Now().UnixNano()) + var err error + encodable.Identities, err = participants.Shuffle() + require.NoError(t, err) root = inmem.SnapshotFromEncodable(encodable) bootstrap(t, root, func(state *bprotocol.State, err error) { assert.Error(t, err) diff --git a/state/protocol/badger/validity_test.go b/state/protocol/badger/validity_test.go index 2c0e3372e4b..06877b424c6 100644 --- a/state/protocol/badger/validity_test.go +++ b/state/protocol/badger/validity_test.go @@ -2,7 +2,6 @@ package badger import ( "testing" - "time" "github.com/stretchr/testify/require" @@ -29,9 +28,10 @@ func TestEpochSetupValidity(t *testing.T) { _, result, _ := unittest.BootstrapFixture(participants) setup := result.ServiceEvents[0].Event.(*flow.EpochSetup) // randomly shuffle the identities so they are not canonically ordered - setup.Participants = setup.Participants.DeterministicShuffle(time.Now().UnixNano()) - - err := verifyEpochSetup(setup, true) + var err error + setup.Participants, err = setup.Participants.Shuffle() + require.NoError(t, err) + err = verifyEpochSetup(setup, true) require.Error(t, err) }) diff --git a/state/protocol/seed/prg_test.go b/state/protocol/seed/prg_test.go index 5111fa50aa6..90e93e2ac23 100644 --- a/state/protocol/seed/prg_test.go +++ b/state/protocol/seed/prg_test.go @@ -1,26 +1,23 @@ package seed import ( - "math/rand" + "crypto/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func getRandomSource(t *testing.T) []byte { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) +func getSeed(t *testing.T) []byte { seed := make([]byte, RandomSourceLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) return seed } // check PRGs created from the same source give the same outputs func TestDeterministic(t *testing.T) { - seed := getRandomSource(t) + seed := getSeed(t) customizer := []byte("test") prg1, err := PRGFromRandomSource(seed, customizer) require.NoError(t, err) @@ -36,7 +33,7 @@ func TestDeterministic(t *testing.T) { } func TestCustomizer(t *testing.T) { - seed := getRandomSource(t) + seed := getSeed(t) customizer1 := []byte("test1") prg1, err := PRGFromRandomSource(seed, customizer1) require.NoError(t, err) diff --git a/storage/badger/cleaner.go b/storage/badger/cleaner.go index d5c4bd7af57..c8d9f36fe88 100644 --- a/storage/badger/cleaner.go +++ b/storage/badger/cleaner.go @@ -3,13 +3,13 @@ package badger import ( - "math/rand" "time" "github.com/dgraph-io/badger/v2" "github.com/rs/zerolog" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/utils/rand" ) type Cleaner struct { @@ -18,13 +18,13 @@ type Cleaner struct { metrics module.CleanerMetrics enabled bool ratio float64 - freq int - calls int + freq uint + calls uint } // NewCleaner returns a cleaner that runs the badger value log garbage collection once every `frequency` calls // if a frequency of zero is passed in, we will not run the GC at all -func NewCleaner(log zerolog.Logger, db *badger.DB, metrics module.CleanerMetrics, frequency int) *Cleaner { +func NewCleaner(log zerolog.Logger, db *badger.DB, metrics module.CleanerMetrics, frequency uint) *Cleaner { // NOTE: we run garbage collection frequently at points in our business // logic where we are likely to have a small breather in activity; it thus // makes sense to run garbage collection often, with a smaller ratio, rather @@ -40,24 +40,33 @@ func NewCleaner(log zerolog.Logger, db *badger.DB, metrics module.CleanerMetrics // we don't want the entire network to run GC at the same time, so // distribute evenly over time if c.enabled { - c.calls = rand.Intn(c.freq) + var err error + c.calls, err = rand.Uintn(c.freq) + if err != nil { + // if true randomess in the node is broken, set `calls` to zero. + c.calls = 0 + } } return c } -func (c *Cleaner) RunGC() { +func (c *Cleaner) RunGC() error { if !c.enabled { - return + return nil } // only actually run approximately every frequency number of calls c.calls++ if c.calls < c.freq { - return + return nil } // we add 20% jitter into the interval, so that we don't risk nodes syncing // up on their GC calls over time - c.calls = rand.Intn(c.freq / 5) + var err error + c.calls, err = rand.Uintn(c.freq / 5) + if err != nil { + return err + } // run the garbage collection in own goroutine and handle sentinel errors go func() { @@ -84,4 +93,5 @@ func (c *Cleaner) RunGC() { Msg("garbage collection on value log executed") c.metrics.RanGC(runtime) }() + return nil } diff --git a/storage/badger/dkg_state_test.go b/storage/badger/dkg_state_test.go index 7d763adb2ae..11b6e3ca9a1 100644 --- a/storage/badger/dkg_state_test.go +++ b/storage/badger/dkg_state_test.go @@ -4,7 +4,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -53,7 +52,6 @@ func TestDKGState_BeaconKeys(t *testing.T) { store, err := bstorage.NewDKGState(metrics, db) require.NoError(t, err) - rand.Seed(time.Now().UnixNano()) epochCounter := rand.Uint64() // attempt to get a non-existent key @@ -96,7 +94,6 @@ func TestDKGState_EndState(t *testing.T) { store, err := bstorage.NewDKGState(metrics, db) require.NoError(t, err) - rand.Seed(time.Now().UnixNano()) epochCounter := rand.Uint64() endState := flow.DKGEndStateNoKey diff --git a/storage/badger/operation/common_test.go b/storage/badger/operation/common_test.go index 592627b490f..f91f50dc472 100644 --- a/storage/badger/operation/common_test.go +++ b/storage/badger/operation/common_test.go @@ -5,10 +5,8 @@ package operation import ( "bytes" "fmt" - "math/rand" "reflect" "testing" - "time" "github.com/dgraph-io/badger/v2" "github.com/stretchr/testify/assert" @@ -21,7 +19,7 @@ import ( ) func init() { - rand.Seed(time.Now().UnixNano()) + } type Entity struct { diff --git a/storage/cleaner.go b/storage/cleaner.go index 80db6dca072..4a1071e0140 100644 --- a/storage/cleaner.go +++ b/storage/cleaner.go @@ -3,5 +3,5 @@ package storage type Cleaner interface { - RunGC() + RunGC() error } diff --git a/storage/merkle/proof_test.go b/storage/merkle/proof_test.go index 44e93a90bef..826b61b6ed8 100644 --- a/storage/merkle/proof_test.go +++ b/storage/merkle/proof_test.go @@ -3,7 +3,6 @@ package merkle import ( "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -142,7 +141,7 @@ func TestValidateFormat(t *testing.T) { // when trie includes many random keys. (only a random subset of keys are checked for proofs) func TestProofsWithRandomKeys(t *testing.T) { // initialize random generator, two trees and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 numberOfInsertions := 10000 numberOfProofsToVerify := 100 diff --git a/storage/merkle/tree_test.go b/storage/merkle/tree_test.go index b20ee26d7e5..f3f5f54daea 100644 --- a/storage/merkle/tree_test.go +++ b/storage/merkle/tree_test.go @@ -6,8 +6,8 @@ import ( "encoding/hex" "fmt" "math/rand" + crand "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,7 +64,7 @@ func TestEmptyTreeHash(t *testing.T) { // generate random key-value pair key := make([]byte, keyLength) - rand.Read(key) + crand.Read(key) val := []byte{1} // add key-value pair: hash should be non-empty @@ -239,7 +239,7 @@ func Test_KeyLengthChecked(t *testing.T) { // of a _single_ key-value pair to an otherwise empty tree. func TestTreeSingle(t *testing.T) { // initialize the random generator, tree and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree, err := NewTree(keyLength) assert.NoError(t, err) @@ -275,7 +275,7 @@ func TestTreeSingle(t *testing.T) { // Key-value pairs are added and deleted in the same order. func TestTreeBatch(t *testing.T) { // initialize random generator, tree, zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree, err := NewTree(keyLength) assert.NoError(t, err) @@ -321,7 +321,7 @@ func TestTreeBatch(t *testing.T) { // in which the elements were added. func TestRandomOrder(t *testing.T) { // initialize random generator, two trees and zero hash - rand.Seed(time.Now().UnixNano()) + keyLength := 32 tree1, err := NewTree(keyLength) assert.NoError(t, err) @@ -382,8 +382,8 @@ func BenchmarkTree(b *testing.B) { func randomKeyValuePair(keySize, valueSize int) ([]byte, []byte) { key := make([]byte, keySize) val := make([]byte, valueSize) - _, _ = rand.Read(key) - _, _ = rand.Read(val) + _, _ = crand.Read(key) + _, _ = crand.Read(val) return key, val } diff --git a/storage/mock/cleaner.go b/storage/mock/cleaner.go index abaecdc9186..c2d2fb1dcaa 100644 --- a/storage/mock/cleaner.go +++ b/storage/mock/cleaner.go @@ -10,8 +10,17 @@ type Cleaner struct { } // RunGC provides a mock function with given fields: -func (_m *Cleaner) RunGC() { - _m.Called() +func (_m *Cleaner) RunGC() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 } type mockConstructorTestingTNewCleaner interface { diff --git a/utils/math/math.go b/utils/math/math.go deleted file mode 100644 index 33c9064fa6e..00000000000 --- a/utils/math/math.go +++ /dev/null @@ -1,16 +0,0 @@ -package math - -// MinUint returns the minimum of a list of uints. -func MinUint(uints ...uint) uint { - if len(uints) == 0 { - return 0 - } - - min := uints[0] - for _, u := range uints { - if u < min { - min = u - } - } - return min -} diff --git a/utils/rand/rand.go b/utils/rand/rand.go new file mode 100644 index 00000000000..2912c2eec58 --- /dev/null +++ b/utils/rand/rand.go @@ -0,0 +1,169 @@ +package rand + +import ( + "crypto/rand" + "encoding/binary" + "errors" + "fmt" +) + +// This package is a wrppaer around true RNG crypto/rand. +// It implements useful tools using the true RNG and that +// are not exported by the crypto/rand package. +// This package does not implement any determinstic RNG (Pseudo RNG) +// unlike the package flow-go/crypto/random. + +var randFailure = errors.New("crypto/rand failed") + +// returns a random uint64 +func Uint64() (uint64, error) { + buffer := make([]byte, 8) // TODO: declare as a global variable and add a lock? + if _, err := rand.Read(buffer); err != nil { + return 0, randFailure + } + r := binary.LittleEndian.Uint64(buffer) + return r, nil +} + +// returns a random uint64 strictly less than n +// errors if n==0 +func Uint64n(n uint64) (uint64, error) { + if n == 0 { + return 0, fmt.Errorf("n should be strictly positive, got %d", n) + } + // the max returned random is n-1 > 0 + max := n - 1 + // count the bytes size of max + size := 0 + for tmp := max; tmp != 0; tmp >>= 8 { + size++ + } + buffer := make([]byte, 8) // TODO: declare as a global variable and add a lock? + // get the bit size of max + mask := uint64(0) + for max&mask != max { + mask = (mask << 1) | 1 + } + + // Using 64 bits of random and reducing modulo n does not guarantee a high uniformity + // of the result. + // For a better uniformity, loop till a sample is less or equal to `max`. + // This means the function might take longer time to output a random. + // Using the size of `max` in bits helps the loop end earlier (the algo stops after one loop + // with more than 50%) + // a different approach would be to pull at least 128 bits from the random source + // and use big number modular reduction by `n`. + random := n + for random > max { + if _, err := rand.Read(buffer[:size]); err != nil { + return 0, randFailure + } + random = binary.LittleEndian.Uint64(buffer) + random &= mask // adjust to the size of max in bits + } + + return random, nil +} + +// returns a random uint32 +func Uint32() (uint32, error) { + // for 64-bits machines, doing 64 bits operations and then casting + // should be faster than dealing with 32 bits operations + r, err := Uint64() + return uint32(r), err +} + +// returns a random uint32 strictly less than n +// errors if n==0 +func Uint32n(n uint32) (uint32, error) { + if n == 0 { + return 0, fmt.Errorf("n should be strictly positive, got %d", n) + } + // the max returned random is n-1 > 0 + max := n - 1 + // count the bytes size of max + size := 0 + for tmp := max; tmp != 0; tmp >>= 8 { + size++ + } + buffer := make([]byte, 4) // TODO: declare as a global variable and add a lock? + // get the bit size of max + mask := uint32(0) + for max&mask != max { + mask = (mask << 1) | 1 + } + + // Using 32 bits of random and reducing modulo n does not guarantee a high uniformity + // of the result. + // For a better uniformity, loop till a sample is less or equal to `max`. + // This means the function might take longer time to output a random. + // Using the size of `max` in bits helps the loop end earlier (the algo stops after one loop + // with more than 50%) + // a different approach would be to pull at least 128 bits from the random source + // and use big number modular reduction by `n`. + random := n + for random > max { + if _, err := rand.Read(buffer[:size]); err != nil { + return 0, randFailure + } + random = binary.LittleEndian.Uint32(buffer) + random &= mask // adjust to the size of max in bits + } + + return random, nil +} + +// returns a random uint +func Uint() (uint, error) { + r, err := Uint64() + return uint(r), err +} + +// returns a random uint strictly less than n +// errors if n==0 +func Uintn(n uint) (uint, error) { + r, err := Uint64n(uint64(n)) + return uint(r), err +} + +// Shuffle permutes a data structure in place +// based on the provided `swap` function. +// It is not deterministic. +// +// It implements Fisher-Yates Shuffle using crypto/rand as a source of randoms. +// +// O(1) space and O(n) time +func Shuffle(n uint, swap func(i, j uint)) error { + for i := n - 1; i > 0; i-- { + j, err := Uintn(i + 1) + if err != nil { + return err + } + swap(i, j) + } + return nil +} + +// Samples picks randomly m elements out of n elemnts in a data structure +// and places them in random order at indices [0,m-1], +// the swapping being implemented in place. The data structure is defined +// by the `swap` function. +// Sampling is not deterministic. +// +// It implements the first (m) elements of Fisher-Yates Shuffle using +// crypto/rand as a source of randoms. +// +// O(1) space and O(m) time +func Samples(n uint, m uint, swap func(i, j uint)) error { + if n < m { + return fmt.Errorf("sample size (%d) cannot be larger than entire population (%d)", m, n) + } + for i := uint(0); i < m; i++ { + j, err := Uintn(n - i) + if err != nil { + return err + } + swap(i, i+j) + } + return nil +} diff --git a/utils/unittest/chain_suite.go b/utils/unittest/chain_suite.go index bd7b97fe52b..a2ebc59f8d0 100644 --- a/utils/unittest/chain_suite.go +++ b/utils/unittest/chain_suite.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/chunks" @@ -504,7 +505,8 @@ func (bc *BaseChainSuite) ValidSubgraphFixture() subgraphFixture { assignedVerifiersPerChunk := uint(len(bc.Approvers) / 2) approvals := make(map[uint64]map[flow.Identifier]*flow.ResultApproval) for _, chunk := range incorporatedResult.Result.Chunks { - assignedVerifiers := bc.Approvers.Sample(assignedVerifiersPerChunk) + assignedVerifiers, err := bc.Approvers.Sample(assignedVerifiersPerChunk) + require.NoError(bc.T(), err) assignment.Add(chunk, assignedVerifiers.NodeIDs()) // generate approvals @@ -543,7 +545,8 @@ func (bc *BaseChainSuite) Extend(block *flow.Block) { assignedVerifiersPerChunk := uint(len(bc.Approvers) / 2) approvals := make(map[uint64]map[flow.Identifier]*flow.ResultApproval) for _, chunk := range incorporatedResult.Result.Chunks { - assignedVerifiers := bc.Approvers.Sample(assignedVerifiersPerChunk) + assignedVerifiers, err := bc.Approvers.Sample(assignedVerifiersPerChunk) + require.NoError(bc.T(), err) assignment.Add(chunk, assignedVerifiers.NodeIDs()) // generate approvals diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 8b558403bc4..8117b49383c 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -50,6 +50,15 @@ const ( DefaultAddress = "localhost:0" ) +// returns a deterministic math/rand PRG that can be used for deterministic randomness in tests only. +// The PRG seed is logged in case the test iteration needs to be reproduced. +func GetPRG(t *testing.T) *rand.Rand { + random := time.Now().UnixNano() + t.Logf("rng seed is %d", random) + rng := rand.New(rand.NewSource(random)) + return rng +} + func IPPort(port string) string { return net.JoinHostPort("localhost", port) } diff --git a/utils/unittest/network/fixtures.go b/utils/unittest/network/fixtures.go index 0d0e3e30379..d0cefd7622c 100644 --- a/utils/unittest/network/fixtures.go +++ b/utils/unittest/network/fixtures.go @@ -1,6 +1,7 @@ package network import ( + crand "crypto/rand" "fmt" "math/rand" "net" @@ -20,7 +21,7 @@ type TxtLookupTestCase struct { func NetIPAddrFixture() net.IPAddr { token := make([]byte, 4) - rand.Read(token) + crand.Read(token) ip := net.IPAddr{ IP: net.IPv4(token[0], token[1], token[2], token[3]), @@ -32,7 +33,7 @@ func NetIPAddrFixture() net.IPAddr { func TxtIPFixture() string { token := make([]byte, 4) - rand.Read(token) + crand.Read(token) return "dnsaddr=" + net.IPv4(token[0], token[1], token[2], token[3]).String() } From 93b68dc015b72ac8e1ea85e134c75fcde7465e3d Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 18:08:39 -0600 Subject: [PATCH 004/169] update crypto math/rand usage --- crypto/bls12381_utils_test.go | 10 +-- crypto/bls_test.go | 105 ++++++++++++++----------------- crypto/bls_thresholdsign_test.go | 27 ++++---- crypto/dkg_test.go | 14 ++--- crypto/ecdsa_test.go | 18 +++--- crypto/hash/hash_test.go | 14 +---- crypto/random/rand_test.go | 54 ++++++++-------- crypto/sign_test_utils.go | 31 ++++----- crypto/spock_test.go | 12 ++-- 9 files changed, 133 insertions(+), 152 deletions(-) diff --git a/crypto/bls12381_utils_test.go b/crypto/bls12381_utils_test.go index 8911ada1769..8f206a89058 100644 --- a/crypto/bls12381_utils_test.go +++ b/crypto/bls12381_utils_test.go @@ -4,7 +4,7 @@ package crypto import ( - "crypto/rand" + crand "crypto/rand" "encoding/hex" "testing" @@ -15,7 +15,7 @@ import ( func TestDeterministicKeyGen(t *testing.T) { // 2 keys generated with the same seed should be equal seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) sk1, err := GeneratePrivateKey(BLSBLS12381, seed) @@ -30,7 +30,7 @@ func TestPRGseeding(t *testing.T) { blsInstance.reInit() // 2 scalars generated with the same seed should be equal seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) // 1st scalar (wrapped in a private key) @@ -51,7 +51,7 @@ func TestPRGseeding(t *testing.T) { func BenchmarkScalarMultG1G2(b *testing.B) { blsInstance.reInit() seed := make([]byte, securityBits/8) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) _ = seedRelic(seed) var expo scalar randZr(&expo) @@ -139,7 +139,7 @@ func TestSubgroupCheck(t *testing.T) { blsInstance.reInit() // seed Relic PRG seed := make([]byte, securityBits/8) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) _ = seedRelic(seed) t.Run("G1", func(t *testing.T) { diff --git a/crypto/bls_test.go b/crypto/bls_test.go index d0dc73c066c..d68039a0c37 100644 --- a/crypto/bls_test.go +++ b/crypto/bls_test.go @@ -4,12 +4,11 @@ package crypto import ( - "crypto/rand" + crand "crypto/rand" "encoding/hex" "fmt" mrand "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -67,7 +66,8 @@ func BenchmarkBLSBLS12381Verify(b *testing.B) { } // utility function to generate a random BLS private key -func randomSK(t *testing.T, seed []byte) PrivateKey { +func randomSK(t *testing.T, rand *mrand.Rand) PrivateKey { + seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) n, err := rand.Read(seed) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) @@ -79,7 +79,7 @@ func randomSK(t *testing.T, seed []byte) PrivateKey { // utility function to generate a non BLS private key func invalidSK(t *testing.T) PrivateKey { seed := make([]byte, KeyGenSeedMinLenECDSAP256) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, KeyGenSeedMinLenECDSAP256) require.NoError(t, err) sk, err := GeneratePrivateKey(ECDSAP256, seed) @@ -89,29 +89,30 @@ func invalidSK(t *testing.T) PrivateKey { // BLS tests func TestBLSBLS12381Hasher(t *testing.T) { + rand := getPRG(t) // generate a key pair - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) - sk := randomSK(t, seed) + sk := randomSK(t, rand) sig := make([]byte, SignatureLenBLSBLS12381) + msg := []byte("message") // empty hasher t.Run("Empty hasher", func(t *testing.T) { - _, err := sk.Sign(seed, nil) + _, err := sk.Sign(msg, nil) assert.Error(t, err) assert.True(t, IsNilHasherError(err)) - _, err = sk.PublicKey().Verify(sig, seed, nil) + _, err = sk.PublicKey().Verify(sig, msg, nil) assert.Error(t, err) assert.True(t, IsNilHasherError(err)) }) // short size hasher t.Run("short size hasher", func(t *testing.T) { - s, err := sk.Sign(seed, hash.NewSHA2_256()) + s, err := sk.Sign(msg, hash.NewSHA2_256()) assert.Error(t, err) assert.True(t, IsInvalidHasherSizeError(err)) assert.Nil(t, s) - valid, err := sk.PublicKey().Verify(sig, seed, hash.NewSHA2_256()) + valid, err := sk.PublicKey().Verify(sig, msg, hash.NewSHA2_256()) assert.Error(t, err) assert.True(t, IsInvalidHasherSizeError(err)) assert.False(t, valid) @@ -206,9 +207,9 @@ func TestBLSEquals(t *testing.T) { // TestBLSUtils tests some utility functions func TestBLSUtils(t *testing.T) { + rand := getPRG(t) // generate a key pair - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) - sk := randomSK(t, seed) + sk := randomSK(t, rand) // test Algorithm() testKeysAlgorithm(t, sk, BLSBLS12381) // test Size() @@ -217,9 +218,7 @@ func TestBLSUtils(t *testing.T) { // BLS Proof of Possession test func TestBLSPOP(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // make sure the length is larger than minimum lengths of all the signaure algos seedMinLength := 48 seed := make([]byte, seedMinLength) @@ -228,12 +227,12 @@ func TestBLSPOP(t *testing.T) { t.Run("PoP tests", func(t *testing.T) { loops := 10 for j := 0; j < loops; j++ { - n, err := mrand.Read(seed) + n, err := rand.Read(seed) require.Equal(t, n, seedMinLength) require.NoError(t, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) - _, err = mrand.Read(input) + _, err = rand.Read(input) require.NoError(t, err) s, err := BLSGeneratePOP(sk) require.NoError(t, err) @@ -276,6 +275,7 @@ func TestBLSPOP(t *testing.T) { // Verify the aggregated signature using the multi-signature verification with // one message. func TestBLSAggregateSignatures(t *testing.T) { + rand := getPRG(t) // random message input := make([]byte, 100) _, err := rand.Read(input) @@ -283,19 +283,16 @@ func TestBLSAggregateSignatures(t *testing.T) { // hasher kmac := NewExpandMsgXOFKMAC128("test tag") // number of signatures to aggregate - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) sigsNum := mrand.Intn(100) + 1 sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + var aggSig, expectedSig Signature // create the signatures for i := 0; i < sigsNum; i++ { - sk := randomSK(t, seed) + sk := randomSK(t, rand) s, err := sk.Sign(input, kmac) require.NoError(t, err) sigs = append(sigs, s) @@ -348,7 +345,7 @@ func TestBLSAggregateSignatures(t *testing.T) { // check if one the public keys is not correct t.Run("one invalid public key", func(t *testing.T) { randomIndex := mrand.Intn(sigsNum) - newSk := randomSK(t, seed) + newSk := randomSK(t, rand) sks[randomIndex] = newSk pks[randomIndex] = newSk.PublicKey() aggSk, err := AggregateBLSPrivateKeys(sks) @@ -414,18 +411,15 @@ func TestBLSAggregateSignatures(t *testing.T) { // the public key of the aggregated private key is equal to the aggregated // public key func TestBLSAggregatePubKeys(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // number of keys to aggregate pkNum := mrand.Intn(100) + 1 pks := make([]PublicKey, 0, pkNum) sks := make([]PrivateKey, 0, pkNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) // create the signatures for i := 0; i < pkNum; i++ { - sk := randomSK(t, seed) + sk := randomSK(t, rand) sks = append(sks, sk) pks = append(pks, sk.PublicKey()) } @@ -509,17 +503,14 @@ func TestBLSAggregatePubKeys(t *testing.T) { // BLS multi-signature // public keys removal sanity check func TestBLSRemovePubKeys(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // number of keys to aggregate pkNum := mrand.Intn(100) + 1 pks := make([]PublicKey, 0, pkNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) // generate public keys for i := 0; i < pkNum; i++ { - sk := randomSK(t, seed) + sk := randomSK(t, rand) pks = append(pks, sk.PublicKey()) } // aggregate public keys @@ -546,7 +537,7 @@ func TestBLSRemovePubKeys(t *testing.T) { // remove an extra key and check inequality t.Run("inequality check", func(t *testing.T) { - extraPk := randomSK(t, seed).PublicKey() + extraPk := randomSK(t, rand).PublicKey() partialPk, err := RemoveBLSPublicKeys(aggPk, []PublicKey{extraPk}) assert.NoError(t, err) @@ -562,7 +553,7 @@ func TestBLSRemovePubKeys(t *testing.T) { identityPk, err := RemoveBLSPublicKeys(aggPk, pks) require.NoError(t, err) // identity public key is expected - randomPk := randomSK(t, seed).PublicKey() + randomPk := randomSK(t, rand).PublicKey() randomPkPlusIdentityPk, err := AggregateBLSPublicKeys([]PublicKey{randomPk, identityPk}) require.NoError(t, err) @@ -608,9 +599,7 @@ func TestBLSRemovePubKeys(t *testing.T) { // batch verification technique and compares the result to verifying each signature // separately. func TestBLSBatchVerify(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // random message input := make([]byte, 100) _, err := mrand.Read(input) @@ -622,12 +611,12 @@ func TestBLSBatchVerify(t *testing.T) { sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + expectedValid := make([]bool, 0, sigsNum) // create the signatures for i := 0; i < sigsNum; i++ { - sk := randomSK(t, seed) + sk := randomSK(t, rand) s, err := sk.Sign(input, kmac) require.NoError(t, err) sigs = append(sigs, s) @@ -757,7 +746,7 @@ func alterSignature(s Signature) { func BenchmarkBatchVerify(b *testing.B) { // random message input := make([]byte, 100) - _, _ = mrand.Read(input) + _, _ = crand.Read(input) // hasher kmac := NewExpandMsgXOFKMAC128("bench tag") sigsNum := 100 @@ -767,7 +756,8 @@ func BenchmarkBatchVerify(b *testing.B) { // create the signatures for i := 0; i < sigsNum; i++ { - _, _ = mrand.Read(seed) + _, err := crand.Read(seed) + require.NoError(b, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(b, err) s, err := sk.Sign(input, kmac) @@ -813,9 +803,7 @@ func BenchmarkBatchVerify(b *testing.B) { // and verify the aggregated signature using the multi-signature verification with // many message. func TestBLSAggregateSignaturesManyMessages(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // number of signatures to aggregate sigsNum := mrand.Intn(20) + 1 @@ -824,10 +812,10 @@ func TestBLSAggregateSignaturesManyMessages(t *testing.T) { // number of keys keysNum := mrand.Intn(sigsNum) + 1 sks := make([]PrivateKey, 0, keysNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + // generate the keys for i := 0; i < keysNum; i++ { - sk := randomSK(t, seed) + sk := randomSK(t, rand) sks = append(sks, sk) } @@ -972,14 +960,19 @@ func BenchmarkVerifySignatureManyMessages(b *testing.B) { inputKmacs := make([]hash.Hasher, 0, sigsNum) sigs := make([]Signature, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) + inputMsgs := make([][]byte, 0, sigsNum) kmac := NewExpandMsgXOFKMAC128("bench tag") + seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) // create the signatures for i := 0; i < sigsNum; i++ { input := make([]byte, 100) - _, _ = mrand.Read(seed) + _, err := crand.Read(input) + require.NoError(b, err) + + _, err = crand.Read(seed) + require.NoError(b, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(b, err) s, err := sk.Sign(input, kmac) @@ -1001,20 +994,21 @@ func BenchmarkVerifySignatureManyMessages(b *testing.B) { // Bench of all aggregation functions func BenchmarkAggregate(b *testing.B) { + seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) // random message input := make([]byte, 100) - _, _ = mrand.Read(input) + _, _ = crand.Read(input) // hasher kmac := NewExpandMsgXOFKMAC128("bench tag") sigsNum := 1000 sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) // create the signatures for i := 0; i < sigsNum; i++ { - _, _ = mrand.Read(seed) + _, err := crand.Read(seed) + require.NoError(b, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(b, err) s, err := sk.Sign(input, kmac) @@ -1058,9 +1052,7 @@ func BenchmarkAggregate(b *testing.B) { } func TestBLSIdentity(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) var identitySig []byte msg := []byte("random_message") @@ -1073,8 +1065,7 @@ func TestBLSIdentity(t *testing.T) { assert.True(t, IsBLSSignatureIdentity(identityBLSSignature)) // sum up a random signature and its inverse to get identity - seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) - sk := randomSK(t, seed) + sk := randomSK(t, rand) sig, err := sk.Sign(msg, hasher) require.NoError(t, err) oppositeSig := make([]byte, signatureLengthBLSBLS12381) diff --git a/crypto/bls_thresholdsign_test.go b/crypto/bls_thresholdsign_test.go index cc9be81eeaf..b563920cc0d 100644 --- a/crypto/bls_thresholdsign_test.go +++ b/crypto/bls_thresholdsign_test.go @@ -4,9 +4,8 @@ package crypto import ( - "crypto/rand" + crand "crypto/rand" "fmt" - mrand "math/rand" "sync" "testing" "time" @@ -34,9 +33,9 @@ func testCentralizedStatefulAPI(t *testing.T) { n := 10 for threshold := MinimumThreshold; threshold < n; threshold++ { // generate threshold keys - mrand.Seed(time.Now().UnixNano()) + rand := getPRG(t) seed := make([]byte, SeedMinLenDKG) - _, err := mrand.Read(seed) + _, err := rand.Read(seed) require.NoError(t, err) skShares, pkShares, pkGroup, err := BLSThresholdKeyGen(n, threshold, seed) require.NoError(t, err) @@ -48,7 +47,7 @@ func testCentralizedStatefulAPI(t *testing.T) { for i := 0; i < n; i++ { signers = append(signers, i) } - mrand.Shuffle(n, func(i, j int) { + rand.Shuffle(n, func(i, j int) { signers[i], signers[j] = signers[j], signers[i] }) @@ -138,7 +137,7 @@ func testCentralizedStatefulAPI(t *testing.T) { require.NoError(t, err) // Create a share and add it - i := mrand.Intn(n) + i := rand.Intn(n) share, err := skShares[i].Sign(thresholdSignatureMessage, kmac) require.NoError(t, err) enough, err := ts.TrustedAdd(i, share) @@ -261,7 +260,7 @@ func testCentralizedStatefulAPI(t *testing.T) { t.Run("constructor errors", func(t *testing.T) { // invalid keys size - index := mrand.Intn(n) + index := rand.Intn(n) pkSharesInvalid := make([]PublicKey, ThresholdSignMaxSize+1) tsFollower, err := NewBLSThresholdSignatureInspector(pkGroup, pkSharesInvalid, threshold, thresholdSignatureMessage, thresholdSignatureTag) assert.Error(t, err) @@ -318,9 +317,10 @@ func testDistributedStatefulAPI_FeldmanVSS(t *testing.T) { log.SetLevel(log.ErrorLevel) log.Info("DKG starts") gt = t + rand := getPRG(t) // number of participants to test n := 5 - lead := mrand.Intn(n) // random + lead := rand.Intn(n) // random var sync sync.WaitGroup chans := make([]chan *message, n) processors := make([]testDKGProcessor, 0, n) @@ -377,6 +377,7 @@ func testDistributedStatefulAPI_JointFeldman(t *testing.T) { log.SetLevel(log.ErrorLevel) log.Info("DKG starts") gt = t + rand := getPRG(t) // number of participants to test n := 5 for threshold := MinimumThreshold; threshold < n; threshold++ { @@ -543,12 +544,12 @@ type statelessKeys struct { // Centralized test of threshold signature protocol using the threshold key generation. func testCentralizedStatelessAPI(t *testing.T) { + rand := getPRG(t) n := 10 for threshold := MinimumThreshold; threshold < n; threshold++ { // generate threshold keys - mrand.Seed(time.Now().UnixNano()) seed := make([]byte, SeedMinLenDKG) - _, err := mrand.Read(seed) + _, err := rand.Read(seed) require.NoError(t, err) skShares, pkShares, pkGroup, err := BLSThresholdKeyGen(n, threshold, seed) require.NoError(t, err) @@ -561,7 +562,7 @@ func testCentralizedStatelessAPI(t *testing.T) { for i := 0; i < n; i++ { signers = append(signers, i) } - mrand.Shuffle(n, func(i, j int) { + rand.Shuffle(n, func(i, j int) { signers[i], signers[j] = signers[j], signers[i] }) // create (t+1) signatures of the first randomly chosen signers @@ -585,7 +586,7 @@ func testCentralizedStatelessAPI(t *testing.T) { // check failure with a random redundant signer if threshold > 1 { - randomDuplicate := mrand.Intn(int(threshold)) + 1 // 1 <= duplicate <= threshold + randomDuplicate := rand.Intn(int(threshold)) + 1 // 1 <= duplicate <= threshold tmp := signers[randomDuplicate] signers[randomDuplicate] = signers[0] thresholdSignature, err = BLSReconstructThresholdSignature(n, threshold, signShares, signers[:threshold+1]) @@ -608,7 +609,7 @@ func testCentralizedStatelessAPI(t *testing.T) { func BenchmarkSimpleKeyGen(b *testing.B) { n := 60 seed := make([]byte, SeedMinLenDKG) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) b.ResetTimer() for i := 0; i < b.N; i++ { _, _, _, _ = BLSThresholdKeyGen(n, optimalThreshold(n), seed) diff --git a/crypto/dkg_test.go b/crypto/dkg_test.go index d996ae0835c..3cc1d172cca 100644 --- a/crypto/dkg_test.go +++ b/crypto/dkg_test.go @@ -4,6 +4,7 @@ package crypto import ( + crand "crypto/rand" "fmt" mrand "math/rand" "sync" @@ -193,9 +194,7 @@ func dkgCommonTest(t *testing.T, dkg int, n int, threshold int, test testCase) { } // Update processors depending on the test - rand := time.Now().UnixNano() - mrand.Seed(rand) - t.Logf("math rand seed is %d", rand) + // // r1 and r2 is the number of malicious participants, each group with a slight diffrent behavior. // - r1 participants of indices 0 to r1-1 behave maliciously and will get disqualified by honest participants. // - r2 participants of indices r1 to r1+r2-1 will behave maliciously at first but will recover and won't be @@ -294,9 +293,6 @@ func dkgCommonTest(t *testing.T, dkg int, n int, threshold int, test testCase) { // start DKG in all participants // start listening on the channels seed := make([]byte, SeedMinLenDKG) - read, err := mrand.Read(seed) - require.Equal(t, read, SeedMinLenDKG) - require.NoError(t, err) sync.Add(n) log.Info("DKG protocol starts") @@ -308,11 +304,13 @@ func dkgCommonTest(t *testing.T, dkg int, n int, threshold int, test testCase) { for current := 0; current < n; current++ { // start dkg in parallel - // ( one common PRG is used for all instances which causes a race + // ( one common PRG is used internally for all instances which causes a race // in generating randoms and leads to non-deterministic keys. If deterministic keys // are required, switch to sequential calls to dkg.Start() ) go func(current int) { - err := processors[current].dkg.Start(seed) + _, err := crand.Read(seed) + require.NoError(t, err) + err = processors[current].dkg.Start(seed) require.Nil(t, err) processors[current].startSync.Done() // avoids reading messages when a dkg instance hasn't started yet }(current) diff --git a/crypto/ecdsa_test.go b/crypto/ecdsa_test.go index 3c86a403847..54d1ec7a0d6 100644 --- a/crypto/ecdsa_test.go +++ b/crypto/ecdsa_test.go @@ -8,7 +8,7 @@ import ( "testing" "crypto/elliptic" - "crypto/rand" + crand "crypto/rand" "math/big" "github.com/btcsuite/btcd/btcec/v2" @@ -73,7 +73,7 @@ func TestECDSAHasher(t *testing.T) { // generate a key pair seed := make([]byte, ecdsaMinSeed[curve]) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, ecdsaMinSeed[curve]) require.NoError(t, err) sk, err := GeneratePrivateKey(curve, seed) @@ -156,7 +156,7 @@ func TestECDSAUtils(t *testing.T) { for _, curve := range ecdsaCurves { // generate a key pair seed := make([]byte, ecdsaMinSeed[curve]) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, ecdsaMinSeed[curve]) require.NoError(t, err) sk, err := GeneratePrivateKey(curve, seed) @@ -256,7 +256,7 @@ func TestSignatureFormatCheck(t *testing.T) { t.Run("valid signature", func(t *testing.T) { len := ecdsaSigLen[curve] sig := Signature(make([]byte, len)) - rand.Read(sig) + crand.Read(sig) sig[len/2] = 0 // force s to be less than the curve order sig[len-1] |= 1 // force s to be non zero sig[0] = 0 // force r to be less than the curve order @@ -283,7 +283,7 @@ func TestSignatureFormatCheck(t *testing.T) { // signature with a zero s len := ecdsaSigLen[curve] sig0s := Signature(make([]byte, len)) - rand.Read(sig0s[:len/2]) + crand.Read(sig0s[:len/2]) valid, err := SignatureFormatCheck(curve, sig0s) assert.Nil(t, err) @@ -291,7 +291,7 @@ func TestSignatureFormatCheck(t *testing.T) { // signature with a zero r sig0r := Signature(make([]byte, len)) - rand.Read(sig0r[len/2:]) + crand.Read(sig0r[len/2:]) valid, err = SignatureFormatCheck(curve, sig0r) assert.Nil(t, err) @@ -301,7 +301,7 @@ func TestSignatureFormatCheck(t *testing.T) { t.Run("large values", func(t *testing.T) { len := ecdsaSigLen[curve] sigLargeS := Signature(make([]byte, len)) - rand.Read(sigLargeS[:len/2]) + crand.Read(sigLargeS[:len/2]) // make sure s is larger than the curve order for i := len / 2; i < len; i++ { sigLargeS[i] = 0xFF @@ -312,7 +312,7 @@ func TestSignatureFormatCheck(t *testing.T) { assert.False(t, valid) sigLargeR := Signature(make([]byte, len)) - rand.Read(sigLargeR[len/2:]) + crand.Read(sigLargeR[len/2:]) // make sure s is larger than the curve order for i := 0; i < len/2; i++ { sigLargeR[i] = 0xFF @@ -357,7 +357,7 @@ func TestEllipticUnmarshalSecp256k1(t *testing.T) { func BenchmarkECDSADecode(b *testing.B) { // random message seed := make([]byte, 50) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) for _, curve := range []SigningAlgorithm{ECDSASecp256k1, ECDSAP256} { sk, _ := GeneratePrivateKey(curve, seed) diff --git a/crypto/hash/hash_test.go b/crypto/hash/hash_test.go index f2eef310e94..8ec62e950ea 100644 --- a/crypto/hash/hash_test.go +++ b/crypto/hash/hash_test.go @@ -1,10 +1,9 @@ package hash import ( + "crypto/rand" "encoding/hex" - "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -120,9 +119,6 @@ func TestHashersAPI(t *testing.T) { NewKeccak_256, } - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) data := make([]byte, 1801) rand.Read(data) @@ -164,10 +160,6 @@ func TestHashersAPI(t *testing.T) { // It compares the hashes of random data of different lengths to // the output of standard Go sha3. func TestSHA3(t *testing.T) { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) - t.Run("SHA3_256", func(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) @@ -203,10 +195,6 @@ func TestSHA3(t *testing.T) { // It compares the hashes of random data of different lengths to // the output of Go LegacyKeccak. func TestKeccak(t *testing.T) { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) - for i := 0; i < 5000; i++ { value := make([]byte, i) rand.Read(value) diff --git a/crypto/random/rand_test.go b/crypto/random/rand_test.go index e0e022a8119..c3a4fff9ceb 100644 --- a/crypto/random/rand_test.go +++ b/crypto/random/rand_test.go @@ -2,8 +2,9 @@ package random import ( "bytes" + crand "crypto/rand" "fmt" - "math/rand" + mrand "math/rand" "testing" "time" @@ -82,10 +83,11 @@ func TestChacha20Compliance(t *testing.T) { }) } -func seedMathRand(t *testing.T) { - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) +func getPRG(t *testing.T) *mrand.Rand { + random := time.Now().UnixNano() + t.Logf("rng seed is %d", random) + rng := mrand.New(mrand.NewSource(random)) + return rng } // The tests are targeting the PRG implementations in the package. @@ -95,12 +97,12 @@ func seedMathRand(t *testing.T) { // Simple unit testing of Uint using a very basic randomness test. // It doesn't evaluate randomness of the output and doesn't perform advanced statistical tests. func TestUint(t *testing.T) { - seedMathRand(t) + rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - rand.Read(customizer) + crand.Read(customizer) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -108,7 +110,7 @@ func TestUint(t *testing.T) { t.Run("basic randomness", func(t *testing.T) { sampleSize := 80000 tolerance := 0.05 - sampleSpace := uint64(10 + rand.Intn(100)) + sampleSpace := uint64(10 + crand.Intn(100)) distribution := make([]float64, sampleSpace) for i := 0; i < sampleSize; i++ { @@ -133,12 +135,12 @@ func TestUint(t *testing.T) { // // SubPermutation tests cover Permutation as well. func TestSubPermutation(t *testing.T) { - seedMathRand(t) + rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - rand.Read(customizer) + crand.Read(customizer) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -155,7 +157,7 @@ func TestSubPermutation(t *testing.T) { samplingDistribution := make([]float64, listSize) // tests the subset ordering randomness (using a particular element testElement) orderingDistribution := make([]float64, subsetSize) - testElement := rand.Intn(listSize) + testElement := crand.Intn(listSize) for i := 0; i < sampleSize; i++ { shuffledlist, err := rng.SubPermutation(listSize, subsetSize) @@ -216,12 +218,12 @@ func TestSubPermutation(t *testing.T) { // Simple unit testing of Shuffle using a very basic randomness test. // It doesn't evaluate randomness of the output and doesn't perform advanced statistical tests. func TestShuffle(t *testing.T) { - seedMathRand(t) + rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - rand.Read(customizer) + crand.Read(customizer) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -233,7 +235,7 @@ func TestShuffle(t *testing.T) { tolerance := 0.05 // the distribution of a particular element of the list, testElement distribution := make([]float64, listSize) - testElement := rand.Intn(listSize) + testElement := crand.Intn(listSize) // Slice to shuffle list := make([]int, 0, listSize) for i := 0; i < listSize; i++ { @@ -299,12 +301,12 @@ func TestShuffle(t *testing.T) { } func TestSamples(t *testing.T) { - seedMathRand(t) + rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - rand.Read(customizer) + crand.Read(customizer) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -321,7 +323,7 @@ func TestSamples(t *testing.T) { samplingDistribution := make([]float64, listSize) // tests the subset ordering randomness (using a particular element testElement) orderingDistribution := make([]float64, samplesSize) - testElement := rand.Intn(listSize) + testElement := crand.Intn(listSize) // Slice to shuffle list := make([]int, 0, listSize) for i := 0; i < listSize; i++ { @@ -390,13 +392,13 @@ func TestSamples(t *testing.T) { // TestStateRestore tests the serilaization and deserialization functions // Store and Restore func TestStateRestore(t *testing.T) { - seedMathRand(t) + rand := getPRG(t) // generate a seed seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, _ = crand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - rand.Read(customizer) + crand.Read(customizer) t.Logf("seed is %x, customizer is %x\n", seed, customizer) // create an rng @@ -404,7 +406,7 @@ func TestStateRestore(t *testing.T) { require.NoError(t, err) // evolve the internal state of the rng - iterations := rand.Intn(1000) + iterations := crand.Intn(1000) for i := 0; i < iterations; i++ { _ = rng.UintN(1024) } @@ -421,7 +423,7 @@ func TestStateRestore(t *testing.T) { assert.True(t, bytes.Equal(state, secondRng.Store()), "Store o Restore is not identity") // check the 2 PRGs are generating identical outputs - iterations = rand.Intn(1000) + iterations = crand.Intn(1000) for i := 0; i < iterations; i++ { rand1 := rng.UintN(1024) rand2 := secondRng.UintN(1024) diff --git a/crypto/sign_test_utils.go b/crypto/sign_test_utils.go index 0f5d38a1d97..ec966563431 100644 --- a/crypto/sign_test_utils.go +++ b/crypto/sign_test_utils.go @@ -12,6 +12,13 @@ import ( "github.com/onflow/flow-go/crypto/hash" ) +func getPRG(t *testing.T) *mrand.Rand { + random := time.Now().UnixNano() + t.Logf("rng seed is %d", random) + rng := mrand.New(mrand.NewSource(random)) + return rng +} + func TestKeyGenErrors(t *testing.T) { seed := make([]byte, 50) invalidSigAlgo := SigningAlgorithm(20) @@ -52,18 +59,16 @@ func testGenSignVerify(t *testing.T, salg SigningAlgorithm, halg hash.Hasher) { seedMinLength := 48 seed := make([]byte, seedMinLength) input := make([]byte, 100) - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) loops := 50 for j := 0; j < loops; j++ { - n, err := mrand.Read(seed) + n, err := rand.Read(seed) require.Equal(t, n, seedMinLength) require.NoError(t, err) sk, err := GeneratePrivateKey(salg, seed) require.NoError(t, err) - _, err = mrand.Read(input) + _, err = rand.Read(input) require.NoError(t, err) s, err := sk.Sign(input, halg) require.NoError(t, err) @@ -93,8 +98,8 @@ func testGenSignVerify(t *testing.T, salg SigningAlgorithm, halg hash.Hasher) { "Verification should fail:\n signature:%s\n message:%x\n private key:%s", s, input, sk)) // test a wrong signature length - invalidLen := mrand.Intn(2 * len(s)) // try random invalid lengths - if invalidLen == len(s) { // map to an invalid length + invalidLen := rand.Intn(2 * len(s)) // try random invalid lengths + if invalidLen == len(s) { // map to an invalid length invalidLen = 0 } invalidSig := make([]byte, invalidLen) @@ -126,9 +131,7 @@ func testKeyGenSeed(t *testing.T, salg SigningAlgorithm, minLen int, maxLen int) func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { t.Logf("Testing encode/decode for %s", salg) - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // make sure the length is larger than minimum lengths of all the signaure algos seedMinLength := 48 @@ -137,7 +140,7 @@ func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { for j := 0; j < loops; j++ { // generate a private key seed := make([]byte, seedMinLength) - read, err := mrand.Read(seed) + read, err := rand.Read(seed) require.Equal(t, read, seedMinLength) require.NoError(t, err) sk, err := GeneratePrivateKey(salg, seed) @@ -230,15 +233,13 @@ func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { func testEquals(t *testing.T, salg SigningAlgorithm, otherSigAlgo SigningAlgorithm) { t.Logf("Testing Equals for %s", salg) - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + rand := getPRG(t) // make sure the length is larger than minimum lengths of all the signaure algos seedMinLength := 48 // generate a key pair seed := make([]byte, seedMinLength) - n, err := mrand.Read(seed) + n, err := rand.Read(seed) require.Equal(t, n, seedMinLength) require.NoError(t, err) diff --git a/crypto/spock_test.go b/crypto/spock_test.go index e617c6d0518..d8b5becbf8c 100644 --- a/crypto/spock_test.go +++ b/crypto/spock_test.go @@ -4,7 +4,7 @@ package crypto import ( - "crypto/rand" + crand "crypto/rand" "testing" "github.com/stretchr/testify/assert" @@ -16,12 +16,12 @@ func TestSPOCKProveVerifyAgainstData(t *testing.T) { seed := make([]byte, KeyGenSeedMinLenBLSBLS12381) data := make([]byte, 100) - n, err := rand.Read(seed) + n, err := crand.Read(seed) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) sk, err := GeneratePrivateKey(BLSBLS12381, seed) require.NoError(t, err) - _, err = rand.Read(data) + _, err = crand.Read(data) require.NoError(t, err) // generate a SPoCK proof @@ -87,16 +87,16 @@ func TestSPOCKProveVerify(t *testing.T) { data := make([]byte, 100) // data - _, err := rand.Read(data) + _, err := crand.Read(data) require.NoError(t, err) // sk1 - n, err := rand.Read(seed1) + n, err := crand.Read(seed1) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) sk1, err := GeneratePrivateKey(BLSBLS12381, seed1) require.NoError(t, err) // sk2 - n, err = rand.Read(seed2) + n, err = crand.Read(seed2) require.Equal(t, n, KeyGenSeedMinLenBLSBLS12381) require.NoError(t, err) sk2, err := GeneratePrivateKey(BLSBLS12381, seed2) From 9d19b5194c73d10da36251c79126ed46486ef581 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 20:04:31 -0600 Subject: [PATCH 005/169] add tests for new package utils/rand --- utils/rand/rand.go | 6 +- utils/rand/rand_test.go | 258 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 utils/rand/rand_test.go diff --git a/utils/rand/rand.go b/utils/rand/rand.go index 2912c2eec58..8d87712e2b0 100644 --- a/utils/rand/rand.go +++ b/utils/rand/rand.go @@ -134,12 +134,12 @@ func Uintn(n uint) (uint, error) { // // O(1) space and O(n) time func Shuffle(n uint, swap func(i, j uint)) error { - for i := n - 1; i > 0; i-- { - j, err := Uintn(i + 1) + for i := int(n - 1); i > 0; i-- { + j, err := Uintn(uint(i + 1)) if err != nil { return err } - swap(i, j) + swap(uint(i), j) } return nil } diff --git a/utils/rand/rand_test.go b/utils/rand/rand_test.go new file mode 100644 index 00000000000..8baf9d956ca --- /dev/null +++ b/utils/rand/rand_test.go @@ -0,0 +1,258 @@ +package rand + +import ( + "fmt" + "math" + mrand "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gonum.org/v1/gonum/stat" +) + +// Simple unit tests using a very basic randomness test. +// It doesn't evaluate randomness of the output and doesn't perform advanced statistical tests. +func TestRandomIntegers(t *testing.T) { + + t.Run("basic randomness", func(t *testing.T) { + sampleSize := 80000 + tolerance := 0.05 + n := 10 + mrand.Intn(100) + distribution := make([]float64, n) + + t.Run("Uint", func(t *testing.T) { + // partition all outputs into `n` classes and compute the distribution + // over the partition. Each class has a width of `classWidth` + classWidth := math.MaxUint / uint(n) + // populate the distribution + for i := 0; i < sampleSize; i++ { + r, err := Uint() + require.NoError(t, err) + distribution[r/classWidth] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("Uint64", func(t *testing.T) { + // partition all outputs into `n` classes and compute the distribution + // over the partition. Each class has a width of `classWidth` + classWidth := math.MaxUint64 / uint64(n) + // populate the distribution + for i := 0; i < sampleSize; i++ { + r, err := Uint64() + require.NoError(t, err) + distribution[r/classWidth] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("Uint32", func(t *testing.T) { + // partition all outputs into `n` classes and compute the distribution + // over the partition. Each class has a width of `classWidth` + classWidth := math.MaxUint32 / uint32(n) + // populate the distribution + for i := 0; i < sampleSize; i++ { + r, err := Uint32() + require.NoError(t, err) + distribution[r/classWidth] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("Uintn", func(t *testing.T) { + // partition all outputs into `n` classes, each of width 1, + // and compute the distribution over the partition. + for i := 0; i < sampleSize; i++ { + r, err := Uintn(uint(n)) + require.NoError(t, err) + require.Less(t, r, uint(n)) + distribution[r] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("Uint64n", func(t *testing.T) { + for i := 0; i < sampleSize; i++ { + r, err := Uint64n(uint64(n)) + require.NoError(t, err) + require.Less(t, r, uint64(n)) + distribution[r] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("Uint32n", func(t *testing.T) { + for i := 0; i < sampleSize; i++ { + r, err := Uint32n(uint32(n)) + require.NoError(t, err) + require.Less(t, r, uint32(n)) + distribution[r] += 1.0 + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + }) + + t.Run("zero n error", func(t *testing.T) { + t.Run("Uintn", func(t *testing.T) { + _, err := Uintn(uint(0)) + require.Error(t, err) + }) + t.Run("Uint64n", func(t *testing.T) { + _, err := Uint64n(uint64(0)) + require.Error(t, err) + }) + t.Run("Uint32n", func(t *testing.T) { + _, err := Uint32n(uint32(0)) + require.Error(t, err) + }) + }) +} + +// Simple unit testing of Shuffle using a very basic randomness test. +// It doesn't evaluate randomness of the output and doesn't perform advanced statistical tests. +func TestShuffle(t *testing.T) { + + t.Run("basic randomness", func(t *testing.T) { + listSize := 100 + // test parameters + sampleSize := 80000 + tolerance := 0.05 + // the distribution of a particular element of the list, testElement + distribution := make([]float64, listSize) + testElement := mrand.Intn(listSize) + // Slice to shuffle + list := make([]int, listSize) + + shuffleAndCount := func(t *testing.T) { + err := Shuffle(uint(listSize), func(i, j uint) { + list[i], list[j] = list[j], list[i] + }) + require.NoError(t, err) + has := make(map[int]struct{}) + for j, e := range list { + // check for repetition + _, ok := has[e] + require.False(t, ok, "duplicated item") + has[e] = struct{}{} + // fill the distribution + if e == testElement { + distribution[j] += 1.0 + } + } + } + + t.Run("shuffle a random permutation", func(t *testing.T) { + // initialize the list + for i := 0; i < listSize; i++ { + list[i] = i + } + // shuffle and count multiple times + for k := 0; k < sampleSize; k++ { + shuffleAndCount(t) + } + // compute the distribution + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("shuffle a same permutation", func(t *testing.T) { + for k := 0; k < sampleSize; k++ { + for i := 0; i < listSize; i++ { + list[i] = i + } + // suffle the same permutation + shuffleAndCount(t) + } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + }) + + t.Run("empty slice", func(t *testing.T) { + emptySlice := make([]float64, 0) + err := Shuffle(0, func(i, j uint) { + emptySlice[i], emptySlice[j] = emptySlice[j], emptySlice[i] + }) + require.NoError(t, err) + assert.True(t, len(emptySlice) == 0) + }) +} + +func TestSamples(t *testing.T) { + t.Run("basic randmoness", func(t *testing.T) { + listSize := 100 + samplesSize := 20 + // statictics parameters + sampleSize := 100000 + tolerance := 0.05 + // tests the subset sampling randomness + samplingDistribution := make([]float64, listSize) + // tests the subset ordering randomness (using a particular element testElement) + orderingDistribution := make([]float64, samplesSize) + testElement := mrand.Intn(listSize) + // Slice to shuffle + list := make([]int, 0, listSize) + for i := 0; i < listSize; i++ { + list = append(list, i) + } + + for i := 0; i < sampleSize; i++ { + err := Samples(uint(listSize), uint(samplesSize), func(i, j uint) { + list[i], list[j] = list[j], list[i] + }) + require.NoError(t, err) + has := make(map[int]struct{}) + for j, e := range list[:samplesSize] { + // check for repetition + _, ok := has[e] + require.False(t, ok, "duplicated item") + has[e] = struct{}{} + // fill the distribution + samplingDistribution[e] += 1.0 + if e == testElement { + orderingDistribution[j] += 1.0 + } + } + } + stdev := stat.StdDev(samplingDistribution, nil) + mean := stat.Mean(samplingDistribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic subset randomness test failed. stdev %v, mean %v", stdev, mean)) + stdev = stat.StdDev(orderingDistribution, nil) + mean = stat.Mean(orderingDistribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic ordering randomness test failed. stdev %v, mean %v", stdev, mean)) + }) + + t.Run("zero edge cases", func(t *testing.T) { + // Sampling from an empty set + emptySlice := make([]float64, 0) + err := Samples(0, 0, func(i, j uint) { + emptySlice[i], emptySlice[j] = emptySlice[j], emptySlice[i] + }) + require.NoError(t, err) + assert.True(t, len(emptySlice) == 0) + + // drawing a sample of size zero from an non-empty list should leave the original list unmodified + constant := []float64{0, 1, 2, 3, 4, 5} + fullSlice := constant + err = Samples(uint(len(fullSlice)), 0, func(i, j uint) { // modifies fullSlice in-place + emptySlice[i], emptySlice[j] = emptySlice[j], emptySlice[i] + }) + require.NoError(t, err) + assert.Equal(t, constant, fullSlice) + }) +} From 92fdcaee5559070afdbd547a99716ec9832386e6 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 20:44:38 -0600 Subject: [PATCH 006/169] fix more errors --- crypto/bls_test.go | 10 +++---- crypto/ecdsa_test.go | 15 ++++++---- crypto/random/rand_test.go | 28 +++++++++++-------- crypto/sign_test_utils.go | 7 ++--- .../wintermute/attackOrchestrator_test.go | 14 ++++++++-- 5 files changed, 46 insertions(+), 28 deletions(-) diff --git a/crypto/bls_test.go b/crypto/bls_test.go index 187ce1aaaf2..35afa21c1ed 100644 --- a/crypto/bls_test.go +++ b/crypto/bls_test.go @@ -65,7 +65,7 @@ func TestBLSMainMethods(t *testing.T) { for _, sk := range []PrivateKey{sk1, skMinus1} { input := make([]byte, 100) - _, err = mrand.Read(input) + _, err = crand.Read(input) require.NoError(t, err) s, err := sk.Sign(input, hasher) require.NoError(t, err) @@ -628,12 +628,12 @@ func TestBLSBatchVerify(t *testing.T) { rand := getPRG(t) // random message input := make([]byte, 100) - _, err := mrand.Read(input) + _, err := rand.Read(input) require.NoError(t, err) // hasher kmac := NewExpandMsgXOFKMAC128("test tag") // number of signatures to aggregate - sigsNum := mrand.Intn(100) + 2 + sigsNum := rand.Intn(100) + 2 sigs := make([]Signature, 0, sigsNum) sks := make([]PrivateKey, 0, sigsNum) pks := make([]PublicKey, 0, sigsNum) @@ -669,14 +669,14 @@ func TestBLSBatchVerify(t *testing.T) { }) // pick a random number of invalid signatures - invalidSigsNum := mrand.Intn(sigsNum-1) + 1 + invalidSigsNum := rand.Intn(sigsNum-1) + 1 // generate a random permutation of indices to pick the // invalid signatures. indices := make([]int, 0, sigsNum) for i := 0; i < sigsNum; i++ { indices = append(indices, i) } - mrand.Shuffle(sigsNum, func(i, j int) { + rand.Shuffle(sigsNum, func(i, j int) { indices[i], indices[j] = indices[j], indices[i] }) diff --git a/crypto/ecdsa_test.go b/crypto/ecdsa_test.go index de3b58e491c..342162668cf 100644 --- a/crypto/ecdsa_test.go +++ b/crypto/ecdsa_test.go @@ -247,7 +247,8 @@ func TestSignatureFormatCheck(t *testing.T) { t.Run("valid signature", func(t *testing.T) { len := ecdsaSigLen[curve] sig := Signature(make([]byte, len)) - crand.Read(sig) + _, err := crand.Read(sig) + require.NoError(t, err) sig[len/2] = 0 // force s to be less than the curve order sig[len-1] |= 1 // force s to be non zero sig[0] = 0 // force r to be less than the curve order @@ -274,7 +275,8 @@ func TestSignatureFormatCheck(t *testing.T) { // signature with a zero s len := ecdsaSigLen[curve] sig0s := Signature(make([]byte, len)) - crand.Read(sig0s[:len/2]) + _, err := crand.Read(sig0s[:len/2]) + require.NoError(t, err) valid, err := SignatureFormatCheck(curve, sig0s) assert.Nil(t, err) @@ -282,7 +284,8 @@ func TestSignatureFormatCheck(t *testing.T) { // signature with a zero r sig0r := Signature(make([]byte, len)) - crand.Read(sig0r[len/2:]) + _, err = crand.Read(sig0r[len/2:]) + require.NoError(t, err) valid, err = SignatureFormatCheck(curve, sig0r) assert.Nil(t, err) @@ -292,7 +295,8 @@ func TestSignatureFormatCheck(t *testing.T) { t.Run("large values", func(t *testing.T) { len := ecdsaSigLen[curve] sigLargeS := Signature(make([]byte, len)) - crand.Read(sigLargeS[:len/2]) + _, err := crand.Read(sigLargeS[:len/2]) + require.NoError(t, err) // make sure s is larger than the curve order for i := len / 2; i < len; i++ { sigLargeS[i] = 0xFF @@ -303,7 +307,8 @@ func TestSignatureFormatCheck(t *testing.T) { assert.False(t, valid) sigLargeR := Signature(make([]byte, len)) - crand.Read(sigLargeR[len/2:]) + _, err = crand.Read(sigLargeR[len/2:]) + require.NoError(t, err) // make sure s is larger than the curve order for i := 0; i < len/2; i++ { sigLargeR[i] = 0xFF diff --git a/crypto/random/rand_test.go b/crypto/random/rand_test.go index 5ce36464824..0fd1b6f1b24 100644 --- a/crypto/random/rand_test.go +++ b/crypto/random/rand_test.go @@ -2,7 +2,6 @@ package random import ( "bytes" - crand "crypto/rand" "fmt" mrand "math/rand" "testing" @@ -100,9 +99,11 @@ func TestUint(t *testing.T) { rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) customizer := make([]byte, Chacha20CustomizerMaxLen) - crand.Read(customizer) + _, err = rand.Read(customizer) + require.NoError(t, err) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -138,9 +139,11 @@ func TestSubPermutation(t *testing.T) { rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) customizer := make([]byte, Chacha20CustomizerMaxLen) - crand.Read(customizer) + _, err = rand.Read(customizer) + require.NoError(t, err) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -221,9 +224,11 @@ func TestShuffle(t *testing.T) { rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) customizer := make([]byte, Chacha20CustomizerMaxLen) - crand.Read(customizer) + _, err = rand.Read(customizer) + require.NoError(t, err) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -304,9 +309,10 @@ func TestSamples(t *testing.T) { rand := getPRG(t) seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, err := rand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - crand.Read(customizer) + _, err = rand.Read(customizer) + require.NoError(t, err) rng, err := NewChacha20PRG(seed, customizer) require.NoError(t, err) @@ -396,9 +402,9 @@ func TestStateRestore(t *testing.T) { // generate a seed seed := make([]byte, Chacha20SeedLen) - _, _ = rand.Read(seed) + _, err := rand.Read(seed) customizer := make([]byte, Chacha20CustomizerMaxLen) - crand.Read(customizer) + _, err = rand.Read(customizer) t.Logf("seed is %x, customizer is %x\n", seed, customizer) // create an rng diff --git a/crypto/sign_test_utils.go b/crypto/sign_test_utils.go index f3eba279647..a98f7d0713b 100644 --- a/crypto/sign_test_utils.go +++ b/crypto/sign_test_utils.go @@ -1,6 +1,7 @@ package crypto import ( + crand "crypto/rand" "fmt" mrand "math/rand" "testing" @@ -137,12 +138,10 @@ func testKeyGenSeed(t *testing.T, salg SigningAlgorithm, minLen int, maxLen int) }) t.Run("deterministic generation", func(t *testing.T) { - r := time.Now().UnixNano() - mrand.Seed(r) - t.Logf("math rand seed is %d", r) + // same seed results in the same key seed := make([]byte, minLen) - read, err := mrand.Read(seed) + read, err := crand.Read(seed) require.Equal(t, read, minLen) require.NoError(t, err) sk1, err := GeneratePrivateKey(salg, seed) diff --git a/insecure/wintermute/attackOrchestrator_test.go b/insecure/wintermute/attackOrchestrator_test.go index fdd7768c229..1c5d46f6899 100644 --- a/insecure/wintermute/attackOrchestrator_test.go +++ b/insecure/wintermute/attackOrchestrator_test.go @@ -557,8 +557,11 @@ func TestPassingThroughMiscellaneousEvents(t *testing.T) { // creates a block event fixture that is out of the context of // the wintermute attack. + random, err := rand.Uintn(uint(len(corruptedIds))) + require.NoError(t, err) + miscellaneousEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], + CorruptOriginId: corruptedIds[random], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, @@ -631,8 +634,11 @@ func TestPassingThrough_ResultApproval(t *testing.T) { approval := unittest.ResultApprovalFixture() require.NotEqual(t, wintermuteOrchestrator.state.originalResult.ID(), approval.ID()) require.NotEqual(t, wintermuteOrchestrator.state.corruptedResult.ID(), approval.ID()) + + random, err := rand.Uintn(uint(len(corruptedIds))) + require.NoError(t, err) approvalEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], + CorruptOriginId: corruptedIds[random], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, @@ -703,8 +709,10 @@ func TestWintermute_ResultApproval(t *testing.T) { } // generates a result approval event for one of the chunks of the original result. + random, err := rand.Uintn(uint(len(corruptedIds))) + require.NoError(t, err) approvalEvent := &insecure.EgressEvent{ - CorruptOriginId: corruptedIds[rand.Uint64n(len(corruptedIds))], + CorruptOriginId: corruptedIds[random], Channel: channels.TestNetworkChannel, Protocol: insecure.Protocol_MULTICAST, TargetNum: 3, From 5e2b2553015fe1e19380962f1b44bbe7a80d6832 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 22:14:45 -0600 Subject: [PATCH 007/169] more linter errors --- crypto/hash/hash_test.go | 15 ++++++++++----- fvm/crypto/crypto_test.go | 18 ++++++++++++------ ledger/common/hash/hash_test.go | 4 +++- utils/unittest/fixtures.go | 2 +- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/crypto/hash/hash_test.go b/crypto/hash/hash_test.go index 5bdef372f8f..1545ba53780 100644 --- a/crypto/hash/hash_test.go +++ b/crypto/hash/hash_test.go @@ -166,7 +166,8 @@ func TestSHA2(t *testing.T) { t.Run("SHA2_256", func(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) expected := sha256.Sum256(value) // test hash computation using the hasher @@ -184,7 +185,8 @@ func TestSHA2(t *testing.T) { t.Run("SHA2_384", func(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) expected := sha512.Sum384(value) hasher := NewSHA2_384() @@ -201,7 +203,8 @@ func TestSHA3(t *testing.T) { t.Run("SHA3_256", func(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) expected := sha3.Sum256(value) // test hash computation using the hasher @@ -219,7 +222,8 @@ func TestSHA3(t *testing.T) { t.Run("SHA3_384", func(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) expected := sha3.Sum384(value) hasher := NewSHA3_384() @@ -235,7 +239,8 @@ func TestSHA3(t *testing.T) { func TestKeccak(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) k := sha3.NewLegacyKeccak256() k.Write(value) expected := k.Sum(nil) diff --git a/fvm/crypto/crypto_test.go b/fvm/crypto/crypto_test.go index 1e9b3a4bffc..1640f03e9f8 100644 --- a/fvm/crypto/crypto_test.go +++ b/fvm/crypto/crypto_test.go @@ -88,7 +88,8 @@ func TestVerifySignatureFromRuntime(t *testing.T) { for _, h := range hashAlgos { t.Run(fmt.Sprintf("combination: %v, %v", s, h), func(t *testing.T) { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) pk, err := gocrypto.GeneratePrivateKey(crypto.RuntimeToCryptoSigningAlgorithm(s), seed) require.NoError(t, err) @@ -179,7 +180,8 @@ func TestVerifySignatureFromRuntime(t *testing.T) { for _, c := range cases { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) pk, err := gocrypto.GeneratePrivateKey(gocrypto.BLSBLS12381, seed) require.NoError(t, err) @@ -261,7 +263,8 @@ func TestVerifySignatureFromRuntime(t *testing.T) { t.Run(fmt.Sprintf("hash tag: %v, verify tag: %v [%v, %v]", c.signTag, c.verifyTag, s, h), func(t *testing.T) { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) pk, err := gocrypto.GeneratePrivateKey(crypto.RuntimeToCryptoSigningAlgorithm(s), seed) require.NoError(t, err) @@ -326,7 +329,8 @@ func TestVerifySignatureFromTransaction(t *testing.T) { for _, h := range hashAlgos { t.Run(fmt.Sprintf("combination: %v, %v", s, h), func(t *testing.T) { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) sk, err := gocrypto.GeneratePrivateKey(s, seed) require.NoError(t, err) @@ -397,7 +401,8 @@ func TestVerifySignatureFromTransaction(t *testing.T) { for h := range hMaps { t.Run(fmt.Sprintf("sign tag: %v [%v, %v]", c.signTag, s, h), func(t *testing.T) { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) sk, err := gocrypto.GeneratePrivateKey(s, seed) require.NoError(t, err) @@ -425,7 +430,8 @@ func TestValidatePublicKey(t *testing.T) { validPublicKey := func(t *testing.T, s runtime.SignatureAlgorithm) []byte { seed := make([]byte, seedLength) - rand.Read(seed) + _, err := rand.Read(seed) + require.NoError(t, err) pk, err := gocrypto.GeneratePrivateKey(crypto.RuntimeToCryptoSigningAlgorithm(s), seed) require.NoError(t, err) return pk.PublicKey().Encode() diff --git a/ledger/common/hash/hash_test.go b/ledger/common/hash/hash_test.go index 9713340a3a9..fb51fb1e44d 100644 --- a/ledger/common/hash/hash_test.go +++ b/ledger/common/hash/hash_test.go @@ -7,6 +7,7 @@ import ( "golang.org/x/crypto/sha3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" cryhash "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/ledger" @@ -24,7 +25,8 @@ func TestHash(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) rand.Read(path[:]) - rand.Read(value) + _, err := rand.Read(value) + require.NoError(t, err) h := hash.HashLeaf(path, value) hasher := sha3.New256() diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 4f7025d58dc..578e9a6c81d 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -436,7 +436,7 @@ func BlockHeaderFixture(opts ...func(header *flow.Header)) *flow.Header { func CidFixture() cid.Cid { data := make([]byte, 1024) - rand.Read(data) + _, _ = rand.Read(data) return blocks.NewBlock(data).Cid() } From 5b58584a1f59d32b062ecdfae4b023d215020d43 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 22:38:25 -0600 Subject: [PATCH 008/169] fix more linter errors --- fvm/crypto/crypto_test.go | 2 +- fvm/crypto/hash_test.go | 14 ++++++-------- fvm/environment/unsafe_random_generator.go | 5 ++++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/fvm/crypto/crypto_test.go b/fvm/crypto/crypto_test.go index 1640f03e9f8..fe6c400c1b4 100644 --- a/fvm/crypto/crypto_test.go +++ b/fvm/crypto/crypto_test.go @@ -1,8 +1,8 @@ package crypto_test import ( + "crypto/rand" "fmt" - "math/rand" "testing" "unicode/utf8" diff --git a/fvm/crypto/hash_test.go b/fvm/crypto/hash_test.go index afd4803edf4..bb9bb64172b 100644 --- a/fvm/crypto/hash_test.go +++ b/fvm/crypto/hash_test.go @@ -3,7 +3,6 @@ package crypto_test import ( "math/rand" "testing" - "time" "crypto/sha256" "crypto/sha512" @@ -45,16 +44,13 @@ func TestPrefixedHash(t *testing.T) { }, } - r := time.Now().UnixNano() - rand.Seed(r) - t.Logf("math rand seed is %d", r) - for hashAlgo, testFunction := range hashingAlgoToTestingAlgo { t.Run(hashAlgo.String()+" with a prefix", func(t *testing.T) { for i := flow.DomainTagLength; i < 5000; i++ { // first 32 bytes of data are the tag data := make([]byte, i) - rand.Read(data) + _, err := rand.Read(data) + require.NoError(t, err) expected := testFunction(data) tag := string(data[:flow.DomainTagLength]) @@ -69,7 +65,8 @@ func TestPrefixedHash(t *testing.T) { t.Run(hashAlgo.String()+" without a prefix", func(t *testing.T) { for i := 0; i < 5000; i++ { data := make([]byte, i) - rand.Read(data) + _, err := rand.Read(data) + require.NoError(t, err) expected := testFunction(data) tag := "" @@ -82,7 +79,8 @@ func TestPrefixedHash(t *testing.T) { t.Run(hashAlgo.String()+" with tagged prefix", func(t *testing.T) { data := make([]byte, 100) // data to hash - rand.Read(data) + _, err := rand.Read(data) + require.NoError(t, err) tag := "tag" // tag to be padded hasher, err := crypto.NewPrefixedHashing(hashAlgo, tag) diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index 950c9d94c2a..0d9c36306db 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -82,7 +82,10 @@ func (gen *unsafeRandomGenerator) seed() { // extract the entropy from `id` and expand it into the required seed hkdf := hkdf.New(func() hash.Hash { return sha256.New() }, id[:], nil, nil) seed := make([]byte, random.Chacha20SeedLen) - hkdf.Read(seed) + n, err := hkdf.Read(seed) + if n != len(seed) || err != nil { + return + } // initialize a fresh CSPRNG with the seed (crypto-secure PRG) source, err := random.NewChacha20PRG(seed, []byte{}) if err != nil { From 2b00584150a59b4f398b1d250c9c897b6998b9bf Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 15 Mar 2023 22:40:16 -0600 Subject: [PATCH 009/169] 1.20 never ending linter errors --- .../signature/randombeacon_signer_store_test.go | 2 -- crypto/hash/hash_test.go | 5 +++-- ledger/common/bitutils/utils_test.go | 7 +++++-- ledger/common/hash/hash_test.go | 15 +++++++++------ ledger/common/testutils/testutils.go | 4 ++-- ledger/complete/mtrie/flattener/encoding_test.go | 3 ++- ledger/complete/mtrie/trie/trie_test.go | 6 ++++-- ledger/complete/mtrie/trieCache_test.go | 4 ++-- ledger/complete/wal/checkpoint_v6_test.go | 6 +++--- ledger/complete/wal/triequeue_test.go | 4 ++-- model/flow/identifier_test.go | 3 ++- storage/merkle/tree_test.go | 10 ++++++---- utils/unittest/network/fixtures.go | 4 ++-- 13 files changed, 42 insertions(+), 31 deletions(-) diff --git a/consensus/hotstuff/signature/randombeacon_signer_store_test.go b/consensus/hotstuff/signature/randombeacon_signer_store_test.go index 87ceeb0a7fe..c578e1b2e97 100644 --- a/consensus/hotstuff/signature/randombeacon_signer_store_test.go +++ b/consensus/hotstuff/signature/randombeacon_signer_store_test.go @@ -4,7 +4,6 @@ import ( "errors" "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,7 +30,6 @@ func TestBeaconKeyStore(t *testing.T) { } func (suite *BeaconKeyStore) SetupTest() { - rand.Seed(time.Now().Unix()) suite.epochLookup = mockmodule.NewEpochLookup(suite.T()) suite.beaconKeys = mockstorage.NewSafeBeaconKeys(suite.T()) suite.store = NewEpochAwareRandomBeaconKeyStore(suite.epochLookup, suite.beaconKeys) diff --git a/crypto/hash/hash_test.go b/crypto/hash/hash_test.go index 1545ba53780..e1b30efd6a8 100644 --- a/crypto/hash/hash_test.go +++ b/crypto/hash/hash_test.go @@ -122,7 +122,8 @@ func TestHashersAPI(t *testing.T) { } data := make([]byte, 1801) - rand.Read(data) + _, err := rand.Read(data) + require.NoError(t, err) for _, newFunction := range newHasherFunctions { // Reset should empty the state @@ -256,7 +257,7 @@ func TestKeccak(t *testing.T) { func BenchmarkComputeHash(b *testing.B) { m := make([]byte, 32) - rand.Read(m) + _, _ = rand.Read(m) b.Run("SHA2_256", func(b *testing.B) { b.ResetTimer() diff --git a/ledger/common/bitutils/utils_test.go b/ledger/common/bitutils/utils_test.go index 8671711fdf3..975bf70ec10 100644 --- a/ledger/common/bitutils/utils_test.go +++ b/ledger/common/bitutils/utils_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestBitVectorAllocation(t *testing.T) { @@ -72,7 +73,8 @@ func TestBitTools(t *testing.T) { t.Run("testing WriteBit", func(t *testing.T) { b.SetInt64(0) bytes := MakeBitVector(maxBits) - crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + _, err := crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + require.NoError(t, err) // build a random big bit by bit for idx := 0; idx < maxBits; idx++ { @@ -92,7 +94,8 @@ func TestBitTools(t *testing.T) { t.Run("testing ClearBit and SetBit", func(t *testing.T) { b.SetInt64(0) bytes := MakeBitVector(maxBits) - crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + _, err := crand.Read(bytes) // fill bytes with random values to verify that writing to each individual bit works + require.NoError(t, err) // build a random big bit by bit for idx := 0; idx < maxBits; idx++ { diff --git a/ledger/common/hash/hash_test.go b/ledger/common/hash/hash_test.go index fb51fb1e44d..69a1102e358 100644 --- a/ledger/common/hash/hash_test.go +++ b/ledger/common/hash/hash_test.go @@ -24,8 +24,9 @@ func TestHash(t *testing.T) { for i := 0; i < 5000; i++ { value := make([]byte, i) - rand.Read(path[:]) - _, err := rand.Read(value) + _, err := rand.Read(path[:]) + require.NoError(t, err) + _, err = rand.Read(value) require.NoError(t, err) h := hash.HashLeaf(path, value) @@ -41,8 +42,10 @@ func TestHash(t *testing.T) { var h1, h2 hash.Hash for i := 0; i < 5000; i++ { - rand.Read(h1[:]) - rand.Read(h2[:]) + _, err := rand.Read(h1[:]) + require.NoError(t, err) + _, err = rand.Read(h2[:]) + require.NoError(t, err) h := hash.HashInterNode(h1, h2) hasher := sha3.New256() @@ -91,8 +94,8 @@ func Test_ComputeCompactValue(t *testing.T) { func BenchmarkHash(b *testing.B) { var h1, h2 hash.Hash - rand.Read(h1[:]) - rand.Read(h2[:]) + _, _ = rand.Read(h1[:]) + _, _ = rand.Read(h2[:]) // customized sha3 for ledger b.Run("LedgerSha3", func(b *testing.B) { diff --git a/ledger/common/testutils/testutils.go b/ledger/common/testutils/testutils.go index 8543abbc0de..1e937cc94b1 100644 --- a/ledger/common/testutils/testutils.go +++ b/ledger/common/testutils/testutils.go @@ -152,7 +152,7 @@ func RandomPaths(n int) []l.Path { i := 0 for i < n { var path l.Path - crand.Read(path[:]) + _, _ = crand.Read(path[:]) // deduplicate if _, found := alreadySelectPaths[path]; !found { paths = append(paths, path) @@ -167,7 +167,7 @@ func RandomPaths(n int) []l.Path { func RandomPayload(minByteSize int, maxByteSize int) *l.Payload { keyByteSize := minByteSize + rand.Intn(maxByteSize-minByteSize) keydata := make([]byte, keyByteSize) - crand.Read(keydata) + _, _ = crand.Read(keydata) key := l.Key{KeyParts: []l.KeyPart{{Type: 0, Value: keydata}}} valueByteSize := minByteSize + rand.Intn(maxByteSize-minByteSize) valuedata := make([]byte, valueByteSize) diff --git a/ledger/complete/mtrie/flattener/encoding_test.go b/ledger/complete/mtrie/flattener/encoding_test.go index 1876f2199ac..8b157a1e9d7 100644 --- a/ledger/complete/mtrie/flattener/encoding_test.go +++ b/ledger/complete/mtrie/flattener/encoding_test.go @@ -161,7 +161,8 @@ func TestRandomLeafNodeEncodingDecoding(t *testing.T) { height := rand.Intn(257) var hashValue hash.Hash - crand.Read(hashValue[:]) + _, err := crand.Read(hashValue[:]) + require.NoError(t, err) n := node.NewNode(height, nil, nil, paths[i], payloads[i], hashValue) diff --git a/ledger/complete/mtrie/trie/trie_test.go b/ledger/complete/mtrie/trie/trie_test.go index 780c63c1410..ca62da06de2 100644 --- a/ledger/complete/mtrie/trie/trie_test.go +++ b/ledger/complete/mtrie/trie/trie_test.go @@ -363,7 +363,8 @@ func TestSplitByPath(t *testing.T) { paths := make([]ledger.Path, 0, pathsNumber) for i := 0; i < pathsNumber-redundantPaths; i++ { var p ledger.Path - rand.Read(p[:]) + _, err := rand.Read(p[:]) + require.NoError(t, err) paths = append(paths, p) } for i := 0; i < redundantPaths; i++ { @@ -652,7 +653,8 @@ func Test_Pruning(t *testing.T) { for i := 0; i < numberOfUpdates; { var path ledger.Path - rand.Read(path[:]) + _, err := rand.Read(path[:]) + require.NoError(t, err) // deduplicate if _, found := allPaths[path]; !found { payload := testutils.RandomPayload(1, 100) diff --git a/ledger/complete/mtrie/trieCache_test.go b/ledger/complete/mtrie/trieCache_test.go index bc5130ddd60..f39b3e741a1 100644 --- a/ledger/complete/mtrie/trieCache_test.go +++ b/ledger/complete/mtrie/trieCache_test.go @@ -174,10 +174,10 @@ func TestConcurrentAccess(t *testing.T) { func randomMTrie() (*trie.MTrie, error) { var randomPath ledger.Path - rand.Read(randomPath[:]) + _, _ = rand.Read(randomPath[:]) var randomHashValue hash.Hash - rand.Read(randomHashValue[:]) + _, _ = rand.Read(randomHashValue[:]) root := node.NewNode(256, nil, nil, randomPath, nil, randomHashValue) diff --git a/ledger/complete/wal/checkpoint_v6_test.go b/ledger/complete/wal/checkpoint_v6_test.go index ce3dc406f43..f17a71de5bb 100644 --- a/ledger/complete/wal/checkpoint_v6_test.go +++ b/ledger/complete/wal/checkpoint_v6_test.go @@ -87,7 +87,7 @@ func createSimpleTrie(t *testing.T) []*trie.MTrie { func randPathPayload() (ledger.Path, ledger.Payload) { var path ledger.Path - rand.Read(path[:]) + _, _ = rand.Read(path[:]) payload := testutils.RandomPayload(1, 100) return path, *payload } @@ -193,10 +193,10 @@ func TestEncodeSubTrie(t *testing.T) { func randomNode() *node.Node { var randomPath ledger.Path - rand.Read(randomPath[:]) + _, _ = rand.Read(randomPath[:]) var randomHashValue hash.Hash - rand.Read(randomHashValue[:]) + _, _ = rand.Read(randomHashValue[:]) return node.NewNode(256, nil, nil, randomPath, nil, randomHashValue) } diff --git a/ledger/complete/wal/triequeue_test.go b/ledger/complete/wal/triequeue_test.go index 4f93006c3ec..415ba484dc9 100644 --- a/ledger/complete/wal/triequeue_test.go +++ b/ledger/complete/wal/triequeue_test.go @@ -127,10 +127,10 @@ func TestTrieQueueWithInitialValues(t *testing.T) { func randomMTrie() (*trie.MTrie, error) { var randomPath ledger.Path - rand.Read(randomPath[:]) + _, _ = rand.Read(randomPath[:]) var randomHashValue hash.Hash - rand.Read(randomHashValue[:]) + _, _ = rand.Read(randomHashValue[:]) root := node.NewNode(256, nil, nil, randomPath, nil, randomHashValue) diff --git a/model/flow/identifier_test.go b/model/flow/identifier_test.go index 3a6d3c33aa8..901e9dfb777 100644 --- a/model/flow/identifier_test.go +++ b/model/flow/identifier_test.go @@ -134,7 +134,8 @@ func TestCIDConversion(t *testing.T) { // generate random CID data := make([]byte, 4) - rand.Read(data) + _, err := rand.Read(data) + require.NoError(t, err) cid = blocks.NewBlock(data).Cid() id, err = flow.CidToId(cid) diff --git a/storage/merkle/tree_test.go b/storage/merkle/tree_test.go index f3f5f54daea..aea20cca8db 100644 --- a/storage/merkle/tree_test.go +++ b/storage/merkle/tree_test.go @@ -3,10 +3,10 @@ package merkle import ( + crand "crypto/rand" "encoding/hex" "fmt" - "math/rand" - crand "math/rand" + mrand "math/rand" "testing" "github.com/stretchr/testify/assert" @@ -64,7 +64,9 @@ func TestEmptyTreeHash(t *testing.T) { // generate random key-value pair key := make([]byte, keyLength) - crand.Read(key) + _, err := crand.Read(key) + require.NoError(t, err) + val := []byte{1} // add key-value pair: hash should be non-empty @@ -346,7 +348,7 @@ func TestRandomOrder(t *testing.T) { } // shuffle the keys and insert them with random order into the second tree - rand.Shuffle(len(keys), func(i int, j int) { + mrand.Shuffle(len(keys), func(i int, j int) { keys[i], keys[j] = keys[j], keys[i] }) for _, key := range keys { diff --git a/utils/unittest/network/fixtures.go b/utils/unittest/network/fixtures.go index d0cefd7622c..9990c1c1dbd 100644 --- a/utils/unittest/network/fixtures.go +++ b/utils/unittest/network/fixtures.go @@ -21,7 +21,7 @@ type TxtLookupTestCase struct { func NetIPAddrFixture() net.IPAddr { token := make([]byte, 4) - crand.Read(token) + _, _ = crand.Read(token) ip := net.IPAddr{ IP: net.IPv4(token[0], token[1], token[2], token[3]), @@ -33,7 +33,7 @@ func NetIPAddrFixture() net.IPAddr { func TxtIPFixture() string { token := make([]byte, 4) - crand.Read(token) + _, _ = crand.Read(token) return "dnsaddr=" + net.IPv4(token[0], token[1], token[2], token[3]).String() } From 5d1a43170899d123b5c2347ada64d26518310e83 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 01:17:40 -0600 Subject: [PATCH 010/169] update bootstrap beacon KG and fix integration errors --- cmd/bootstrap/cmd/dkg.go | 8 +- cmd/bootstrap/cmd/finalize_test.go | 1 - cmd/bootstrap/cmd/rootblock.go | 8 +- cmd/bootstrap/cmd/rootblock_test.go | 4 - cmd/bootstrap/dkg/dkg.go | 195 +----------------- cmd/bootstrap/dkg/dkg_test.go | 25 --- .../combined_vote_processor_v2_test.go | 2 +- .../combined_vote_processor_v3_test.go | 2 +- consensus/integration/nodes_test.go | 2 +- integration/testnet/network.go | 9 +- .../tests/access/consensus_follower_test.go | 8 +- integration/tests/consensus/inclusion_test.go | 1 - integration/tests/consensus/sealing_test.go | 1 - integration/tests/mvp/mvp_test.go | 8 +- 14 files changed, 19 insertions(+), 255 deletions(-) delete mode 100644 cmd/bootstrap/dkg/dkg_test.go diff --git a/cmd/bootstrap/cmd/dkg.go b/cmd/bootstrap/cmd/dkg.go index 5f9c5df8bd3..d7069534e64 100644 --- a/cmd/bootstrap/cmd/dkg.go +++ b/cmd/bootstrap/cmd/dkg.go @@ -11,7 +11,7 @@ import ( "github.com/onflow/flow-go/state/protocol/inmem" ) -func runDKG(nodes []model.NodeInfo) dkg.DKGData { +func runBeaconKG(nodes []model.NodeInfo) dkg.DKGData { n := len(nodes) log.Info().Msgf("read %v node infos for DKG", n) @@ -19,11 +19,7 @@ func runDKG(nodes []model.NodeInfo) dkg.DKGData { log.Debug().Msgf("will run DKG") var dkgData dkg.DKGData var err error - if flagFastKG { - dkgData, err = bootstrapDKG.RunFastKG(n, GenerateRandomSeed(crypto.SeedMinLenDKG)) - } else { - dkgData, err = bootstrapDKG.RunDKG(n, GenerateRandomSeeds(n, crypto.SeedMinLenDKG)) - } + dkgData, err = bootstrapDKG.RandomBeaconKG(n, GenerateRandomSeed(crypto.SeedMinLenDKG)) if err != nil { log.Fatal().Err(err).Msg("error running DKG") } diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 6890788da39..7ce723709d0 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -63,7 +63,6 @@ func TestFinalize_HappyPath(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true flagRootChain = chainName flagRootParent = hex.EncodeToString(rootParent[:]) flagRootHeight = rootHeight diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index f1275551657..7060fdf1a4b 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -11,7 +11,6 @@ import ( ) var ( - flagFastKG bool flagRootChain string flagRootParent string flagRootHeight uint64 @@ -22,7 +21,7 @@ var ( var rootBlockCmd = &cobra.Command{ Use: "rootblock", Short: "Generate root block data", - Long: `Run DKG, generate root block and votes for root block needed for constructing QC. Serialize all info into file`, + Long: `Run Beacon KeyGen, generate root block and votes for root block needed for constructing QC. Serialize all info into file`, Run: rootBlock, } @@ -58,9 +57,6 @@ func addRootBlockCmdFlags() { cmd.MarkFlagRequired(rootBlockCmd, "root-chain") cmd.MarkFlagRequired(rootBlockCmd, "root-parent") cmd.MarkFlagRequired(rootBlockCmd, "root-height") - - // optional parameters to influence various aspects of identity generation - rootBlockCmd.Flags().BoolVar(&flagFastKG, "fast-kg", false, "use fast (centralized) random beacon key generation instead of DKG") } func rootBlock(cmd *cobra.Command, args []string) { @@ -93,7 +89,7 @@ func rootBlock(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("running DKG for consensus nodes") - dkgData := runDKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) + dkgData := runBeaconKG(model.FilterByRole(stakingNodes, flow.RoleConsensus)) log.Info().Msg("") log.Info().Msg("constructing root block") diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index 61b11379e8e..a2ccb177e79 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -53,8 +53,6 @@ func TestRootBlock_HappyPath(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true - flagRootParent = hex.EncodeToString(rootParent[:]) flagRootChain = chainName flagRootHeight = rootHeight @@ -86,8 +84,6 @@ func TestRootBlock_Deterministic(t *testing.T) { flagPartnerWeights = partnerWeights flagInternalNodePrivInfoDir = internalPrivDir - flagFastKG = true - flagRootParent = hex.EncodeToString(rootParent[:]) flagRootChain = chainName flagRootHeight = rootHeight diff --git a/cmd/bootstrap/dkg/dkg.go b/cmd/bootstrap/dkg/dkg.go index b519c59829b..21b1992e147 100644 --- a/cmd/bootstrap/dkg/dkg.go +++ b/cmd/bootstrap/dkg/dkg.go @@ -2,205 +2,14 @@ package dkg import ( "fmt" - "sync" - "time" - - "github.com/rs/zerolog/log" "github.com/onflow/flow-go/crypto" model "github.com/onflow/flow-go/model/dkg" "github.com/onflow/flow-go/module/signature" ) -// RunDKG simulates a distributed DKG protocol by running the protocol locally -// and generating the DKG output info -func RunDKG(n int, seeds [][]byte) (model.DKGData, error) { - - if n != len(seeds) { - return model.DKGData{}, fmt.Errorf("n needs to match the number of seeds (%v != %v)", n, len(seeds)) - } - - // separate the case whith one node - if n == 1 { - sk, pk, pkGroup, err := thresholdSignKeyGenOneNode(seeds[0]) - if err != nil { - return model.DKGData{}, fmt.Errorf("run dkg failed: %w", err) - } - - dkgData := model.DKGData{ - PrivKeyShares: sk, - PubGroupKey: pkGroup, - PubKeyShares: pk, - } - - return dkgData, nil - } - - processors := make([]localDKGProcessor, 0, n) - - // create the message channels for node communication - chans := make([]chan *message, n) - for i := 0; i < n; i++ { - chans[i] = make(chan *message, 5*n) - } - - // create processors for all nodes - for i := 0; i < n; i++ { - processors = append(processors, localDKGProcessor{ - current: i, - chans: chans, - }) - } - - // create DKG instances for all nodes - for i := 0; i < n; i++ { - var err error - processors[i].dkg, err = crypto.NewJointFeldman(n, - signature.RandomBeaconThreshold(n), i, &processors[i]) - if err != nil { - return model.DKGData{}, err - } - } - - var wg sync.WaitGroup - phase := 0 - - // start DKG in all nodes - // start listening on the channels - wg.Add(n) - for i := 0; i < n; i++ { - // start dkg could also run in parallel - // but they are run sequentially to avoid having non-deterministic - // output (the PRG used is common) - err := processors[i].dkg.Start(seeds[i]) - if err != nil { - return model.DKGData{}, err - } - go dkgRunChan(&processors[i], &wg, phase) - } - phase++ - - // sync the two timeouts and start the next phase - for ; phase <= 2; phase++ { - wg.Wait() - wg.Add(n) - for i := 0; i < n; i++ { - go dkgRunChan(&processors[i], &wg, phase) - } - } - - // synchronize the main thread to end all DKGs - wg.Wait() - - skShares := make([]crypto.PrivateKey, 0, n) - - for _, processor := range processors { - skShares = append(skShares, processor.privkey) - } - - dkgData := model.DKGData{ - PrivKeyShares: skShares, - PubGroupKey: processors[0].pubgroupkey, - PubKeyShares: processors[0].pubkeys, - } - - return dkgData, nil -} - -// localDKGProcessor implements DKGProcessor interface -type localDKGProcessor struct { - current int - dkg crypto.DKGState - chans []chan *message - privkey crypto.PrivateKey - pubgroupkey crypto.PublicKey - pubkeys []crypto.PublicKey -} - -const ( - broadcast int = iota - private -) - -type message struct { - orig int - channel int - data []byte -} - -// PrivateSend sends a message from one node to another -func (proc *localDKGProcessor) PrivateSend(dest int, data []byte) { - newMsg := &message{proc.current, private, data} - proc.chans[dest] <- newMsg -} - -// Broadcast a message from one node to all nodes -func (proc *localDKGProcessor) Broadcast(data []byte) { - newMsg := &message{proc.current, broadcast, data} - for i := 0; i < len(proc.chans); i++ { - if i != proc.current { - proc.chans[i] <- newMsg - } - } -} - -// Disqualify a node -func (proc *localDKGProcessor) Disqualify(node int, log string) { -} - -// FlagMisbehavior flags a node for misbehaviour -func (proc *localDKGProcessor) FlagMisbehavior(node int, log string) { -} - -// dkgRunChan simulates processing incoming messages by a node -// it assumes proc.dkg is already running -func dkgRunChan(proc *localDKGProcessor, sync *sync.WaitGroup, phase int) { - for { - select { - case newMsg := <-proc.chans[proc.current]: - var err error - if newMsg.channel == private { - err = proc.dkg.HandlePrivateMsg(newMsg.orig, newMsg.data) - } else { - err = proc.dkg.HandleBroadcastMsg(newMsg.orig, newMsg.data) - } - if err != nil { - log.Fatal().Err(err).Msg("failed to receive DKG mst") - } - // if timeout, stop and finalize - case <-time.After(1 * time.Second): - switch phase { - case 0: - err := proc.dkg.NextTimeout() - if err != nil { - log.Fatal().Err(err).Msg("failed to wait for next timeout") - } - case 1: - err := proc.dkg.NextTimeout() - if err != nil { - log.Fatal().Err(err).Msg("failed to wait for next timeout") - } - case 2: - privkey, pubgroupkey, pubkeys, err := proc.dkg.End() - if err != nil { - log.Fatal().Err(err).Msg("end dkg error should be nit") - } - if privkey == nil { - log.Fatal().Msg("privkey was nil") - } - - proc.privkey = privkey - proc.pubgroupkey = pubgroupkey - proc.pubkeys = pubkeys - } - sync.Done() - return - } - } -} - -// RunFastKG is an alternative to RunDKG that runs much faster by using a centralized threshold signature key generation. -func RunFastKG(n int, seed []byte) (model.DKGData, error) { +// RandomBeaconKG is centralized BLS threshold signature key generation. +func RandomBeaconKG(n int, seed []byte) (model.DKGData, error) { if n == 1 { sk, pk, pkGroup, err := thresholdSignKeyGenOneNode(seed) diff --git a/cmd/bootstrap/dkg/dkg_test.go b/cmd/bootstrap/dkg/dkg_test.go deleted file mode 100644 index 9835cdca538..00000000000 --- a/cmd/bootstrap/dkg/dkg_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package dkg - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/onflow/flow-go/crypto" - "github.com/onflow/flow-go/utils/unittest" -) - -func TestRunDKG(t *testing.T) { - seedLen := crypto.SeedMinLenDKG - _, err := RunDKG(0, unittest.SeedFixtures(2, seedLen)) - require.EqualError(t, err, "n needs to match the number of seeds (0 != 2)") - - _, err = RunDKG(3, unittest.SeedFixtures(2, seedLen)) - require.EqualError(t, err, "n needs to match the number of seeds (3 != 2)") - - data, err := RunDKG(4, unittest.SeedFixtures(4, seedLen)) - require.NoError(t, err) - - require.Len(t, data.PrivKeyShares, 4) - require.Len(t, data.PubKeyShares, 4) -} diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go index fe574e4f283..47403f78a82 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go @@ -788,7 +788,7 @@ func TestCombinedVoteProcessorV2_BuildVerifyQC(t *testing.T) { epochLookup.On("EpochForViewWithFallback", view).Return(epochCounter, nil) // all committee members run DKG - dkgData, err := bootstrapDKG.RunFastKG(11, unittest.RandomBytes(32)) + dkgData, err := bootstrapDKG.RandomBeaconKG(11, unittest.RandomBytes(32)) require.NoError(t, err) // signers hold objects that are created with private key and can sign votes and proposals diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go index a4fe0e03dde..6343887d94c 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go @@ -923,7 +923,7 @@ func TestCombinedVoteProcessorV3_BuildVerifyQC(t *testing.T) { view := uint64(20) epochLookup.On("EpochForViewWithFallback", view).Return(epochCounter, nil) - dkgData, err := bootstrapDKG.RunFastKG(11, unittest.RandomBytes(32)) + dkgData, err := bootstrapDKG.RandomBeaconKG(11, unittest.RandomBytes(32)) require.NoError(t, err) // signers hold objects that are created with private key and can sign votes and proposals diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index cee5020dcce..10904534b6c 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -314,7 +314,7 @@ func createConsensusIdentities(t *testing.T, n int) *run.ParticipantData { // completeConsensusIdentities runs KG process and fills nodeInfos with missing random beacon keys func completeConsensusIdentities(t *testing.T, nodeInfos []bootstrap.NodeInfo) *run.ParticipantData { - dkgData, err := bootstrapDKG.RunFastKG(len(nodeInfos), unittest.RandomBytes(48)) + dkgData, err := bootstrapDKG.RandomBeaconKG(len(nodeInfos), unittest.RandomBytes(48)) require.NoError(t, err) participantData := &run.ParticipantData{ diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 390fc88cf7b..2af69f4b24a 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -5,7 +5,6 @@ import ( crand "crypto/rand" "encoding/hex" "fmt" - "math/rand" gonet "net" "os" "path/filepath" @@ -1146,7 +1145,7 @@ func BootstrapNetwork(networkConf NetworkConfig, bootstrapDir string, chainID fl // this ordering defines the DKG participant's indices stakedNodeInfos := bootstrap.Sort(toNodeInfos(stakedConfs), order.Canonical) - dkg, err := runDKG(stakedConfs) + dkg, err := runBeaconKG(stakedConfs) if err != nil { return nil, fmt.Errorf("failed to run DKG: %w", err) } @@ -1383,11 +1382,11 @@ func setupKeys(networkConf NetworkConfig) ([]ContainerConfig, error) { return confs, nil } -// runDKG simulates the distributed key generation process for all consensus nodes +// runBeaconKG simulates the distributed key generation process for all consensus nodes // and returns all DKG data. This includes the group private key, node indices, // and per-node public and private key-shares. // Only consensus nodes participate in the DKG. -func runDKG(confs []ContainerConfig) (dkgmod.DKGData, error) { +func runBeaconKG(confs []ContainerConfig) (dkgmod.DKGData, error) { // filter by consensus nodes consensusNodes := bootstrap.FilterByRole(toNodeInfos(confs), flow.RoleConsensus) @@ -1399,7 +1398,7 @@ func runDKG(confs []ContainerConfig) (dkgmod.DKGData, error) { return dkgmod.DKGData{}, err } - dkg, err := dkg.RunFastKG(nConsensusNodes, dkgSeed) + dkg, err := dkg.RandomBeaconKG(nConsensusNodes, dkgSeed) if err != nil { return dkgmod.DKGData{}, err } diff --git a/integration/tests/access/consensus_follower_test.go b/integration/tests/access/consensus_follower_test.go index ab71a4503f0..0dfd429dd02 100644 --- a/integration/tests/access/consensus_follower_test.go +++ b/integration/tests/access/consensus_follower_test.go @@ -2,7 +2,6 @@ package access import ( "context" - "crypto/rand" "fmt" "testing" "time" @@ -176,12 +175,7 @@ func (suite *ConsensusFollowerSuite) buildNetworkConfig() { // TODO: Move this to unittest and resolve the circular dependency issue func UnstakedNetworkingKey() (crypto.PrivateKey, error) { - seed := make([]byte, crypto.KeyGenSeedMinLenECDSASecp256k1) - _, err := rand.Read(seed) - if err != nil { - return nil, err - } - return utils.GeneratePublicNetworkingKey(unittest.SeedFixture(n)) + return utils.GeneratePublicNetworkingKey(unittest.SeedFixture(crypto.KeyGenSeedMinLenECDSASecp256k1)) } // followerManager is a convenience wrapper around the consensus follower diff --git a/integration/tests/consensus/inclusion_test.go b/integration/tests/consensus/inclusion_test.go index 572cfa6c13a..c39aa000460 100644 --- a/integration/tests/consensus/inclusion_test.go +++ b/integration/tests/consensus/inclusion_test.go @@ -2,7 +2,6 @@ package consensus import ( "context" - "math/rand" "testing" "time" diff --git a/integration/tests/consensus/sealing_test.go b/integration/tests/consensus/sealing_test.go index ddb62ae96aa..14c9ddcc69e 100644 --- a/integration/tests/consensus/sealing_test.go +++ b/integration/tests/consensus/sealing_test.go @@ -2,7 +2,6 @@ package consensus import ( "context" - "math/rand" "testing" "time" diff --git a/integration/tests/mvp/mvp_test.go b/integration/tests/mvp/mvp_test.go index 166c87688ad..5741646dbcc 100644 --- a/integration/tests/mvp/mvp_test.go +++ b/integration/tests/mvp/mvp_test.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow-go/integration/tests/lib" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/utils/rand" "github.com/onflow/flow-go/utils/unittest" ) @@ -92,9 +93,10 @@ func TestMVP_Bootstrap(t *testing.T) { flowNetwork.RemoveContainers() // pick 1 consensus node to restart with empty database and downloaded snapshot - con1 := flowNetwork.Identities(). - Filter(filter.HasRole(flow.RoleConsensus)). - Sample(1)[0] + cons := flowNetwork.Identities().Filter(filter.HasRole(flow.RoleConsensus)) + random, err := rand.Uintn(uint(len(cons))) + require.NoError(t, err) + con1 := cons[random] t.Log("@@ booting from non-root state on consensus node ", con1.NodeID) From 6f76e40641519d9253653fc6c6f6512df9a92558 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 10:16:45 -0600 Subject: [PATCH 011/169] Revert "Update golangci-lint" This reverts commit 026312d540e62693dfbe0abc2bc8918bec6ec086. --- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1827c2a86b7..486497efc9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.51 + version: v1.49 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index 8058a656f29..f6637edf0ae 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -36,7 +36,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.51 + version: v1.49 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 From cd908b836f479cb99f628327b3652fdd8163fc09 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 10:17:01 -0600 Subject: [PATCH 012/169] Revert "Go 1.20" This reverts commit 0e877570896f960829a0709557fb68f55cb654de. --- .github/workflows/bench.yml | 2 +- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- .github/workflows/test-monitor-flaky.yml | 2 +- .github/workflows/test-monitor-regular-skipped.yml | 2 +- .github/workflows/tools.yml | 2 +- cmd/Dockerfile | 4 ++-- cmd/testclient/go.mod | 2 +- crypto/Dockerfile | 2 +- crypto/go.mod | 2 +- go.mod | 2 +- insecure/go.mod | 2 +- integration/benchmark/cmd/manual/Dockerfile | 2 +- integration/go.mod | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ef5b88d7f55..e78d7a18c85 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -36,7 +36,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.20" + go-version: "1.19" cache: true - name: Build relic diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 9079fb06a98..eb28e840078 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: '1.19' - name: Checkout repo uses: actions/checkout@v2 - name: Build relic diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 486497efc9b..4f7c116436d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: - 'v[0-9]+.[0-9]+' env: - GO_VERSION: "1.20" + GO_VERSION: 1.19 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index f6637edf0ae..3a5b47e2c2f 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -5,7 +5,7 @@ on: branches: - '**/*flaky-test-debug*' env: - GO_VERSION: "1.20" + GO_VERSION: 1.19 #concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-flaky.yml b/.github/workflows/test-monitor-flaky.yml index e34642e6d8c..8a951583285 100644 --- a/.github/workflows/test-monitor-flaky.yml +++ b/.github/workflows/test-monitor-flaky.yml @@ -13,7 +13,7 @@ on: env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: test_results - GO_VERSION: "1.20" + GO_VERSION: 1.19 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-regular-skipped.yml b/.github/workflows/test-monitor-regular-skipped.yml index 9276b28db18..8eb48c1129e 100644 --- a/.github/workflows/test-monitor-regular-skipped.yml +++ b/.github/workflows/test-monitor-regular-skipped.yml @@ -15,7 +15,7 @@ env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: skipped_tests BIGQUERY_TABLE2: test_results - GO_VERSION: "1.20" + GO_VERSION: 1.19 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 852247cbed7..8a057d9dfb5 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -24,7 +24,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: '1.19' - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@v1 with: diff --git a/cmd/Dockerfile b/cmd/Dockerfile index a1c500ef760..473effbef9b 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -3,7 +3,7 @@ #################################### ## (1) Setup the build environment -FROM golang:1.20-bullseye AS build-setup +FROM golang:1.19-bullseye AS build-setup RUN apt-get update RUN apt-get -y install cmake zip @@ -67,7 +67,7 @@ RUN --mount=type=ssh \ RUN chmod a+x /app/app ## (4) Add the statically linked debug binary to a distroless image configured for debugging -FROM golang:1.20-bullseye as debug +FROM golang:1.19-bullseye as debug RUN go install github.com/go-delve/delve/cmd/dlv@latest diff --git a/cmd/testclient/go.mod b/cmd/testclient/go.mod index dbe66a78fb5..0a02e69ad42 100644 --- a/cmd/testclient/go.mod +++ b/cmd/testclient/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/cmd/testclient -go 1.20 +go 1.19 require ( github.com/onflow/flow-go-sdk v0.4.1 diff --git a/crypto/Dockerfile b/crypto/Dockerfile index d75e9543de4..37a0b373171 100644 --- a/crypto/Dockerfile +++ b/crypto/Dockerfile @@ -1,6 +1,6 @@ # gcr.io/dl-flow/golang-cmake -FROM golang:1.20-buster +FROM golang:1.19-buster RUN apt-get update RUN apt-get -y install cmake zip RUN go install github.com/axw/gocov/gocov@latest diff --git a/crypto/go.mod b/crypto/go.mod index 9895e1c35db..c7fe54f9ff5 100644 --- a/crypto/go.mod +++ b/crypto/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/crypto -go 1.20 +go 1.19 require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 diff --git a/go.mod b/go.mod index f44a19e3dcc..8c539911ad8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go -go 1.20 +go 1.19 require ( cloud.google.com/go/compute/metadata v0.2.1 diff --git a/insecure/go.mod b/insecure/go.mod index 66693d2fdd2..1c316e0a955 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/insecure -go 1.20 +go 1.19 require ( github.com/golang/protobuf v1.5.2 diff --git a/integration/benchmark/cmd/manual/Dockerfile b/integration/benchmark/cmd/manual/Dockerfile index 58f2b71d42b..1ad38985a43 100644 --- a/integration/benchmark/cmd/manual/Dockerfile +++ b/integration/benchmark/cmd/manual/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:experimental # NOTE: Must be run in the context of the repo's root directory -FROM golang:1.20-buster AS build-setup +FROM golang:1.19-buster AS build-setup RUN apt-get update RUN apt-get -y install cmake zip diff --git a/integration/go.mod b/integration/go.mod index f5c6315e7ff..b887da02851 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/integration -go 1.20 +go 1.19 require ( cloud.google.com/go/bigquery v1.43.0 From 12f6663ef9ff29f86848582c50e2857cec68e4de Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 10:55:25 -0600 Subject: [PATCH 013/169] more fixing --- crypto/random/rand_test.go | 3 +++ go.mod | 6 ++++-- go.sum | 1 + ledger/common/testutils/testutils.go | 15 ++++++++++++--- model/flow/identifier_test.go | 2 +- module/mempool/mock/back_data.go | 13 ++----------- 6 files changed, 23 insertions(+), 17 deletions(-) diff --git a/crypto/random/rand_test.go b/crypto/random/rand_test.go index 0fd1b6f1b24..1485e7e674d 100644 --- a/crypto/random/rand_test.go +++ b/crypto/random/rand_test.go @@ -310,6 +310,7 @@ func TestSamples(t *testing.T) { seed := make([]byte, Chacha20SeedLen) _, err := rand.Read(seed) + require.NoError(t, err) customizer := make([]byte, Chacha20CustomizerMaxLen) _, err = rand.Read(customizer) require.NoError(t, err) @@ -403,8 +404,10 @@ func TestStateRestore(t *testing.T) { // generate a seed seed := make([]byte, Chacha20SeedLen) _, err := rand.Read(seed) + require.NoError(t, err) customizer := make([]byte, Chacha20CustomizerMaxLen) _, err = rand.Read(customizer) + require.NoError(t, err) t.Logf("seed is %x, customizer is %x\n", seed, customizer) // create an rng diff --git a/go.mod b/go.mod index 8c539911ad8..f6808ae33cf 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,10 @@ require ( pgregory.net/rapid v0.4.7 ) -require github.com/slok/go-http-metrics v0.10.0 +require ( + github.com/slok/go-http-metrics v0.10.0 + gonum.org/v1/gonum v0.8.2 +) require ( cloud.google.com/go v0.105.0 // indirect @@ -267,7 +270,6 @@ require ( golang.org/x/oauth2 v0.3.0 // indirect golang.org/x/term v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - gonum.org/v1/gonum v0.8.2 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 96dd1dfe10b..998715a1424 100644 --- a/go.sum +++ b/go.sum @@ -1999,6 +1999,7 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= diff --git a/ledger/common/testutils/testutils.go b/ledger/common/testutils/testutils.go index 1e937cc94b1..d3961108de6 100644 --- a/ledger/common/testutils/testutils.go +++ b/ledger/common/testutils/testutils.go @@ -171,7 +171,10 @@ func RandomPayload(minByteSize int, maxByteSize int) *l.Payload { key := l.Key{KeyParts: []l.KeyPart{{Type: 0, Value: keydata}}} valueByteSize := minByteSize + rand.Intn(maxByteSize-minByteSize) valuedata := make([]byte, valueByteSize) - crand.Read(valuedata) + _, err := crand.Read(valuedata) + if err != nil { + panic("random generation failed") + } value := l.Value(valuedata) return l.NewPayload(key, value) } @@ -197,7 +200,10 @@ func RandomValues(n int, minByteSize, maxByteSize int) []l.Value { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } value := make([]byte, byteSize) - crand.Read(value) + _, err := rand.Read(value) + if err != nil { + panic("random generation failed") + } values = append(values, value) } return values @@ -219,7 +225,10 @@ func RandomUniqueKeys(n, m, minByteSize, maxByteSize int) []l.Key { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } keyPartData := make([]byte, byteSize) - crand.Read(keyPartData) + _, err := crand.Read(keyPartData) + if err != nil { + panic("random generation failed") + } keyParts = append(keyParts, l.NewKeyPart(uint16(j), keyPartData)) } key := l.NewKey(keyParts) diff --git a/model/flow/identifier_test.go b/model/flow/identifier_test.go index 901e9dfb777..7ac5dd3df89 100644 --- a/model/flow/identifier_test.go +++ b/model/flow/identifier_test.go @@ -134,7 +134,7 @@ func TestCIDConversion(t *testing.T) { // generate random CID data := make([]byte, 4) - _, err := rand.Read(data) + _, err = rand.Read(data) require.NoError(t, err) cid = blocks.NewBlock(data).Cid() diff --git a/module/mempool/mock/back_data.go b/module/mempool/mock/back_data.go index d12e05bbb8c..68661aa9c23 100644 --- a/module/mempool/mock/back_data.go +++ b/module/mempool/mock/back_data.go @@ -96,17 +96,8 @@ func (_m *BackData) ByID(entityID flow.Identifier) (flow.Entity, bool) { } // Clear provides a mock function with given fields: -func (_m *BackData) Clear() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *BackData) Clear() { + _m.Called() } // Entities provides a mock function with given fields: From 437736996ef7c38e840e6bff84f81ef87ec8b6c1 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 12:02:10 -0600 Subject: [PATCH 014/169] log error in pool random ejection and fallback to LRU --- module/mempool/herocache/backdata/cache.go | 4 +-- .../herocache/backdata/heropool/pool.go | 27 ++++++++++++------- .../herocache/backdata/heropool/pool_test.go | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/module/mempool/herocache/backdata/cache.go b/module/mempool/herocache/backdata/cache.go index f684adc11f9..d7167917711 100644 --- a/module/mempool/herocache/backdata/cache.go +++ b/module/mempool/herocache/backdata/cache.go @@ -127,7 +127,7 @@ func NewCache(sizeLimit uint32, sizeLimit: sizeLimit, buckets: make([]slotBucket, bucketNum), ejectionMode: ejectionMode, - entities: heropool.NewHeroPool(sizeLimit, ejectionMode), + entities: heropool.NewHeroPool(sizeLimit, ejectionMode, logger), availableSlotHistogram: make([]uint64, slotsPerBucket+1), // +1 is to account for empty buckets as well. interactionCounter: atomic.NewUint64(0), lastTelemetryDump: atomic.NewInt64(0), @@ -252,7 +252,7 @@ func (c *Cache) Clear() { defer c.logTelemetry() c.buckets = make([]slotBucket, c.bucketNum) - c.entities = heropool.NewHeroPool(c.sizeLimit, c.ejectionMode) + c.entities = heropool.NewHeroPool(c.sizeLimit, c.ejectionMode, c.logger) c.availableSlotHistogram = make([]uint64, slotsPerBucket+1) c.interactionCounter = atomic.NewUint64(0) c.lastTelemetryDump = atomic.NewInt64(0) diff --git a/module/mempool/herocache/backdata/heropool/pool.go b/module/mempool/herocache/backdata/heropool/pool.go index 695d370e47d..736588a936a 100644 --- a/module/mempool/herocache/backdata/heropool/pool.go +++ b/module/mempool/herocache/backdata/heropool/pool.go @@ -3,6 +3,7 @@ package heropool import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/rand" + "github.com/rs/zerolog" ) type EjectionMode string @@ -46,6 +47,7 @@ func (p PoolEntity) Entity() flow.Entity { } type Pool struct { + logger zerolog.Logger size uint32 free state // keeps track of free slots. used state // keeps track of allocated slots to cachedEntities. @@ -53,7 +55,7 @@ type Pool struct { ejectionMode EjectionMode } -func NewHeroPool(sizeLimit uint32, ejectionMode EjectionMode) *Pool { +func NewHeroPool(sizeLimit uint32, ejectionMode EjectionMode, logger zerolog.Logger) *Pool { l := &Pool{ free: state{ head: poolIndex{index: 0}, @@ -65,6 +67,7 @@ func NewHeroPool(sizeLimit uint32, ejectionMode EjectionMode) *Pool { }, poolEntities: make([]poolEntity, sizeLimit), ejectionMode: ejectionMode, + logger: logger, } l.initFreeEntities() @@ -159,28 +162,34 @@ func (p Pool) Head() (flow.Entity, bool) { // Ejection happens if there is no available slot, and there is an ejection mode set. // If an ejection occurred, ejectedEntity holds the ejected entity. func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectedEntity flow.Entity) { + lruEject := func() (EIndex, bool, flow.Entity) { + // LRU ejection + // the used head is the oldest entity, so we turn the used head to a free head here. + invalidatedEntity := p.invalidateUsedHead() + return p.claimFreeHead(), true, invalidatedEntity + } + if p.free.head.isUndefined() { // the free list is empty, so we are out of space, and we need to eject. switch p.ejectionMode { case NoEjection: // pool is set for no ejection, hence, no slice index is selected, abort immediately. return 0, false, nil - case LRUEjection: - // LRU ejection - // the used head is the oldest entity, so we turn the used head to a free head here. - invalidatedEntity := p.invalidateUsedHead() - return p.claimFreeHead(), true, invalidatedEntity case RandomEjection: // we only eject randomly when the pool is full and random ejection is on. random, err := rand.Uint32n(p.size) if err != nil { - // TODO: to check with Yahya - // randomness failed and no ejection has happened - return 0, false, nil + p.logger.Warn().Err(err). + Msg("hero pool random ejection failed - falling back to LRU ejection") + // fall back to LRU ejection only for this instance + return lruEject() } randomIndex := EIndex(random) invalidatedEntity := p.invalidateEntityAtIndex(randomIndex) return p.claimFreeHead(), true, invalidatedEntity + case LRUEjection: + // LRU ejection + return lruEject() } } diff --git a/module/mempool/herocache/backdata/heropool/pool_test.go b/module/mempool/herocache/backdata/heropool/pool_test.go index 8f3a83db681..9b8b15bea3a 100644 --- a/module/mempool/herocache/backdata/heropool/pool_test.go +++ b/module/mempool/herocache/backdata/heropool/pool_test.go @@ -645,7 +645,7 @@ func withTestScenario(t *testing.T, ejectionMode EjectionMode, helpers ...func(*testing.T, *Pool, []*unittest.MockEntity)) { - pool := NewHeroPool(limit, ejectionMode) + pool := NewHeroPool(limit, ejectionMode, unittest.Logger()) // head on underlying linked-list value should be uninitialized require.True(t, pool.used.head.isUndefined()) From 4919ba3cecda4b02689592f281b64cd4759e0dc8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 12:22:17 -0600 Subject: [PATCH 015/169] fix import order --- module/mempool/herocache/backdata/heropool/pool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/mempool/herocache/backdata/heropool/pool.go b/module/mempool/herocache/backdata/heropool/pool.go index 736588a936a..a0a5752b2cd 100644 --- a/module/mempool/herocache/backdata/heropool/pool.go +++ b/module/mempool/herocache/backdata/heropool/pool.go @@ -1,9 +1,10 @@ package heropool import ( + "github.com/rs/zerolog" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/rand" - "github.com/rs/zerolog" ) type EjectionMode string From 07dce629e654d567f5d920985d69a01f150488e7 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 16 Mar 2023 19:04:29 -0600 Subject: [PATCH 016/169] fix a new issue after merging master --- engine/common/follower/pending_tree/pending_tree_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine/common/follower/pending_tree/pending_tree_test.go b/engine/common/follower/pending_tree/pending_tree_test.go index 9e9484294bd..a24512093b7 100644 --- a/engine/common/follower/pending_tree/pending_tree_test.go +++ b/engine/common/follower/pending_tree/pending_tree_test.go @@ -3,7 +3,6 @@ package pending_tree import ( "math/rand" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,7 +26,6 @@ type PendingTreeSuite struct { } func (s *PendingTreeSuite) SetupTest() { - rand.Seed(time.Now().UnixNano()) s.finalized = unittest.BlockHeaderFixture() s.pendingTree = NewPendingTree(s.finalized) } From 77d83f0d1087d49fe56a3c6d3bc914527b3b89a5 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 17 Mar 2023 13:30:44 -0600 Subject: [PATCH 017/169] improve unsafeRandom test for randomness and add determinicity test --- .../unsafe_random_generator_test.go | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/unsafe_random_generator_test.go index 5e41cb3e215..2ad211c19c5 100644 --- a/fvm/environment/unsafe_random_generator_test.go +++ b/fvm/environment/unsafe_random_generator_test.go @@ -1,36 +1,64 @@ package environment_test import ( + "fmt" + "math" + mrand "math/rand" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gonum.org/v1/gonum/stat" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) func TestUnsafeRandomGenerator(t *testing.T) { - t.Run("UnsafeRandom doesnt re-seed the random", func(t *testing.T) { - bh := &flow.Header{} - + // basic randomness test to check outputs are "uniformly" spread over the + // output space + t.Run("randomness test", func(t *testing.T) { + bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) urg := environment.NewUnsafeRandomGenerator(tracing.NewTracerSpan(), bh) - // 10 random numbers. extremely unlikely to get the same number all the time and just fail the test by chance - N := 10 - - numbers := make([]uint64, N) + sampleSize := 80000 + tolerance := 0.05 + n := 10 + mrand.Intn(100) + distribution := make([]float64, n) - for i := 0; i < N; i++ { - u, err := urg.UnsafeRandom() + // partition all outputs into `n` classes and compute the distribution + // over the partition. Each class is `classWidth`-big + classWidth := math.MaxUint64 / uint64(n) + // populate the distribution + for i := 0; i < sampleSize; i++ { + r, err := urg.UnsafeRandom() require.NoError(t, err) - numbers[i] = u + distribution[r/classWidth] += 1.0 } + stdev := stat.StdDev(distribution, nil) + mean := stat.Mean(distribution, nil) + assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed. stdev %v, mean %v", stdev, mean)) + }) - allEqual := true - for i := 1; i < N; i++ { - allEqual = allEqual && numbers[i] == numbers[0] + // tests that unsafeRandom is PRG based and hence has deterministic outputs. + t.Run("PRG-based UnsafeRandom", func(t *testing.T) { + bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) + N := 100 + getRandoms := func() []uint64 { + // seed the RG with the same block header + urg := environment.NewUnsafeRandomGenerator(tracing.NewTracerSpan(), bh) + numbers := make([]uint64, N) + for i := 0; i < N; i++ { + u, err := urg.UnsafeRandom() + require.NoError(t, err) + numbers[i] = u + } + return numbers } - require.True(t, !allEqual) + r1 := getRandoms() + r2 := getRandoms() + require.Equal(t, r1, r2) }) } From 910c04c8a073839cd6b7f0792e663ef39319a4dc Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 21 Apr 2023 19:56:22 -0600 Subject: [PATCH 018/169] remove added spaces --- cmd/execution_builder.go | 1 - cmd/observer/node_builder/observer_builder.go | 1 - cmd/scaffold.go | 1 - .../hotstuff/votecollector/combined_vote_processor_v2_test.go | 2 -- .../hotstuff/votecollector/combined_vote_processor_v3_test.go | 2 -- engine/access/rpc/backend/backend_test.go | 1 - engine/collection/compliance/core_test.go | 2 -- engine/collection/message_hub/message_hub_test.go | 2 -- engine/collection/synchronization/engine_test.go | 2 -- engine/common/synchronization/engine_test.go | 2 -- engine/consensus/compliance/core_test.go | 2 -- engine/consensus/message_hub/message_hub_test.go | 2 -- module/builder/collection/builder_test.go | 2 -- module/chunks/chunkVerifier_test.go | 2 -- module/finalizer/collection/finalizer_test.go | 3 --- state/cluster/badger/mutator_test.go | 2 -- state/cluster/badger/snapshot_test.go | 2 -- 17 files changed, 31 deletions(-) diff --git a/cmd/execution_builder.go b/cmd/execution_builder.go index b490bf26019..e3f7ccd6676 100644 --- a/cmd/execution_builder.go +++ b/cmd/execution_builder.go @@ -497,7 +497,6 @@ func (exeNode *ExecutionNode) LoadProviderEngine( chunkDataPackRequestQueueMetrics = metrics.ChunkDataPackRequestQueueMetricsFactory(node.MetricsRegisterer) } chdpReqQueue := queue.NewHeroStore(exeNode.exeConf.chunkDataPackRequestsCacheSize, node.Logger, chunkDataPackRequestQueueMetrics) - exeNode.providerEngine, err = exeprovider.New( node.Logger, node.Tracer, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 59507048242..472ae398260 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -973,7 +973,6 @@ func (builder *ObserverServiceBuilder) enqueuePublicNetworkInit() { if builder.HeroCacheMetricsEnable { heroCacheCollector = metrics.NetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer) } - receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize, builder.Logger, heroCacheCollector) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index c0f9fc58213..c7e0a81f401 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -1764,7 +1764,6 @@ func (fnb *FlowNodeBuilder) Build() (Node, error) { } func (fnb *FlowNodeBuilder) onStart() error { - // init nodeinfo by reading the private bootstrap file if not already set if fnb.NodeID == flow.ZeroID { if err := fnb.initNodeInfo(); err != nil { diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go index 47403f78a82..4b40acb9b8b 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v2_test.go @@ -596,7 +596,6 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -744,7 +743,6 @@ func TestCombinedVoteProcessorV2_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) diff --git a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go index 6343887d94c..831a68e1650 100644 --- a/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go +++ b/consensus/hotstuff/votecollector/combined_vote_processor_v3_test.go @@ -646,7 +646,6 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCCorrectness(testifyT *testing } // shuffle votes in random order - rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) @@ -879,7 +878,6 @@ func TestCombinedVoteProcessorV3_PropertyCreatingQCLiveness(testifyT *testing.T) } // shuffle votes in random order - rand.Shuffle(len(votes), func(i, j int) { votes[i], votes[j] = votes[j], votes[i] }) diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index f29f895749c..de6b8c16090 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -55,7 +55,6 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - suite.log = zerolog.New(zerolog.NewConsoleWriter()) suite.state = new(protocol.State) suite.snapshot = new(protocol.Snapshot) diff --git a/engine/collection/compliance/core_test.go b/engine/collection/compliance/core_test.go index b6ecd85b944..1a7ab5eff51 100644 --- a/engine/collection/compliance/core_test.go +++ b/engine/collection/compliance/core_test.go @@ -63,8 +63,6 @@ type CommonSuite struct { } func (cs *CommonSuite) SetupTest() { - // seed the RNG - block := unittest.ClusterBlockFixture() cs.head = &block diff --git a/engine/collection/message_hub/message_hub_test.go b/engine/collection/message_hub/message_hub_test.go index 04cf80eb025..7e60e4d7877 100644 --- a/engine/collection/message_hub/message_hub_test.go +++ b/engine/collection/message_hub/message_hub_test.go @@ -67,8 +67,6 @@ type MessageHubSuite struct { } func (s *MessageHubSuite) SetupTest() { - // seed the RNG - // initialize the paramaters s.cluster = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection), diff --git a/engine/collection/synchronization/engine_test.go b/engine/collection/synchronization/engine_test.go index 3498b9292e7..a637a9eedec 100644 --- a/engine/collection/synchronization/engine_test.go +++ b/engine/collection/synchronization/engine_test.go @@ -57,8 +57,6 @@ type SyncSuite struct { } func (ss *SyncSuite) SetupTest() { - // seed the RNG - // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleCollection)) ss.myID = ss.participants[0].NodeID diff --git a/engine/common/synchronization/engine_test.go b/engine/common/synchronization/engine_test.go index 84264e382cf..e4ac030c35b 100644 --- a/engine/common/synchronization/engine_test.go +++ b/engine/common/synchronization/engine_test.go @@ -58,8 +58,6 @@ type SyncSuite struct { } func (ss *SyncSuite) SetupTest() { - // seed the RNG - // generate own ID ss.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus)) keys := unittest.NetworkingKeys(len(ss.participants)) diff --git a/engine/consensus/compliance/core_test.go b/engine/consensus/compliance/core_test.go index b310a13c270..b2f2de03aa6 100644 --- a/engine/consensus/compliance/core_test.go +++ b/engine/consensus/compliance/core_test.go @@ -78,8 +78,6 @@ type CommonSuite struct { } func (cs *CommonSuite) SetupTest() { - // seed the RNG - // initialize the paramaters cs.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus), diff --git a/engine/consensus/message_hub/message_hub_test.go b/engine/consensus/message_hub/message_hub_test.go index cae615a23f3..a68ce9eeb7a 100644 --- a/engine/consensus/message_hub/message_hub_test.go +++ b/engine/consensus/message_hub/message_hub_test.go @@ -64,8 +64,6 @@ type MessageHubSuite struct { } func (s *MessageHubSuite) SetupTest() { - // seed the RNG - // initialize the paramaters s.participants = unittest.IdentityListFixture(3, unittest.WithRole(flow.RoleConsensus), diff --git a/module/builder/collection/builder_test.go b/module/builder/collection/builder_test.go index 382992f5bd1..31c7d4ebd8e 100644 --- a/module/builder/collection/builder_test.go +++ b/module/builder/collection/builder_test.go @@ -61,8 +61,6 @@ type BuilderSuite struct { func (suite *BuilderSuite) SetupTest() { var err error - // seed the RNG - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID diff --git a/module/chunks/chunkVerifier_test.go b/module/chunks/chunkVerifier_test.go index 85ce136013c..daa66c158e6 100644 --- a/module/chunks/chunkVerifier_test.go +++ b/module/chunks/chunkVerifier_test.go @@ -65,8 +65,6 @@ type ChunkVerifierTestSuite struct { // Make sure variables are set properly // SetupTest is executed prior to each individual test in this test suite func (s *ChunkVerifierTestSuite) SetupSuite() { - // seed the RNG - vm := new(vmMock) systemOkVm := new(vmSystemOkMock) systemBadVm := new(vmSystemBadMock) diff --git a/module/finalizer/collection/finalizer_test.go b/module/finalizer/collection/finalizer_test.go index c3c837f8738..c31a7193c42 100644 --- a/module/finalizer/collection/finalizer_test.go +++ b/module/finalizer/collection/finalizer_test.go @@ -23,9 +23,6 @@ import ( func TestFinalizer(t *testing.T) { unittest.RunWithBadgerDB(t, func(db *badger.DB) { - - // seed the RNG - // reference block on the main consensus chain refBlock := unittest.BlockHeaderFixture() // genesis block for the cluster chain diff --git a/state/cluster/badger/mutator_test.go b/state/cluster/badger/mutator_test.go index 507f46d4c3b..88336d1c531 100644 --- a/state/cluster/badger/mutator_test.go +++ b/state/cluster/badger/mutator_test.go @@ -51,8 +51,6 @@ type MutatorSuite struct { func (suite *MutatorSuite) SetupTest() { var err error - // seed the RNG - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID diff --git a/state/cluster/badger/snapshot_test.go b/state/cluster/badger/snapshot_test.go index eb71fa64133..865b36659aa 100644 --- a/state/cluster/badger/snapshot_test.go +++ b/state/cluster/badger/snapshot_test.go @@ -41,8 +41,6 @@ type SnapshotSuite struct { func (suite *SnapshotSuite) SetupTest() { var err error - // seed the RNG - suite.genesis = model.Genesis() suite.chainID = suite.genesis.Header.ChainID From 0bebc15bd8e9128fdf7665b47cf6ba51858e0b30 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 21 Apr 2023 22:26:42 -0600 Subject: [PATCH 019/169] minor cleanups --- consensus/integration/nodes_test.go | 3 --- engine/common/rpc/convert/convert_test.go | 2 +- engine/execution/provider/engine_test.go | 2 ++ engine/protocol/api_test.go | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/consensus/integration/nodes_test.go b/consensus/integration/nodes_test.go index dd6161e20a2..837c1301fda 100644 --- a/consensus/integration/nodes_test.go +++ b/consensus/integration/nodes_test.go @@ -436,9 +436,6 @@ func createNode( notifier.AddConsumer(counterConsumer) notifier.AddConsumer(logConsumer) - cleaner := &storagemock.Cleaner{} - cleaner.On("RunGC").Return(nil) - require.Equal(t, participant.nodeInfo.NodeID, localID) privateKeys, err := participant.nodeInfo.PrivateKeys() require.NoError(t, err) diff --git a/engine/common/rpc/convert/convert_test.go b/engine/common/rpc/convert/convert_test.go index ec0c3dc930c..a98f828d0f6 100644 --- a/engine/common/rpc/convert/convert_test.go +++ b/engine/common/rpc/convert/convert_test.go @@ -2,7 +2,7 @@ package convert_test import ( "bytes" - "crypto/rand" + "math/rand" "testing" "github.com/stretchr/testify/assert" diff --git a/engine/execution/provider/engine_test.go b/engine/execution/provider/engine_test.go index 9346bfe02df..1411061b123 100644 --- a/engine/execution/provider/engine_test.go +++ b/engine/execution/provider/engine_test.go @@ -98,6 +98,7 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) + e, err := New( unittest.Logger(), trace.NewNoopTracer(), @@ -156,6 +157,7 @@ func TestProviderEngine_onChunkDataRequest(t *testing.T) { net.On("Register", channels.PushReceipts, mock.Anything).Return(&mocknetwork.Conduit{}, nil) net.On("Register", channels.ProvideChunks, mock.Anything).Return(chunkConduit, nil) requestQueue := queue.NewHeroStore(10, unittest.Logger(), metrics.NewNoopCollector()) + e, err := New( unittest.Logger(), trace.NewNoopTracer(), diff --git a/engine/protocol/api_test.go b/engine/protocol/api_test.go index 4025f612513..f5e029181ed 100644 --- a/engine/protocol/api_test.go +++ b/engine/protocol/api_test.go @@ -35,7 +35,6 @@ func TestHandler(t *testing.T) { } func (suite *Suite) SetupTest() { - suite.snapshot = new(protocol.Snapshot) suite.state = new(protocol.State) From cb1db830da9e313c8bb3bebd8667d64a007e6021 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 15 May 2023 21:00:56 -0600 Subject: [PATCH 020/169] fix merging bug --- engine/common/synchronization/engine.go | 6 ------ engine/testutil/nodes.go | 7 ++----- model/encodable/keys_test.go | 3 ++- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/engine/common/synchronization/engine.go b/engine/common/synchronization/engine.go index 87a86390074..03b1222554a 100644 --- a/engine/common/synchronization/engine.go +++ b/engine/common/synchronization/engine.go @@ -398,12 +398,6 @@ func (e *Engine) pollHeight() { func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsync.Range, batches []chainsync.Batch) { var errs *multierror.Error - nonce, err := rand.Uint64() - if err != nil { - e.log.Error().Err(err).Msg("nonce generation failed") - return - } - for _, ran := range ranges { nonce, err := rand.Uint64() if err != nil { diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index 32aefad4238..91c9753c461 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -299,14 +299,13 @@ func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ro return coll, err } - store := queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()) providerEngine, err := provider.New( node.Log, node.Metrics, node.Net, node.Me, node.State, - store, + queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()), uint(1000), channels.ProvideCollections, selector, @@ -611,8 +610,6 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit ) require.NoError(t, err) - store := queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()) - pusherEngine, err := executionprovider.New( node.Log, node.Tracer, @@ -621,7 +618,7 @@ func ExecutionNode(t *testing.T, hub *stub.Hub, identity *flow.Identity, identit execState, metricsCollector, checkAuthorizedAtBlock, - store, + queue.NewHeroStore(uint32(1000), unittest.Logger(), metrics.NewNoopCollector()), executionprovider.DefaultChunkDataPackRequestWorker, executionprovider.DefaultChunkDataPackQueryTimeout, executionprovider.DefaultChunkDataPackDeliveryTimeout, diff --git a/model/encodable/keys_test.go b/model/encodable/keys_test.go index 5b396fb6f99..ccdf63cd044 100644 --- a/model/encodable/keys_test.go +++ b/model/encodable/keys_test.go @@ -252,7 +252,8 @@ func TestEncodableRandomBeaconPrivKeyMsgPack(t *testing.T) { func generateRandomSeed(t *testing.T) []byte { seed := make([]byte, 48) - _, err := rand.Read(seed) + n, err := rand.Read(seed) require.Nil(t, err) + require.Equal(t, n, 48) return seed } From 5aaa8ca8c993045017047b1c218df10af1e40443 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Wed, 31 May 2023 16:08:37 +0300 Subject: [PATCH 021/169] Separated grpc server creation into grpc server package, refactored rpc and state_stream engines, refactored access_node_builder --- .../node_builder/access_node_builder.go | 54 +++++- engine/access/rpc/engine.go | 130 +------------ engine/access/rpc/engine_builder.go | 12 +- engine/access/state_stream/engine.go | 65 +------ module/grpcserver/server.go | 176 ++++++++++++++++++ 5 files changed, 245 insertions(+), 192 deletions(-) create mode 100644 module/grpcserver/server.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 9373cbf7e24..bf2a95b7ff4 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -50,6 +50,7 @@ import ( "github.com/onflow/flow-go/module/chainsync" "github.com/onflow/flow-go/module/executiondatasync/execution_data" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/mempool/stdmap" "github.com/onflow/flow-go/module/metrics" @@ -238,6 +239,10 @@ type FlowAccessNodeBuilder struct { FollowerEng *followereng.ComplianceEngine SyncEng *synceng.Engine StateStreamEng *state_stream.Engine + + // grpc server builders + secureGrpcServer *grpcserver.GrpcServerBuilder + unsecureGrpcServer *grpcserver.GrpcServerBuilder } func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { @@ -579,9 +584,8 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN node.Storage.Results, node.RootChainID, builder.executionDataConfig.InitialBlockHeight, - builder.apiRatelimits, - builder.apiBurstlimits, heroCacheCollector, + builder.unsecureGrpcServer, ) if err != nil { return nil, fmt.Errorf("could not create state stream engine: %w", err) @@ -950,6 +954,30 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.rpcConf.TransportCredentials = credentials.NewTLS(tlsConfig) return nil }). + Module("creating grpc servers", func(node *cmd.NodeConfig) error { + secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) + + builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + secureGrpcServerConfig, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits) + + unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + ) + + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + unsecureGrpcServerConfig, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits) + + return nil + }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { engineBuilder, err := rpc.NewBuilder( node.Logger, @@ -970,9 +998,9 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.executionGRPCPort, builder.retryEnabled, builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits, builder.Me, + builder.secureGrpcServer, + builder.unsecureGrpcServer, ) if err != nil { return nil, err @@ -1062,6 +1090,24 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.BuildExecutionDataRequester() } + builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + secureGrpcServer, err := builder.secureGrpcServer.Build() + if err != nil { + return nil, err + } + + return secureGrpcServer, nil + }) + + builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + unsecureGrpcServer, err := builder.unsecureGrpcServer.Build() + if err != nil { + return nil, err + } + + return unsecureGrpcServer, nil + }) + builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { ping, err := pingeng.New( node.Logger, diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 75d5e8fc543..0979dbffed7 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -9,21 +9,19 @@ import ( "sync" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" lru "github.com/hashicorp/golang-lru" accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" - "google.golang.org/grpc" "google.golang.org/grpc/credentials" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc/backend" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/events" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" @@ -46,7 +44,7 @@ type Config struct { ConnectionPoolSize uint // size of the cache for storing collection and execution connections MaxHeightRange uint // max size of height range requests PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs - FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node node ID can be chosen from the PreferredExecutionNodeIDs + FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node ID can be chosen from the PreferredExecutionNodeIDs ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node } @@ -62,18 +60,16 @@ type Engine struct { log zerolog.Logger restCollector module.RestMetrics - backend *backend.Backend // the gRPC service implementation - unsecureGrpcServer *grpc.Server // the unsecure gRPC server - secureGrpcServer *grpc.Server // the secure gRPC server + backend *backend.Backend // the gRPC service implementation + unsecureGrpcServer *grpcserver.GrpcServerBuilder // the unsecure gRPC server + secureGrpcServer *grpcserver.GrpcServerBuilder // the secure gRPC server httpServer *http.Server restServer *http.Server config Config chain flow.Chain - addrLock sync.RWMutex - unsecureGrpcAddress net.Addr - secureGrpcAddress net.Addr - restAPIAddress net.Addr + addrLock sync.RWMutex + restAPIAddress net.Addr } // NewBuilder returns a new RPC engine builder. @@ -95,48 +91,15 @@ func NewBuilder(log zerolog.Logger, executionGRPCPort uint, retryEnabled bool, rpcMetricsEnabled bool, - apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 - apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 me module.Local, + secureGrpcServer *grpcserver.GrpcServerBuilder, + unsecureGrpcServer *grpcserver.GrpcServerBuilder, ) (*RPCEngineBuilder, error) { log = log.With().Str("engine", "rpc").Logger() - // create a GRPC server to serve GRPC clients - grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), - grpc.MaxSendMsgSize(int(config.MaxMsgSize)), - } - - var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors - // if rpc metrics is enabled, first create the grpc metrics interceptor - if rpcMetricsEnabled { - interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) - } - - if len(apiRatelimits) > 0 { - // create a rate limit interceptor - rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRatelimits, apiBurstLimits).UnaryServerInterceptor - // append the rate limit interceptor to the list of interceptors - interceptors = append(interceptors, rateLimitInterceptor) - } - - // add the logging interceptor, ensure it is innermost wrapper - interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) - - // create a chained unary interceptor - chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) - grpcOpts = append(grpcOpts, chainedInterceptors) - - // create an unsecured grpc server - unsecureGrpcServer := grpc.NewServer(grpcOpts...) - - // create a secure server by using the secure grpc credentials that are passed in as part of config - grpcOpts = append(grpcOpts, grpc.Creds(config.TransportCredentials)) - secureGrpcServer := grpc.NewServer(grpcOpts...) - // wrap the unsecured server with an HTTP proxy server to serve HTTP clients - httpServer := newHTTPProxyServer(unsecureGrpcServer) + httpServer := newHTTPProxyServer(unsecureGrpcServer.Server()) var cache *lru.Cache cacheSize := config.ConnectionPoolSize @@ -216,8 +179,6 @@ func NewBuilder(log zerolog.Logger, eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). - AddWorker(eng.serveUnsecureGRPCWorker). - AddWorker(eng.serveSecureGRPCWorker). AddWorker(eng.serveGRPCWebProxyWorker). AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). @@ -246,8 +207,6 @@ func (e *Engine) shutdown() { // use unbounded context, rely on shutdown logic to have timeout ctx := context.Background() - e.unsecureGrpcServer.GracefulStop() - e.secureGrpcServer.GracefulStop() err := e.httpServer.Shutdown(ctx) if err != nil { e.log.Error().Err(err).Msg("error stopping http server") @@ -274,22 +233,6 @@ func (e *Engine) notifyBackendOnBlockFinalized(_ *model.Block) error { return nil } -// UnsecureGRPCAddress returns the listen address of the unsecure GRPC server. -// Guaranteed to be non-nil after Engine.Ready is closed. -func (e *Engine) UnsecureGRPCAddress() net.Addr { - e.addrLock.RLock() - defer e.addrLock.RUnlock() - return e.unsecureGrpcAddress -} - -// SecureGRPCAddress returns the listen address of the secure GRPC server. -// Guaranteed to be non-nil after Engine.Ready is closed. -func (e *Engine) SecureGRPCAddress() net.Addr { - e.addrLock.RLock() - defer e.addrLock.RUnlock() - return e.secureGrpcAddress -} - // RestApiAddress returns the listen address of the REST API server. // Guaranteed to be non-nil after Engine.Ready is closed. func (e *Engine) RestApiAddress() net.Addr { @@ -298,59 +241,6 @@ func (e *Engine) RestApiAddress() net.Addr { return e.restAPIAddress } -// serveUnsecureGRPCWorker is a worker routine which starts the unsecure gRPC server. -// The ready callback is called after the server address is bound and set. -func (e *Engine) serveUnsecureGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - e.log.Info().Str("grpc_address", e.config.UnsecureGRPCListenAddr).Msg("starting grpc server on address") - - l, err := net.Listen("tcp", e.config.UnsecureGRPCListenAddr) - if err != nil { - e.log.Err(err).Msg("failed to start the grpc server") - ctx.Throw(err) - return - } - - // save the actual address on which we are listening (may be different from e.config.UnsecureGRPCListenAddr if not port - // was specified) - e.addrLock.Lock() - e.unsecureGrpcAddress = l.Addr() - e.addrLock.Unlock() - e.log.Debug().Str("unsecure_grpc_address", e.unsecureGrpcAddress.String()).Msg("listening on port") - ready() - - err = e.unsecureGrpcServer.Serve(l) // blocking call - if err != nil { - e.log.Err(err).Msg("fatal error in unsecure grpc server") - ctx.Throw(err) - } -} - -// serveSecureGRPCWorker is a worker routine which starts the secure gRPC server. -// The ready callback is called after the server address is bound and set. -func (e *Engine) serveSecureGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - e.log.Info().Str("secure_grpc_address", e.config.SecureGRPCListenAddr).Msg("starting grpc server on address") - - l, err := net.Listen("tcp", e.config.SecureGRPCListenAddr) - if err != nil { - e.log.Err(err).Msg("failed to start the grpc server") - ctx.Throw(err) - return - } - - e.addrLock.Lock() - e.secureGrpcAddress = l.Addr() - e.addrLock.Unlock() - - e.log.Debug().Str("secure_grpc_address", e.secureGrpcAddress.String()).Msg("listening on port") - ready() - - err = e.secureGrpcServer.Serve(l) // blocking call - if err != nil { - e.log.Err(err).Msg("fatal error in secure grpc server") - ctx.Throw(err) - } -} - // serveGRPCWebProxyWorker is a worker routine which starts the gRPC web proxy server. func (e *Engine) serveGRPCWebProxyWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { log := e.log.With().Str("http_proxy_address", e.config.HTTPListenAddr).Logger() diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index a4694547b03..e3e09495400 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -68,11 +68,11 @@ func (builder *RPCEngineBuilder) WithNewHandler(handler accessproto.AccessAPISer func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { // Register legacy gRPC handlers for backwards compatibility, to be removed at a later date legacyaccessproto.RegisterAccessAPIServer( - builder.unsecureGrpcServer, + builder.unsecureGrpcServer.Server(), legacyaccess.NewHandler(builder.backend, builder.chain), ) legacyaccessproto.RegisterAccessAPIServer( - builder.secureGrpcServer, + builder.secureGrpcServer.Server(), legacyaccess.NewHandler(builder.backend, builder.chain), ) return builder @@ -83,8 +83,8 @@ func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { func (builder *RPCEngineBuilder) WithMetrics() *RPCEngineBuilder { // Not interested in legacy metrics, so initialize here grpc_prometheus.EnableHandlingTimeHistogram() - grpc_prometheus.Register(builder.unsecureGrpcServer) - grpc_prometheus.Register(builder.secureGrpcServer) + grpc_prometheus.Register(builder.unsecureGrpcServer.Server()) + grpc_prometheus.Register(builder.secureGrpcServer.Server()) return builder } @@ -100,7 +100,7 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) } } - accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer, handler) - accessproto.RegisterAccessAPIServer(builder.secureGrpcServer, handler) + accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer.Server(), handler) + accessproto.RegisterAccessAPIServer(builder.secureGrpcServer.Server(), handler) return builder.Engine, nil } diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index a3687065c26..6fbed00695f 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -2,21 +2,18 @@ package state_stream import ( "fmt" - "net" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" access "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/rs/zerolog" "google.golang.org/grpc" "github.com/onflow/flow-go/engine" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/executiondatasync/execution_data" - "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/mempool/herocache" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" @@ -69,8 +66,6 @@ type Engine struct { execDataBroadcaster *engine.Broadcaster execDataCache *herocache.BlockExecutionData - - stateStreamGrpcAddress net.Addr } // NewEng returns a new ingress server. @@ -84,40 +79,11 @@ func NewEng( results storage.ExecutionResults, chainID flow.ChainID, initialBlockHeight uint64, - apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the gRPC API e.g. Ping->100, GetExecutionDataByBlockID->300 - apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the gRPC API e.g. Ping->50, GetExecutionDataByBlockID->10 heroCacheMetrics module.HeroCacheMetrics, + server *grpcserver.GrpcServerBuilder, ) (*Engine, error) { logger := log.With().Str("engine", "state_stream_rpc").Logger() - // create a GRPC server to serve GRPC clients - grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(int(config.MaxExecutionDataMsgSize)), - grpc.MaxSendMsgSize(int(config.MaxExecutionDataMsgSize)), - } - - var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors - // if rpc metrics is enabled, add the grpc metrics interceptor as a server option - if config.RpcMetricsEnabled { - interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) - } - - if len(apiRatelimits) > 0 { - // create a rate limit interceptor - rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRatelimits, apiBurstLimits).UnaryServerInterceptor - // append the rate limit interceptor to the list of interceptors - interceptors = append(interceptors, rateLimitInterceptor) - } - - // add the logging interceptor, ensure it is innermost wrapper - interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) - - // create a chained unary interceptor - chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) - grpcOpts = append(grpcOpts, chainedInterceptors) - - server := grpc.NewServer(grpcOpts...) - execDataCache := herocache.NewBlockExecutionData(config.ExecutionDataCacheSize, logger, heroCacheMetrics) broadcaster := engine.NewBroadcaster() @@ -130,7 +96,7 @@ func NewEng( e := &Engine{ log: logger, backend: backend, - server: server, + server: server.Server(), chain: chainID.Chain(), config: config, handler: NewHandler(backend, chainID.Chain(), config.EventFilterConfig, config.MaxGlobalStreams), @@ -139,7 +105,6 @@ func NewEng( } e.ComponentManager = component.NewComponentManagerBuilder(). - AddWorker(e.serve). Build() access.RegisterExecutionDataAPIServer(e.server, e.handler) @@ -159,27 +124,3 @@ func (e *Engine) OnExecutionData(executionData *execution_data.BlockExecutionDat e.execDataBroadcaster.Publish() } - -// serve starts the gRPC server. -// When this function returns, the server is considered ready. -func (e *Engine) serve(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - e.log.Info().Str("state_stream_address", e.config.ListenAddr).Msg("starting grpc server on address") - l, err := net.Listen("tcp", e.config.ListenAddr) - if err != nil { - ctx.Throw(fmt.Errorf("error starting grpc server: %w", err)) - } - - e.stateStreamGrpcAddress = l.Addr() - e.log.Debug().Str("state_stream_address", e.stateStreamGrpcAddress.String()).Msg("listening on port") - - go func() { - ready() - err = e.server.Serve(l) - if err != nil { - ctx.Throw(fmt.Errorf("error trying to serve grpc server: %w", err)) - } - }() - - <-ctx.Done() - e.server.GracefulStop() -} diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go new file mode 100644 index 00000000000..25e483d986e --- /dev/null +++ b/module/grpcserver/server.go @@ -0,0 +1,176 @@ +package grpcserver + +import ( + "net" + "sync" + + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/rs/zerolog" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/onflow/flow-go/engine/common/rpc" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" +) + +// GrpcServerConfig defines the configurable options for the access node server +// GRPC server here implies a server that presents a self-signed TLS certificate and a client that authenticates +// the server via a pre-shared public key +type GrpcServerConfig struct { + GRPCListenAddr string // the GRPC server address as ip:port + TransportCredentials credentials.TransportCredentials // the GRPC credentials + MaxMsgSize uint // GRPC max message size +} + +// NewGrpcServerConfig initializes a new grpc server config. +func NewGrpcServerConfig(grpcListenAddr string, maxMsgSize uint, opts ...Option) GrpcServerConfig { + server := GrpcServerConfig{ + GRPCListenAddr: grpcListenAddr, + MaxMsgSize: maxMsgSize, + } + for _, applyOption := range opts { + applyOption(&server) + } + + return server +} + +type Option func(*GrpcServerConfig) + +// WithTransportCredentials sets the transport credentials parameters for a grpc server config. +func WithTransportCredentials(transportCredentials credentials.TransportCredentials) Option { + return func(c *GrpcServerConfig) { + c.TransportCredentials = transportCredentials + } +} + +// GrpcServer defines a grpc server that starts once and uses in different Engines. +// It makes it easy to configure the node to use the same port for both APIs. +type GrpcServer struct { + component.Component + log zerolog.Logger + cm *component.ComponentManager + grpcServer *grpc.Server + + config GrpcServerConfig + + addrLock sync.RWMutex + grpcAddress net.Addr +} + +// NewGrpcServer returns a new grpc server. +func NewGrpcServer(log zerolog.Logger, + config GrpcServerConfig, + grpcServer *grpc.Server, +) (*GrpcServer, error) { + server := &GrpcServer{ + log: log, + grpcServer: grpcServer, + config: config, + } + server.cm = component.NewComponentManagerBuilder(). + AddWorker(server.serveGRPCWorker). + AddWorker(server.shutdownWorker). + Build() + server.Component = server.cm + return server, nil +} + +// serveGRPCWorker is a worker routine which starts the gRPC server. +// The ready callback is called after the server address is bound and set. +func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + g.log.Info().Str("grpc_address", g.config.GRPCListenAddr).Msg("starting grpc server on address") + + l, err := net.Listen("tcp", g.config.GRPCListenAddr) + if err != nil { + g.log.Err(err).Msg("failed to start the grpc server") + ctx.Throw(err) + return + } + + // save the actual address on which we are listening (may be different from g.config.GRPCListenAddr if not port + // was specified) + g.addrLock.Lock() + g.grpcAddress = l.Addr() + g.addrLock.Unlock() + g.log.Debug().Str("grpc_address", g.grpcAddress.String()).Msg("listening on port") + ready() + + err = g.grpcServer.Serve(l) // blocking call + if err != nil { + g.log.Err(err).Msg("fatal error in grpc server") + ctx.Throw(err) + } +} + +// GRPCAddress returns the listen address of the GRPC server. +// Guaranteed to be non-nil after Engine.Ready is closed. +func (g *GrpcServer) GRPCAddress() net.Addr { + g.addrLock.RLock() + defer g.addrLock.RUnlock() + return g.grpcAddress +} + +// shutdownWorker is a worker routine which shuts down server when the context is cancelled. +func (g *GrpcServer) shutdownWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-ctx.Done() + g.grpcServer.GracefulStop() +} + +type GrpcServerBuilder struct { + log zerolog.Logger + config GrpcServerConfig + server *grpc.Server +} + +// NewGrpcServerBuilder helps to build a new grpc server. +func NewGrpcServerBuilder(log zerolog.Logger, + config GrpcServerConfig, + rpcMetricsEnabled bool, + apiRateLimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 + apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 +) *GrpcServerBuilder { + log = log.With().Str("component", "grpc_server").Logger() + // create a GRPC server to serve GRPC clients + grpcOpts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), + grpc.MaxSendMsgSize(int(config.MaxMsgSize)), + } + var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors + // if rpc metrics is enabled, first create the grpc metrics interceptor + if rpcMetricsEnabled { + interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) + } + if len(apiRateLimits) > 0 { + // create a rate limit interceptor + rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRateLimits, apiBurstLimits).UnaryServerInterceptor + // append the rate limit interceptor to the list of interceptors + interceptors = append(interceptors, rateLimitInterceptor) + } + // add the logging interceptor, ensure it is innermost wrapper + interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) + // create a chained unary interceptor + chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) + // create an unsecured grpc server + grpcOpts = append(grpcOpts, chainedInterceptors) + if config.TransportCredentials != nil { + // create a secure server by using the secure grpc credentials that are passed in as part of config + grpcOpts = append(grpcOpts, grpc.Creds(config.TransportCredentials)) + } + + return &GrpcServerBuilder{ + log: log, + config: config, + server: grpc.NewServer(grpcOpts...), + } +} + +func (b *GrpcServerBuilder) Server() *grpc.Server { + return b.server +} + +func (b *GrpcServerBuilder) Build() (*GrpcServer, error) { + return NewGrpcServer(b.log, b.config, b.server) +} From 23f79f8439f035a9fd129b6455e8834c61e98a2d Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Wed, 31 May 2023 16:25:30 +0300 Subject: [PATCH 022/169] Updated observer builder --- cmd/observer/node_builder/observer_builder.go | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index d1b714e541d..31471772ef3 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -39,6 +39,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/chainsync" finalizer "github.com/onflow/flow-go/module/finalizer/consensus" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/id" "github.com/onflow/flow-go/module/local" "github.com/onflow/flow-go/module/metrics" @@ -846,6 +847,28 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { } func (builder *ObserverServiceBuilder) enqueueRPCServer() { + secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) + + secureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, + secureGrpcServerConfig, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits) + + unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + ) + + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, + unsecureGrpcServerConfig, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits) + builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { engineBuilder, err := rpc.NewBuilder( node.Logger, @@ -866,9 +889,9 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { 0, false, builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits, builder.Me, + secureGrpcServer, + unsecureGrpcServer, ) if err != nil { return nil, err @@ -903,6 +926,26 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.RpcEng.OnFinalizedBlock) return builder.RpcEng, nil }) + + // build secure grpc server + builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + secureGrpcServer, err := secureGrpcServer.Build() + if err != nil { + return nil, err + } + + return secureGrpcServer, nil + }) + + // build unsecure grpc server + builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + unsecureGrpcServer, err := unsecureGrpcServer.Build() + if err != nil { + return nil, err + } + + return unsecureGrpcServer, nil + }) } // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update From 13e35e8f88a01d89a84cbf166baa60ffac2295c8 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Wed, 31 May 2023 17:03:54 +0300 Subject: [PATCH 023/169] Fixed tests --- engine/access/rest_api_test.go | 56 +++++++++++++++++++++++++- engine/access/rpc/rate_limit_test.go | 59 +++++++++++++++++++++++++--- engine/access/secure_grpcr_test.go | 46 +++++++++++++++++++++- 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 07a0934bcd0..f30f0b29263 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -10,6 +10,10 @@ import ( "testing" "time" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/utils/grpcutils" + "google.golang.org/grpc/credentials" + "github.com/antihax/optional" restclient "github.com/onflow/flow/openapi/go-client-generated" "github.com/rs/zerolog" @@ -63,6 +67,10 @@ type RestAPITestSuite struct { ctx irrecoverable.SignalerContext cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *RestAPITestSuite) SetupTest() { @@ -118,21 +126,65 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + + secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + grpcserver.WithTransportCredentials(config.TransportCredentials)) + + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + secureGrpcServerConfig, + false, + nil, + nil) + + unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + ) + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + unsecureGrpcServerConfig, + false, + nil, + nil) + rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, suite.executionResults, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, - false, nil, nil, suite.me) + false, suite.me, secureGrpcServer, unsecureGrpcServer) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.secureGrpcServer, err = secureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + suite.rpcEng.Start(suite.ctx) - // wait for the server to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) } func (suite *RestAPITestSuite) TearDownTest() { suite.cancel() + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) } diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 8a43b8271a9..b38442445cf 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -3,6 +3,8 @@ package rpc import ( "context" "fmt" + "github.com/onflow/flow-go/module/grpcserver" + "google.golang.org/grpc/credentials" "io" "os" "testing" @@ -61,6 +63,10 @@ type RateLimitTestSuite struct { ctx irrecoverable.SignalerContext cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *RateLimitTestSuite) SetupTest() { @@ -101,6 +107,14 @@ func (suite *RateLimitTestSuite) SetupTest() { HTTPListenAddr: unittest.DefaultAddress, } + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + // set the rate limit to test with suite.rateLimit = 2 // set the burst limit to test with @@ -114,21 +128,55 @@ func (suite *RateLimitTestSuite) SetupTest() { "Ping": suite.rateLimit, } + secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + grpcserver.WithTransportCredentials(config.TransportCredentials)) + + unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + ) + + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + secureGrpcServerConfig, + false, + apiRateLimt, + apiBurstLimt) + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + unsecureGrpcServerConfig, + false, + apiRateLimt, + apiBurstLimt) + block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) rpcEngBuilder, err := NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, apiRateLimt, apiBurstLimt, suite.me) + nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, suite.me, secureGrpcServer, unsecureGrpcServer) require.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() require.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.secureGrpcServer, err = secureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + suite.rpcEng.Start(suite.ctx) - // wait for the server to startup + unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") // create the access api client - suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) + suite.client, suite.closer, err = accessAPIClient(suite.unsecureGrpcServer.GRPCAddress().String()) require.NoError(suite.T(), err) } @@ -140,8 +188,9 @@ func (suite *RateLimitTestSuite) TearDownTest() { if suite.closer != nil { suite.closer.Close() } - // close the server - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) + // close servers + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) } func TestRateLimit(t *testing.T) { diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 5bf94eb2059..f8a930220d7 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -2,6 +2,7 @@ package access import ( "context" + "github.com/onflow/flow-go/module/grpcserver" "io" "os" "testing" @@ -55,6 +56,10 @@ type SecureGRPCTestSuite struct { ctx irrecoverable.SignalerContext cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (suite *SecureGRPCTestSuite) SetupTest() { @@ -105,15 +110,50 @@ func (suite *SecureGRPCTestSuite) SetupTest() { // save the public key to use later in tests later suite.publicKey = networkingKey.PublicKey() + secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + grpcserver.WithTransportCredentials(config.TransportCredentials)) + + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + secureGrpcServerConfig, + false, + nil, + nil) + + unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + ) + + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + unsecureGrpcServerConfig, + false, + nil, + nil) + block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) rpcEngBuilder, err := rpc.NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, nil, nil, suite.me) + nil, suite.chainID, suite.metrics, suite.metrics, 0, 0, false, false, suite.me, secureGrpcServer, unsecureGrpcServer) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.secureGrpcServer, err = secureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() + assert.NoError(suite.T(), err) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + suite.rpcEng.Start(suite.ctx) // wait for the server to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) @@ -121,6 +161,8 @@ func (suite *SecureGRPCTestSuite) SetupTest() { func (suite *SecureGRPCTestSuite) TearDownTest() { suite.cancel() + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Done(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Done(), 2*time.Second) unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) } @@ -160,7 +202,7 @@ func (suite *SecureGRPCTestSuite) secureGRPCClient(publicKey crypto.PublicKey) ( assert.NoError(suite.T(), err) conn, err := grpc.Dial( - suite.rpcEng.SecureGRPCAddress().String(), + suite.secureGrpcServer.GRPCAddress().String(), grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) assert.NoError(suite.T(), err) From 6fc18325760b0298d88298accf287d4d973f38b0 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 1 Jun 2023 11:23:47 +0300 Subject: [PATCH 024/169] Linted --- engine/access/rest_api_test.go | 3 ++- engine/access/rpc/rate_limit_test.go | 4 ++-- engine/access/secure_grpcr_test.go | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index f30f0b29263..9a8ea054b57 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -10,9 +10,10 @@ import ( "testing" "time" + "google.golang.org/grpc/credentials" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/utils/grpcutils" - "google.golang.org/grpc/credentials" "github.com/antihax/optional" restclient "github.com/onflow/flow/openapi/go-client-generated" diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index b38442445cf..98378cbc27b 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -3,8 +3,6 @@ package rpc import ( "context" "fmt" - "github.com/onflow/flow-go/module/grpcserver" - "google.golang.org/grpc/credentials" "io" "os" "testing" @@ -18,11 +16,13 @@ import ( "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index f8a930220d7..0322aae7c2e 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -2,7 +2,7 @@ package access import ( "context" - "github.com/onflow/flow-go/module/grpcserver" + "io" "os" "testing" @@ -20,6 +20,7 @@ import ( accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/grpcserver" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" module "github.com/onflow/flow-go/module/mock" From 6938f4046a9468ca25c1335994069ac52748d798 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 8 Jun 2023 12:46:41 +0300 Subject: [PATCH 025/169] Created new RestServerApi, refactored rest router and server, added BuildFromGrpc handlers for rest forwarder --- engine/access/rest/accounts.go | 21 +- engine/access/rest/blocks.go | 243 +++++++----- engine/access/rest/collections.go | 30 +- engine/access/rest/events.go | 41 +- engine/access/rest/execution_result.go | 43 +- engine/access/rest/handler.go | 12 +- engine/access/rest/models/collection.go | 50 +++ engine/access/rest/models/event.go | 27 ++ engine/access/rest/models/network.go | 5 + .../access/rest/models/node_version_info.go | 10 + engine/access/rest/network.go | 9 +- engine/access/rest/node_version_info.go | 12 +- engine/access/rest/rest_server_api.go | 373 ++++++++++++++++++ engine/access/rest/router.go | 5 +- engine/access/rest/scripts.go | 24 +- engine/access/rest/server.go | 6 +- engine/access/rest/transactions.go | 43 +- engine/common/rpc/convert/convert.go | 13 + integration/localnet/builder/bootstrap.go | 2 + 19 files changed, 657 insertions(+), 312 deletions(-) create mode 100644 engine/access/rest/rest_server_api.go diff --git a/engine/access/rest/accounts.go b/engine/access/rest/accounts.go index 36371bf6c57..a7e194de9e3 100644 --- a/engine/access/rest/accounts.go +++ b/engine/access/rest/accounts.go @@ -1,33 +1,16 @@ package rest import ( - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetAccount handler retrieves account by address and returns the response -func GetAccount(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetAccount(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetAccountRequest() if err != nil { return nil, NewBadRequestError(err) } - // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it - if req.Height == request.FinalHeight || req.Height == request.SealedHeight { - header, _, err := backend.GetLatestBlockHeader(r.Context(), req.Height == request.SealedHeight) - if err != nil { - return nil, err - } - req.Height = header.Height - } - - account, err := backend.GetAccountAtBlockHeight(r.Context(), req.Address, req.Height) - if err != nil { - return nil, err - } - - var response models.Account - err = response.Build(account, link, r.ExpandFields) - return response, err + return srv.GetAccount(req, r.Context(), r.ExpandFields, link) } diff --git a/engine/access/rest/blocks.go b/engine/access/rest/blocks.go index e729f67a9bd..1f24f44d717 100644 --- a/engine/access/rest/blocks.go +++ b/engine/access/rest/blocks.go @@ -3,6 +3,7 @@ package rest import ( "context" "fmt" + "net/http" "google.golang.org/grpc/codes" @@ -11,123 +12,110 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" ) // GetBlocksByIDs gets blocks by provided ID or list of IDs. -func GetBlocksByIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByIDs(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockByIDsRequest() if err != nil { return nil, NewBadRequestError(err) } - blocks := make([]*models.Block, len(req.IDs)) - for i, id := range req.IDs { - block, err := getBlock(forID(&id), r, backend, link) - if err != nil { - return nil, err - } - blocks[i] = block - } + return srv.GetBlocksByIDs(req, r.Context(), r.ExpandFields, link) +} - return blocks, nil +// GetBlocksByHeight gets blocks by height. +func GetBlocksByHeight(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { + return srv.GetBlocksByHeight(r, link) } -func GetBlocksByHeight(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { - req, err := r.GetBlockRequest() +// GetBlockPayloadByID gets block payload by ID +func GetBlockPayloadByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { + req, err := r.GetBlockPayloadRequest() if err != nil { return nil, NewBadRequestError(err) } - if req.FinalHeight || req.SealedHeight { - block, err := getBlock(forFinalized(req.Heights[0]), r, backend, link) - if err != nil { - return nil, err - } + return srv.GetBlockPayloadByID(req, r.Context(), link) +} - return []*models.Block{block}, nil +func getBlock(option blockRequestOption, context context.Context, expandFields map[string]bool, backend access.API, link models.LinkGenerator) (*models.Block, error) { + // lookup block + blkProvider := NewBlockRequestProvider(backend, option) + blk, blockStatus, err := blkProvider.getBlock(context) + if err != nil { + return nil, err } - // if the query is /blocks/height=1000,1008,1049... - if req.HasHeights() { - blocks := make([]*models.Block, len(req.Heights)) - for i, h := range req.Heights { - block, err := getBlock(forHeight(h), r, backend, link) - if err != nil { - return nil, err + // lookup execution result + // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) + var block models.Block + executionResult, err := backend.GetExecutionResultForBlockID(context, blk.ID()) + if err != nil { + // handle case where execution result is not yet available + if se, ok := status.FromError(err); ok { + if se.Code() == codes.NotFound { + err := block.Build(blk, nil, link, blockStatus, expandFields) + if err != nil { + return nil, err + } + return &block, nil } - blocks[i] = block } - - return blocks, nil + return nil, err } - // support providing end height as "sealed" or "final" - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - latest, _, err := backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - req.EndHeight = latest.Header.Height // overwrite special value height with fetched - - if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) - } + err = block.Build(blk, executionResult, link, blockStatus, expandFields) + if err != nil { + return nil, err } + return &block, nil +} - blocks := make([]*models.Block, 0) - // start and end height inclusive - for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := getBlock(forHeight(i), r, backend, link) - if err != nil { - return nil, err - } - blocks = append(blocks, block) +func getForwarderBlock(option blockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { + // lookup block + blkProvider := NewBlockForwarderProvider(upstream, option) + blk, blockStatus, err := blkProvider.getBlock(context) + if err != nil { + return nil, err } - return blocks, nil -} + // lookup execution result + // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) + var block models.Block + getExecutionResultForBlockIDRequest := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: blk.Id, + } -// GetBlockPayloadByID gets block payload by ID -func GetBlockPayloadByID(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { - req, err := r.GetBlockPayloadRequest() + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockIDRequest) if err != nil { - return nil, NewBadRequestError(err) + return nil, err } - blkProvider := NewBlockProvider(backend, forID(&req.ID)) - blk, _, statusErr := blkProvider.getBlock(r.Context()) - if statusErr != nil { - return nil, statusErr + flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) + if err != nil { + return nil, err } - var payload models.BlockPayload - err = payload.Build(blk.Payload) + flowBlock, err := convert.MessageToBlock(blk) if err != nil { return nil, err } - return payload, nil -} - -func getBlock(option blockProviderOption, req *request.Request, backend access.API, link models.LinkGenerator) (*models.Block, error) { - // lookup block - blkProvider := NewBlockProvider(backend, option) - blk, blockStatus, err := blkProvider.getBlock(req.Context()) + flowBlockStatus, err := convert.MessagesToBlockStatus(blockStatus) if err != nil { return nil, err } - // lookup execution result - // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) - var block models.Block - executionResult, err := backend.GetExecutionResultForBlockID(req.Context(), blk.ID()) if err != nil { // handle case where execution result is not yet available if se, ok := status.FromError(err); ok { if se.Code() == codes.NotFound { - err := block.Build(blk, nil, link, blockStatus, req.ExpandFields) + err := block.Build(flowBlock, nil, link, flowBlockStatus, expandFields) if err != nil { return nil, err } @@ -137,60 +125,64 @@ func getBlock(option blockProviderOption, req *request.Request, backend access.A return nil, err } - err = block.Build(blk, executionResult, link, blockStatus, req.ExpandFields) + err = block.Build(flowBlock, flowExecResult, link, flowBlockStatus, expandFields) if err != nil { return nil, err } return &block, nil } -// blockProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to -// look up a block or a block header either by ID or by height -type blockProvider struct { - id *flow.Identifier - height uint64 - latest bool - sealed bool - backend access.API +type blockRequest struct { + id *flow.Identifier + height uint64 + latest bool + sealed bool } -type blockProviderOption func(blkProvider *blockProvider) +type blockRequestOption func(blkRequest *blockRequest) -func forID(id *flow.Identifier) blockProviderOption { - return func(blkProvider *blockProvider) { - blkProvider.id = id +func forID(id *flow.Identifier) blockRequestOption { + return func(blockRequest *blockRequest) { + blockRequest.id = id } } -func forHeight(height uint64) blockProviderOption { - return func(blkProvider *blockProvider) { - blkProvider.height = height +func forHeight(height uint64) blockRequestOption { + return func(blockRequest *blockRequest) { + blockRequest.height = height } } -func forFinalized(queryParam uint64) blockProviderOption { - return func(blkProvider *blockProvider) { +func forFinalized(queryParam uint64) blockRequestOption { + return func(blockRequest *blockRequest) { switch queryParam { case request.SealedHeight: - blkProvider.sealed = true + blockRequest.sealed = true fallthrough case request.FinalHeight: - blkProvider.latest = true + blockRequest.latest = true } } } -func NewBlockProvider(backend access.API, options ...blockProviderOption) *blockProvider { - blkProvider := &blockProvider{ +// blockProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to +// look up a block or a block header either by ID or by height +type blockRequestProvider struct { + blockRequest + backend access.API +} + +func NewBlockRequestProvider(backend access.API, options ...blockRequestOption) *blockRequestProvider { + blockRequestProvider := &blockRequestProvider{ backend: backend, } for _, o := range options { - o(blkProvider) + o(&blockRequestProvider.blockRequest) } - return blkProvider + return blockRequestProvider } -func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { +func (blkProvider *blockRequestProvider) getBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { if blkProvider.id != nil { blk, _, err := blkProvider.backend.GetBlockByID(ctx, *blkProvider.id) if err != nil { // unfortunately backend returns internal error status if not found @@ -218,3 +210,60 @@ func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, fl } return blk, status, nil } + +// blockProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to +// look up a block or a block header either by ID or by height +type blockForwarderProvider struct { + blockRequest + upstream accessproto.AccessAPIClient +} + +func NewBlockForwarderProvider(upstream accessproto.AccessAPIClient, options ...blockRequestOption) *blockForwarderProvider { + blockForwarderProvider := &blockForwarderProvider{ + upstream: upstream, + } + + for _, o := range options { + o(&blockForwarderProvider.blockRequest) + } + return blockForwarderProvider +} + +func (blkProvider *blockForwarderProvider) getBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { + if blkProvider.id != nil { + getBlockByIdRequest := &accessproto.GetBlockByIDRequest{ + Id: []byte(blkProvider.id.String()), + } + blockResponse, err := blkProvider.upstream.GetBlockByID(ctx, getBlockByIdRequest) + if err != nil { // unfortunately grpc returns internal error status if not found + return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewNotFoundError( + fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err, + ) + } + return blockResponse.Block, entities.BlockStatus_BLOCK_UNKNOWN, nil + } + + if blkProvider.latest { + getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ + IsSealed: blkProvider.sealed, + } + blockResponse, err := blkProvider.upstream.GetLatestBlock(ctx, getLatestBlockRequest) + if err != nil { + // cannot be a 'not found' error since final and sealed block should always be found + return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewRestError(http.StatusInternalServerError, "block lookup failed", err) + } + return blockResponse.Block, blockResponse.BlockStatus, nil + } + + getBlockByHeight := &accessproto.GetBlockByHeightRequest{ + Height: blkProvider.height, + FullBlockResponse: true, + } + blockResponse, err := blkProvider.upstream.GetBlockByHeight(ctx, getBlockByHeight) + if err != nil { // unfortunately grpc returns internal error status if not found + return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewNotFoundError( + fmt.Sprintf("error looking up block at height %d", blkProvider.height), err, + ) + } + return blockResponse.Block, blockResponse.BlockStatus, nil +} diff --git a/engine/access/rest/collections.go b/engine/access/rest/collections.go index 807be2c0c41..be1b751b348 100644 --- a/engine/access/rest/collections.go +++ b/engine/access/rest/collections.go @@ -1,42 +1,16 @@ package rest import ( - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/model/flow" ) // GetCollectionByID retrieves a collection by ID and builds a response -func GetCollectionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetCollectionByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetCollectionRequest() if err != nil { return nil, NewBadRequestError(err) } - collection, err := backend.GetCollectionByID(r.Context(), req.ID) - if err != nil { - return nil, err - } - - // if we expand transactions in the query retrieve each transaction data - transactions := make([]*flow.TransactionBody, 0) - if req.ExpandsTransactions { - for _, tid := range collection.Transactions { - tx, err := backend.GetTransaction(r.Context(), tid) - if err != nil { - return nil, err - } - - transactions = append(transactions, tx) - } - } - - var response models.Collection - err = response.Build(collection, transactions, link, r.ExpandFields) - if err != nil { - return nil, err - } - - return response, nil + return srv.GetCollectionByID(req, r.Context(), r.ExpandFields, link, r.Chain) } diff --git a/engine/access/rest/events.go b/engine/access/rest/events.go index 2a79939bc21..e1e3eb6c7ec 100644 --- a/engine/access/rest/events.go +++ b/engine/access/rest/events.go @@ -1,56 +1,19 @@ package rest import ( - "fmt" - "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - - "github.com/onflow/flow-go/access" ) const blockQueryParam = "block_ids" const eventTypeQuery = "type" // GetEvents for the provided block range or list of block IDs filtered by type. -func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { +func GetEvents(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetEventsRequest() if err != nil { return nil, NewBadRequestError(err) } - // if the request has block IDs provided then return events for block IDs - var blocksEvents models.BlocksEvents - if len(req.BlockIDs) > 0 { - events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil - } - - // if end height is provided with special values then load the height - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - latest, _, err := backend.GetLatestBlockHeader(r.Context(), req.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - req.EndHeight = latest.Height - // special check after we resolve special height value - if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) - } - } - - // if request provided block height range then return events for that range - events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil + return srv.GetEvents(req, r.Context()) } diff --git a/engine/access/rest/execution_result.go b/engine/access/rest/execution_result.go index b0583d43b0d..1b9af6c586d 100644 --- a/engine/access/rest/execution_result.go +++ b/engine/access/rest/execution_result.go @@ -1,61 +1,26 @@ package rest import ( - "fmt" - - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func GetExecutionResultsByBlockIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultsByBlockIDs(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultByBlockIDsRequest() if err != nil { return nil, NewBadRequestError(err) } - // for each block ID we retrieve execution result - results := make([]models.ExecutionResult, len(req.BlockIDs)) - for i, id := range req.BlockIDs { - res, err := backend.GetExecutionResultForBlockID(r.Context(), id) - if err != nil { - return nil, err - } - - var response models.ExecutionResult - err = response.Build(res, link) - if err != nil { - return nil, err - } - results[i] = response - } - - return results, nil + return srv.GetExecutionResultsByBlockIDs(req, r.Context(), link) } // GetExecutionResultByID gets execution result by the ID. -func GetExecutionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultRequest() if err != nil { return nil, NewBadRequestError(err) } - res, err := backend.GetExecutionResultByID(r.Context(), req.ID) - if err != nil { - return nil, err - } - - if res == nil { - err := fmt.Errorf("execution result with ID: %s not found", req.ID.String()) - return nil, NewNotFoundError(err.Error(), err) - } - - var response models.ExecutionResult - err = response.Build(res, link) - if err != nil { - return nil, err - } - - return response, nil + return srv.GetExecutionResultByID(req, r.Context(), link) } diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index 028176fc9e0..74e5663172f 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -15,8 +15,6 @@ import ( "github.com/rs/zerolog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - - "github.com/onflow/flow-go/access" ) const MaxRequestSize = 2 << 20 // 2MB @@ -25,7 +23,7 @@ const MaxRequestSize = 2 << 20 // 2MB // it fetches necessary resources and returns an error or response model. type ApiHandlerFunc func( r *request.Request, - backend access.API, + srv RestServerApi, generator models.LinkGenerator, ) (interface{}, error) @@ -34,7 +32,7 @@ type ApiHandlerFunc func( // wraps functionality for handling error and responses outside of endpoint handling. type Handler struct { logger zerolog.Logger - backend access.API + restServerAPI RestServerApi linkGenerator models.LinkGenerator apiHandlerFunc ApiHandlerFunc chain flow.Chain @@ -42,14 +40,14 @@ type Handler struct { func NewHandler( logger zerolog.Logger, - backend access.API, + restServerAPI RestServerApi, handlerFunc ApiHandlerFunc, generator models.LinkGenerator, chain flow.Chain, ) *Handler { return &Handler{ logger: logger, - backend: backend, + restServerAPI: restServerAPI, apiHandlerFunc: handlerFunc, linkGenerator: generator, chain: chain, @@ -74,7 +72,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { decoratedRequest := request.Decorate(r, h.chain) // execute handler function and check for error - response, err := h.apiHandlerFunc(decoratedRequest, h.backend, h.linkGenerator) + response, err := h.apiHandlerFunc(decoratedRequest, h.restServerAPI, h.linkGenerator) if err != nil { h.errorHandler(w, err, errLog) return diff --git a/engine/access/rest/models/collection.go b/engine/access/rest/models/collection.go index c5076fdc7db..b42982a4061 100644 --- a/engine/access/rest/models/collection.go +++ b/engine/access/rest/models/collection.go @@ -1,10 +1,13 @@ package models import ( + "encoding/hex" "fmt" "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow/protobuf/go/flow/entities" ) const ExpandsTransactions = "transactions" @@ -42,6 +45,53 @@ func (c *Collection) Build( return nil } +func (c *Collection) BuildFromGrpc( + collection *entities.Collection, + txs []*entities.Transaction, + link LinkGenerator, + expand map[string]bool, + chain flow.Chain) error { + + self, err := SelfLink(convert.MessageToIdentifier(collection.Id), link.CollectionLink) + if err != nil { + return err + } + + transactionsBody := make([]*flow.TransactionBody, 0) + for _, tx := range txs { + flowTransaction, err := convert.MessageToTransaction(tx, chain) + if err != nil { + return err + } + transactionsBody = append(transactionsBody, &flowTransaction) + } + + var expandable CollectionExpandable + var transactions Transactions + if expand[ExpandsTransactions] { + var txIds []flow.Identifier + for _, id := range collection.TransactionIds { + txIds = append(txIds, convert.MessageToIdentifier(id)) + } + transactions.Build(transactionsBody, link) + } else { + expandable.Transactions = make([]string, len(collection.TransactionIds)) + for i, id := range collection.TransactionIds { + expandable.Transactions[i], err = link.TransactionLink(convert.MessageToIdentifier(id)) + if err != nil { + return err + } + } + } + + c.Id = hex.EncodeToString(collection.Id) + c.Transactions = transactions + c.Links = self + c.Expandable = &expandable + + return nil +} + func (c *CollectionGuarantee) Build(guarantee *flow.CollectionGuarantee) { c.CollectionId = guarantee.CollectionID.String() c.SignerIndices = fmt.Sprintf("%x", guarantee.SignerIndices) diff --git a/engine/access/rest/models/event.go b/engine/access/rest/models/event.go index 929dbb3f42c..8cfb3457dc0 100644 --- a/engine/access/rest/models/event.go +++ b/engine/access/rest/models/event.go @@ -1,8 +1,12 @@ package models import ( + "encoding/hex" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func (e *Event) Build(event flow.Event) { @@ -48,3 +52,26 @@ func (b *BlocksEvents) Build(blocksEvents []flow.BlockEvents) { *b = evs } + +func (b *BlocksEvents) BuildFromGrpc(blocksEvents []*accessproto.EventsResponse_Result) { + evs := make([]BlockEvents, 0) + for _, ev := range blocksEvents { + var blockEvent BlockEvents + blockEvent.BuildFromGrpc(ev) + evs = append(evs, blockEvent) + } + + *b = evs +} + +func (b *BlockEvents) BuildFromGrpc(blockEvents *accessproto.EventsResponse_Result) { + b.BlockHeight = util.FromUint64(blockEvents.BlockHeight) + b.BlockId = hex.EncodeToString(blockEvents.BlockId) + b.BlockTimestamp = blockEvents.BlockTimestamp.AsTime() + + var events Events + flowEvents := convert.MessagesToEvents(blockEvents.Events) + events.Build(flowEvents) + b.Events = events + +} diff --git a/engine/access/rest/models/network.go b/engine/access/rest/models/network.go index 927b5a23362..584e2a73e89 100644 --- a/engine/access/rest/models/network.go +++ b/engine/access/rest/models/network.go @@ -2,8 +2,13 @@ package models import ( "github.com/onflow/flow-go/access" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func (t *NetworkParameters) Build(params *access.NetworkParameters) { t.ChainId = params.ChainID.String() } + +func (t *NetworkParameters) BuildFromGrpc(response *accessproto.GetNetworkParametersResponse) { + t.ChainId = response.ChainId +} diff --git a/engine/access/rest/models/node_version_info.go b/engine/access/rest/models/node_version_info.go index 6a85e9f8d42..f51878f158d 100644 --- a/engine/access/rest/models/node_version_info.go +++ b/engine/access/rest/models/node_version_info.go @@ -1,8 +1,11 @@ package models import ( + "encoding/hex" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow/protobuf/go/flow/entities" ) func (t *NodeVersionInfo) Build(params *access.NodeVersionInfo) { @@ -11,3 +14,10 @@ func (t *NodeVersionInfo) Build(params *access.NodeVersionInfo) { t.SporkId = params.SporkId.String() t.ProtocolVersion = util.FromUint64(params.ProtocolVersion) } + +func (t *NodeVersionInfo) BuildFromGrpc(params *entities.NodeVersionInfo) { + t.Semver = params.Semver + t.Commit = params.Commit + t.SporkId = hex.EncodeToString(params.SporkId) + t.ProtocolVersion = util.FromUint64(params.ProtocolVersion) +} diff --git a/engine/access/rest/network.go b/engine/access/rest/network.go index 6100bc765d5..1fe904e29dd 100644 --- a/engine/access/rest/network.go +++ b/engine/access/rest/network.go @@ -1,16 +1,11 @@ package rest import ( - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNetworkParameters returns network-wide parameters of the blockchain -func GetNetworkParameters(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { - params := backend.GetNetworkParameters(r.Context()) - - var response models.NetworkParameters - response.Build(¶ms) - return response, nil +func GetNetworkParameters(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { + return srv.GetNetworkParameters(r) } diff --git a/engine/access/rest/node_version_info.go b/engine/access/rest/node_version_info.go index 899d159cf4f..a1a04977835 100644 --- a/engine/access/rest/node_version_info.go +++ b/engine/access/rest/node_version_info.go @@ -1,19 +1,11 @@ package rest import ( - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNodeVersionInfo returns node version information -func GetNodeVersionInfo(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { - params, err := backend.GetNodeVersionInfo(r.Context()) - if err != nil { - return nil, err - } - - var response models.NodeVersionInfo - response.Build(params) - return response, nil +func GetNodeVersionInfo(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { + return srv.GetNodeVersionInfo(r) } diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go new file mode 100644 index 00000000000..ebc2e54f831 --- /dev/null +++ b/engine/access/rest/rest_server_api.go @@ -0,0 +1,373 @@ +package rest + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/model/flow" +) + +type RestServerApi interface { + // GetTransactionByID gets a transaction by requested ID. + GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) + // CreateTransaction creates a new transaction from provided payload. + CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) + // GetTransactionResultByID retrieves transaction result by the transaction ID. + GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) + // GetBlocksByIDs gets blocks by provided ID or list of IDs. + GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) + // GetBlocksByHeight gets blocks by provided height. + GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) + // GetBlockPayloadByID gets block payload by ID + GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) + // GetExecutionResultByID gets execution result by the ID. + GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) + // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. + GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) + // GetCollectionByID retrieves a collection by ID and builds a response + GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) + // ExecuteScript handler sends the script from the request to be executed. + ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) + // GetAccount handler retrieves account by address and returns the response. + GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) + // GetEvents for the provided block range or list of block IDs filtered by type. + GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) + // GetNetworkParameters returns network-wide parameters of the blockchain + GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) + // GetNodeVersionInfo returns node version information + GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) +} + +type RequestHandler struct { + RestServerApi + log zerolog.Logger + backend access.API +} + +// NewRequestHandler returns new RequestHandler. +func NewRequestHandler(log zerolog.Logger, backend access.API) RestServerApi { + return &RequestHandler{ + log: log, + backend: backend, + } +} + +// GetTransactionByID gets a transaction by requested ID. +func (h *RequestHandler) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, _ flow.Chain) (models.Transaction, error) { + var response models.Transaction + + tx, err := h.backend.GetTransaction(context, r.ID) + if err != nil { + return response, err + } + + var txr *access.TransactionResult + // only lookup result if transaction result is to be expanded + if r.ExpandsResult { + txr, err = h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) + if err != nil { + return response, err + } + } + + response.Build(tx, txr, link) + return response, nil +} + +// CreateTransaction creates a new transaction from provided payload. +func (h *RequestHandler) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + var response models.Transaction + + err := h.backend.SendTransaction(context, &r.Transaction) + if err != nil { + return response, err + } + + response.Build(&r.Transaction, nil, link) + return response, nil +} + +// GetTransactionResultByID retrieves transaction result by the transaction ID. +func (h *RequestHandler) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + var response models.TransactionResult + + txr, err := h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) + if err != nil { + return response, err + } + + response.Build(txr, r.ID, link) + return response, nil +} + +// GetBlocksByIDs gets blocks by provided ID or list of IDs. +func (h *RequestHandler) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + blocks := make([]*models.Block, len(r.IDs)) + + for i, id := range r.IDs { + block, err := getBlock(forID(&id), context, expandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil +} + +// GetBlocksByHeight gets blocks by provided height. +func (h *RequestHandler) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + req, err := r.GetBlockRequest() + if err != nil { + return nil, NewBadRequestError(err) + } + + if req.FinalHeight || req.SealedHeight { + block, err := getBlock(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + + return []*models.Block{block}, nil + } + + // if the query is /blocks/height=1000,1008,1049... + if req.HasHeights() { + blocks := make([]*models.Block, len(req.Heights)) + for i, height := range req.Heights { + block, err := getBlock(forHeight(height), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil + } + + // support providing end height as "sealed" or "final" + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + latest, _, err := h.backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + req.EndHeight = latest.Header.Height // overwrite special value height with fetched + + if req.StartHeight > req.EndHeight { + return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + } + } + + blocks := make([]*models.Block, 0) + // start and end height inclusive + for i := req.StartHeight; i <= req.EndHeight; i++ { + block, err := getBlock(forHeight(i), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil +} + +// GetBlockPayloadByID gets block payload by ID +func (h *RequestHandler) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { + var payload models.BlockPayload + + blkProvider := NewBlockRequestProvider(h.backend, forID(&r.ID)) + blk, _, statusErr := blkProvider.getBlock(context) + if statusErr != nil { + return payload, statusErr + } + + err := payload.Build(blk.Payload) + if err != nil { + return payload, err + } + + return payload, nil +} + +// GetExecutionResultByID gets execution result by the ID. +func (h *RequestHandler) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + var response models.ExecutionResult + + res, err := h.backend.GetExecutionResultByID(context, r.ID) + if err != nil { + return response, err + } + + if res == nil { + err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) + return response, NewNotFoundError(err.Error(), err) + } + + err = response.Build(res, link) + if err != nil { + return response, err + } + + return response, nil +} + +// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. +func (h *RequestHandler) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + // for each block ID we retrieve execution result + results := make([]models.ExecutionResult, len(r.BlockIDs)) + for i, id := range r.BlockIDs { + res, err := h.backend.GetExecutionResultForBlockID(context, id) + if err != nil { + return nil, err + } + + var response models.ExecutionResult + err = response.Build(res, link) + if err != nil { + return nil, err + } + results[i] = response + } + + return results, nil +} + +// GetCollectionByID retrieves a collection by ID and builds a response +func (h *RequestHandler) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, _ flow.Chain) (models.Collection, error) { + var response models.Collection + + collection, err := h.backend.GetCollectionByID(context, r.ID) + if err != nil { + return response, err + } + + // if we expand transactions in the query retrieve each transaction data + transactions := make([]*flow.TransactionBody, 0) + if r.ExpandsTransactions { + for _, tid := range collection.Transactions { + tx, err := h.backend.GetTransaction(context, tid) + if err != nil { + return response, err + } + + transactions = append(transactions, tx) + } + } + + err = response.Build(collection, transactions, link, expandFields) + if err != nil { + return response, err + } + + return response, nil +} + +// ExecuteScript handler sends the script from the request to be executed. +func (h *RequestHandler) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { + if r.BlockID != flow.ZeroID { + return h.backend.ExecuteScriptAtBlockID(context, r.BlockID, r.Script.Source, r.Script.Args) + } + + // default to sealed height + if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { + return h.backend.ExecuteScriptAtLatestBlock(context, r.Script.Source, r.Script.Args) + } + + if r.BlockHeight == request.FinalHeight { + finalBlock, _, err := h.backend.GetLatestBlockHeader(context, false) + if err != nil { + return nil, err + } + r.BlockHeight = finalBlock.Height + } + + return h.backend.ExecuteScriptAtBlockHeight(context, r.BlockHeight, r.Script.Source, r.Script.Args) +} + +// GetAccount handler retrieves account by address and returns the response. +func (h *RequestHandler) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + var response models.Account + + // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it + if r.Height == request.FinalHeight || r.Height == request.SealedHeight { + header, _, err := h.backend.GetLatestBlockHeader(context, r.Height == request.SealedHeight) + if err != nil { + return response, err + } + r.Height = header.Height + } + + account, err := h.backend.GetAccountAtBlockHeight(context, r.Address, r.Height) + if err != nil { + return response, err + } + + err = response.Build(account, link, expandFields) + return response, err +} + +// GetEvents for the provided block range or list of block IDs filtered by type. +func (h *RequestHandler) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { + // if the request has block IDs provided then return events for block IDs + var blocksEvents models.BlocksEvents + if len(r.BlockIDs) > 0 { + events, err := h.backend.GetEventsForBlockIDs(context, r.Type, r.BlockIDs) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil + } + + // if end height is provided with special values then load the height + if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { + latest, _, err := h.backend.GetLatestBlockHeader(context, r.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + r.EndHeight = latest.Height + // special check after we resolve special height value + if r.StartHeight > r.EndHeight { + return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + } + } + + // if request provided block height range then return events for that range + events, err := h.backend.GetEventsForHeightRange(context, r.Type, r.StartHeight, r.EndHeight) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil +} + +// GetNetworkParameters returns network-wide parameters of the blockchain +func (h *RequestHandler) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { + params := h.backend.GetNetworkParameters(r.Context()) + + var response models.NetworkParameters + response.Build(¶ms) + return response, nil +} + +// GetNodeVersionInfo returns node version information +func (h *RequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { + var response models.NodeVersionInfo + + params, err := h.backend.GetNodeVersionInfo(r.Context()) + if err != nil { + return response, err + } + + response.Build(params) + return response, nil +} diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index da39912eff9..9bb04239b66 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -6,14 +6,13 @@ import ( "github.com/gorilla/mux" "github.com/rs/zerolog" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/middleware" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) -func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { +func newRouter(serverAPI RestServerApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { router := mux.NewRouter().StrictSlash(true) v1SubRouter := router.PathPrefix("/v1").Subrouter() @@ -26,7 +25,7 @@ func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, rest linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) for _, r := range Routes { - h := NewHandler(logger, backend, r.Handler, linkGenerator, chain) + h := NewHandler(logger, serverAPI, r.Handler, linkGenerator, chain) v1SubRouter. Methods(r.Method). Path(r.Pattern). diff --git a/engine/access/rest/scripts.go b/engine/access/rest/scripts.go index 8bd86bae54f..827bd25b13a 100644 --- a/engine/access/rest/scripts.go +++ b/engine/access/rest/scripts.go @@ -3,34 +3,14 @@ package rest import ( "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/model/flow" - - "github.com/onflow/flow-go/access" ) // ExecuteScript handler sends the script from the request to be executed. -func ExecuteScript(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { +func ExecuteScript(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetScriptRequest() if err != nil { return nil, NewBadRequestError(err) } - if req.BlockID != flow.ZeroID { - return backend.ExecuteScriptAtBlockID(r.Context(), req.BlockID, req.Script.Source, req.Script.Args) - } - - // default to sealed height - if req.BlockHeight == request.SealedHeight || req.BlockHeight == request.EmptyHeight { - return backend.ExecuteScriptAtLatestBlock(r.Context(), req.Script.Source, req.Script.Args) - } - - if req.BlockHeight == request.FinalHeight { - finalBlock, _, err := backend.GetLatestBlockHeader(r.Context(), false) - if err != nil { - return nil, err - } - req.BlockHeight = finalBlock.Height - } - - return backend.ExecuteScriptAtBlockHeight(r.Context(), req.BlockHeight, req.Script.Source, req.Script.Args) + return srv.ExecuteScript(req, r.Context(), link) } diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index a1aa83710d8..f196fb1a4a9 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -7,15 +7,13 @@ import ( "github.com/rs/cors" "github.com/rs/zerolog" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) // NewServer returns an HTTP server initialized with the REST API handler -func NewServer(backend access.API, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { - - router, err := newRouter(backend, logger, chain, restCollector) +func NewServer(serverAPI RestServerApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { + router, err := newRouter(serverAPI, logger, chain, restCollector) if err != nil { return nil, err } diff --git a/engine/access/rest/transactions.go b/engine/access/rest/transactions.go index f8dfc83dedb..1acdccfa1e8 100644 --- a/engine/access/rest/transactions.go +++ b/engine/access/rest/transactions.go @@ -1,67 +1,36 @@ package rest import ( - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetTransactionByID gets a transaction by requested ID. -func GetTransactionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetTransactionByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionRequest() if err != nil { return nil, NewBadRequestError(err) } - tx, err := backend.GetTransaction(r.Context(), req.ID) - if err != nil { - return nil, err - } - - var txr *access.TransactionResult - // only lookup result if transaction result is to be expanded - if req.ExpandsResult { - txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) - if err != nil { - return nil, err - } - } - - var response models.Transaction - response.Build(tx, txr, link) - return response, nil + return srv.GetTransactionByID(req, r.Context(), link, r.Chain) } // GetTransactionResultByID retrieves transaction result by the transaction ID. -func GetTransactionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func GetTransactionResultByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionResultRequest() if err != nil { return nil, NewBadRequestError(err) } - txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) - if err != nil { - return nil, err - } - - var response models.TransactionResult - response.Build(txr, req.ID, link) - return response, nil + return srv.GetTransactionResultByID(req, r.Context(), link) } // CreateTransaction creates a new transaction from provided payload. -func CreateTransaction(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { +func CreateTransaction(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.CreateTransactionRequest() if err != nil { return nil, NewBadRequestError(err) } - err = backend.SendTransaction(r.Context(), &req.Transaction) - if err != nil { - return nil, err - } - - var response models.Transaction - response.Build(&req.Transaction, nil, link) - return response, nil + return srv.CreateTransaction(req, r.Context(), link) } diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index 150e760d8de..eb35098f54c 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -374,6 +374,19 @@ func MessageToBlock(m *entities.Block) (*flow.Block, error) { }, nil } +func MessagesToBlockStatus(s entities.BlockStatus) (flow.BlockStatus, error) { + switch s { + case entities.BlockStatus_BLOCK_UNKNOWN: + return flow.BlockStatusUnknown, nil + case entities.BlockStatus_BLOCK_FINALIZED: + return flow.BlockStatusFinalized, nil + case entities.BlockStatus_BLOCK_SEALED: + return flow.BlockStatusSealed, nil + } + + return flow.BlockStatusUnknown, fmt.Errorf("failed to convert block status") +} + func MessagesToExecutionResultMetaList(m []*entities.ExecutionReceiptMeta) flow.ExecutionReceiptMetaList { execMetaList := make([]*flow.ExecutionReceiptMeta, len(m)) for i, message := range m { diff --git a/integration/localnet/builder/bootstrap.go b/integration/localnet/builder/bootstrap.go index 201aaaade58..ed6cb479acf 100644 --- a/integration/localnet/builder/bootstrap.go +++ b/integration/localnet/builder/bootstrap.go @@ -454,12 +454,14 @@ func prepareObserverService(i int, observerName string, agPublicKey string) Serv fmt.Sprintf("--rpc-addr=%s:%s", observerName, testnet.GRPCPort), fmt.Sprintf("--secure-rpc-addr=%s:%s", observerName, testnet.GRPCSecurePort), fmt.Sprintf("--http-addr=%s:%s", observerName, testnet.GRPCWebPort), + fmt.Sprintf("--rest-addr=%s:%s", observerName, testnet.RESTPort), ) service.AddExposedPorts( testnet.GRPCPort, testnet.GRPCSecurePort, testnet.GRPCWebPort, + testnet.RESTPort, ) // observer services rely on the access gateway From fcbf3974124f2f83f3d01d11339314382c0ef638 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 8 Jun 2023 12:52:34 +0300 Subject: [PATCH 026/169] Separated common forwarder logic to forwarder package, created and implemented RestForwarder, refactored FlowAccessAPIForwarder --- engine/access/apiproxy/access_api_proxy.go | 185 ++------ engine/access/rest/rest_server_api.go | 497 +++++++++++++++++++++ module/forwarder/forwarder.go | 144 ++++++ 3 files changed, 673 insertions(+), 153 deletions(-) create mode 100644 module/forwarder/forwarder.go diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index d72ec5bb5e2..6f0925667ea 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -2,26 +2,17 @@ package apiproxy import ( "context" - "fmt" - "sync" "time" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials/insecure" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/protocol" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/forwarder" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/utils/grpcutils" ) // FlowAccessAPIRouter is a structure that represents the routing proxy algorithm. @@ -51,88 +42,6 @@ func (h *FlowAccessAPIRouter) log(handler, rpc string, err error) { logger.Info().Msg("request succeeded") } -// reconnectingClient returns an active client, or -// creates one, if the last one is not ready anymore. -func (h *FlowAccessAPIForwarder) reconnectingClient(i int) error { - timeout := h.timeout - - if h.connections[i] == nil || h.connections[i].GetState() != connectivity.Ready { - identity := h.ids[i] - var connection *grpc.ClientConn - var err error - if identity.NetworkPubKey == nil { - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), - grpc.WithTransportCredentials(insecure.NewCredentials()), - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return err - } - } else { - tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) - if err != nil { - return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) - } - - connection, err = grpc.Dial( - identity.Address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(h.maxMsgSize))), - grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), - backend.WithClientUnaryInterceptor(timeout)) - if err != nil { - return fmt.Errorf("cannot connect to %s %w", identity.Address, err) - } - } - connection.Connect() - time.Sleep(1 * time.Second) - state := connection.GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - return fmt.Errorf("%v", state) - } - h.connections[i] = connection - h.upstream[i] = access.NewAccessAPIClient(connection) - } - - return nil -} - -// faultTolerantClient implements an upstream connection that reconnects on errors -// a reasonable amount of time. -func (h *FlowAccessAPIForwarder) faultTolerantClient() (access.AccessAPIClient, error) { - if h.upstream == nil || len(h.upstream) == 0 { - return nil, status.Errorf(codes.Unimplemented, "method not implemented") - } - - // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. - // A bigger number is problematic due to the DNS resolve and connection times, - // plus the need to log and debug each individual connection failure. - // - // This reasoning eliminates the need of making this parameter configurable. - // The logic works rolling over a single connection as well making clean code. - const retryMax = 3 - - h.lock.Lock() - defer h.lock.Unlock() - - var err error - for i := 0; i < retryMax; i++ { - h.roundRobin++ - h.roundRobin = h.roundRobin % len(h.upstream) - err = h.reconnectingClient(h.roundRobin) - if err != nil { - continue - } - state := h.connections[h.roundRobin].GetState() - if state != connectivity.Ready && state != connectivity.Connecting { - continue - } - return h.upstream[h.roundRobin], nil - } - - return nil, status.Errorf(codes.Unavailable, err.Error()) -} - // Ping pings the service. It is special in the sense that it responds successful, // only if all underlying services are ready. func (h *FlowAccessAPIRouter) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { @@ -292,52 +201,22 @@ func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Conte // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { - lock sync.Mutex - roundRobin int - ids flow.IdentityList - upstream []access.AccessAPIClient - connections []*grpc.ClientConn - timeout time.Duration - maxMsgSize uint + forwarder.Forwarder } func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*FlowAccessAPIForwarder, error) { - forwarder := &FlowAccessAPIForwarder{maxMsgSize: maxMsgSize} - err := forwarder.setFlowAccessAPI(identities, timeout) - return forwarder, err -} - -// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. -// It is used by Observer services, Blockchain Data Service, etc. -// Make sure that this is just for observation and not a staked participant in the flow network. -// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. -func (ret *FlowAccessAPIForwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { - ret.timeout = timeout - ret.ids = accessNodeAddressAndPort - ret.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) - ret.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) - for i, identity := range accessNodeAddressAndPort { - // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that - // we can refresh the API on connection loss - ret.ids[i] = identity - - // We fail on any single error on startup, so that - // we identify bootstrapping errors early - err := ret.reconnectingClient(i) - if err != nil { - return err - } - } + commonForwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) - ret.roundRobin = 0 - return nil + forwarder := &FlowAccessAPIForwarder{} + forwarder.Forwarder = commonForwarder + return forwarder, err } // Ping pings the service. It is special in the sense that it responds successful, // only if all underlying services are ready. func (h *FlowAccessAPIForwarder) Ping(context context.Context, req *access.PingRequest) (*access.PingResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -346,7 +225,7 @@ func (h *FlowAccessAPIForwarder) Ping(context context.Context, req *access.PingR func (h *FlowAccessAPIForwarder) GetNodeVersionInfo(context context.Context, req *access.GetNodeVersionInfoRequest) (*access.GetNodeVersionInfoResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -355,7 +234,7 @@ func (h *FlowAccessAPIForwarder) GetNodeVersionInfo(context context.Context, req func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, req *access.GetLatestBlockHeaderRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -364,7 +243,7 @@ func (h *FlowAccessAPIForwarder) GetLatestBlockHeader(context context.Context, r func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req *access.GetBlockHeaderByIDRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -373,7 +252,7 @@ func (h *FlowAccessAPIForwarder) GetBlockHeaderByID(context context.Context, req func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, req *access.GetBlockHeaderByHeightRequest) (*access.BlockHeaderResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -382,7 +261,7 @@ func (h *FlowAccessAPIForwarder) GetBlockHeaderByHeight(context context.Context, func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *access.GetLatestBlockRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -391,7 +270,7 @@ func (h *FlowAccessAPIForwarder) GetLatestBlock(context context.Context, req *ac func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *access.GetBlockByIDRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -400,7 +279,7 @@ func (h *FlowAccessAPIForwarder) GetBlockByID(context context.Context, req *acce func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req *access.GetBlockByHeightRequest) (*access.BlockResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -409,7 +288,7 @@ func (h *FlowAccessAPIForwarder) GetBlockByHeight(context context.Context, req * func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req *access.GetCollectionByIDRequest) (*access.CollectionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -418,7 +297,7 @@ func (h *FlowAccessAPIForwarder) GetCollectionByID(context context.Context, req func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *access.SendTransactionRequest) (*access.SendTransactionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -427,7 +306,7 @@ func (h *FlowAccessAPIForwarder) SendTransaction(context context.Context, req *a func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -436,7 +315,7 @@ func (h *FlowAccessAPIForwarder) GetTransaction(context context.Context, req *ac func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, req *access.GetTransactionRequest) (*access.TransactionResultResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -445,7 +324,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResult(context context.Context, r func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Context, req *access.GetTransactionByIndexRequest) (*access.TransactionResultResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -454,7 +333,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResultByIndex(context context.Con func (h *FlowAccessAPIForwarder) GetTransactionResultsByBlockID(context context.Context, req *access.GetTransactionsByBlockIDRequest) (*access.TransactionResultsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -462,7 +341,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionResultsByBlockID(context context. } func (h *FlowAccessAPIForwarder) GetTransactionsByBlockID(context context.Context, req *access.GetTransactionsByBlockIDRequest) (*access.TransactionsResponse, error) { - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -471,7 +350,7 @@ func (h *FlowAccessAPIForwarder) GetTransactionsByBlockID(context context.Contex func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access.GetAccountRequest) (*access.GetAccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -480,7 +359,7 @@ func (h *FlowAccessAPIForwarder) GetAccount(context context.Context, req *access func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context, req *access.GetAccountAtLatestBlockRequest) (*access.AccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -489,7 +368,7 @@ func (h *FlowAccessAPIForwarder) GetAccountAtLatestBlock(context context.Context func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context, req *access.GetAccountAtBlockHeightRequest) (*access.AccountResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -498,7 +377,7 @@ func (h *FlowAccessAPIForwarder) GetAccountAtBlockHeight(context context.Context func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Context, req *access.ExecuteScriptAtLatestBlockRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -507,7 +386,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtLatestBlock(context context.Cont func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, req *access.ExecuteScriptAtBlockIDRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -516,7 +395,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockID(context context.Context, func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Context, req *access.ExecuteScriptAtBlockHeightRequest) (*access.ExecuteScriptResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -525,7 +404,7 @@ func (h *FlowAccessAPIForwarder) ExecuteScriptAtBlockHeight(context context.Cont func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context, req *access.GetEventsForHeightRangeRequest) (*access.EventsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -534,7 +413,7 @@ func (h *FlowAccessAPIForwarder) GetEventsForHeightRange(context context.Context func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, req *access.GetEventsForBlockIDsRequest) (*access.EventsResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -543,7 +422,7 @@ func (h *FlowAccessAPIForwarder) GetEventsForBlockIDs(context context.Context, r func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, req *access.GetNetworkParametersRequest) (*access.GetNetworkParametersResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -552,7 +431,7 @@ func (h *FlowAccessAPIForwarder) GetNetworkParameters(context context.Context, r func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context.Context, req *access.GetLatestProtocolStateSnapshotRequest) (*access.ProtocolStateSnapshotResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } @@ -561,7 +440,7 @@ func (h *FlowAccessAPIForwarder) GetLatestProtocolStateSnapshot(context context. func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Context, req *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { // This is a passthrough request - upstream, err := h.faultTolerantClient() + upstream, err := h.FaultTolerantClient() if err != nil { return nil, err } diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go index ebc2e54f831..9d1f3d1815f 100644 --- a/engine/access/rest/rest_server_api.go +++ b/engine/access/rest/rest_server_api.go @@ -3,13 +3,19 @@ package rest import ( "context" "fmt" + "time" "github.com/rs/zerolog" "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/forwarder" + "github.com/onflow/flow/protobuf/go/flow/entities" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) type RestServerApi interface { @@ -371,3 +377,494 @@ func (h *RequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVers response.Build(params) return response, nil } + +type RestForwarder struct { + log zerolog.Logger + forwarder.Forwarder +} + +// NewRestForwarder returns new RestForwarder. +func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (RestServerApi, error) { + commonForwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + + forwarder := &RestForwarder{ + log: log, + } + forwarder.Forwarder = commonForwarder + return forwarder, err +} + +// GetTransactionByID gets a transaction by requested ID. +func (f *RestForwarder) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { + var response models.Transaction + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + } + transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) + if err != nil { + return response, err + } + + var transactionResultResponse *accessproto.TransactionResultResponse + // only lookup result if transaction result is to be expanded + if r.ExpandsResult { + getTransactionResultRequest := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + BlockId: r.BlockID[:], + CollectionId: r.CollectionID[:], + } + transactionResultResponse, err = upstream.GetTransactionResult(context, getTransactionResultRequest) + if err != nil { + return response, err + } + } + flowTransaction, err := convert.MessageToTransaction(transactionResponse.Transaction, chain) + if err != nil { + return response, err + } + + flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) + + response.Build(&flowTransaction, flowTransactionResult, link) + return response, nil +} + +// CreateTransaction creates a new transaction from provided payload. +func (f *RestForwarder) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + var response models.Transaction + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + entitiesTransaction := convert.TransactionToMessage(r.Transaction) + sendTransactionRequest := &accessproto.SendTransactionRequest{ + Transaction: entitiesTransaction, + } + + _, err = upstream.SendTransaction(context, sendTransactionRequest) + if err != nil { + return response, err + } + + response.Build(&r.Transaction, nil, link) + return response, nil +} + +// GetTransactionResultByID retrieves transaction result by the transaction ID. +func (f *RestForwarder) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + var response models.TransactionResult + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getTransactionResult := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + BlockId: r.BlockID[:], + CollectionId: r.CollectionID[:], + } + transactionResultResponse, err := upstream.GetTransactionResult(context, getTransactionResult) + if err != nil { + return response, err + } + + flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) + response.Build(flowTransactionResult, r.ID, link) + return response, nil +} + +// GetBlocksByIDs gets blocks by provided ID or list of IDs. +func (f *RestForwarder) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + blocks := make([]*models.Block, len(r.IDs)) + + upstream, err := f.FaultTolerantClient() + if err != nil { + return blocks, err + } + + for i, id := range r.IDs { + block, err := getForwarderBlock(forID(&id), context, expandFields, upstream, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil +} + +// GetBlocksByHeight gets blocks by provided height. +func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + req, err := r.GetBlockRequest() + if err != nil { + return nil, NewBadRequestError(err) + } + + upstream, err := f.FaultTolerantClient() + if err != nil { + return nil, err + } + + if req.FinalHeight || req.SealedHeight { + block, err := getForwarderBlock(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + + return []*models.Block{block}, nil + } + + // if the query is /blocks/height=1000,1008,1049... + if req.HasHeights() { + blocks := make([]*models.Block, len(req.Heights)) + for i, height := range req.Heights { + block, err := getForwarderBlock(forHeight(height), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil + } + + // support providing end height as "sealed" or "final" + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ + IsSealed: req.EndHeight == request.SealedHeight, + } + blockResponse, err := upstream.GetLatestBlock(r.Context(), getLatestBlockRequest) + if err != nil { + return nil, err + } + + req.EndHeight = blockResponse.Block.BlockHeader.Height // overwrite special value height with fetched + + if req.StartHeight > req.EndHeight { + return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + } + } + + blocks := make([]*models.Block, 0) + // start and end height inclusive + for i := req.StartHeight; i <= req.EndHeight; i++ { + block, err := getForwarderBlock(forHeight(i), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil +} + +// GetBlockPayloadByID gets block payload by ID +func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { + var payload models.BlockPayload + + upstream, err := f.FaultTolerantClient() + if err != nil { + return payload, err + } + + blkProvider := NewBlockForwarderProvider(upstream, forID(&r.ID)) + block, _, statusErr := blkProvider.getBlock(context) + if statusErr != nil { + return payload, statusErr + } + + flowPayload, err := convert.PayloadFromMessage(block) + if err != nil { + return payload, err + } + + err = payload.Build(flowPayload) + if err != nil { + return payload, err + } + + return payload, nil +} + +// GetExecutionResultByID gets execution result by the ID. +func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { /**/ + panic("Need to be implemented after grpc call added") +} + +// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. +func (f *RestForwarder) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + // for each block ID we retrieve execution result + results := make([]models.ExecutionResult, len(r.BlockIDs)) + + upstream, err := f.FaultTolerantClient() + if err != nil { + return results, err + } + + for i, id := range r.BlockIDs { + getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: id[:], + } + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockID) + if err != nil { + return nil, err + } + + var response models.ExecutionResult + flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) + if err != nil { + return nil, err + } + err = response.Build(flowExecResult, link) + if err != nil { + return nil, err + } + results[i] = response + } + + return results, nil +} + +// GetCollectionByID retrieves a collection by ID and builds a response +func (f *RestForwarder) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { + var response models.Collection + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ + Id: r.ID[:], + } + + collectionResponse, err := upstream.GetCollectionByID(context, getCollectionByIDRequest) + if err != nil { + return response, err + } + + // if we expand transactions in the query retrieve each transaction data + transactions := make([]*entities.Transaction, 0) + if r.ExpandsTransactions { + for _, tid := range collectionResponse.Collection.TransactionIds { + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: tid, + } + transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) + if err != nil { + return response, err + } + + transactions = append(transactions, transactionResponse.Transaction) + } + } + + err = response.BuildFromGrpc(collectionResponse.Collection, transactions, link, expandFields, chain) + if err != nil { + return response, err + } + + return response, nil +} + +// ExecuteScript handler sends the script from the request to be executed. +func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { + upstream, err := f.FaultTolerantClient() + if err != nil { + return nil, err + } + + if r.BlockID != flow.ZeroID { + executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ + BlockId: r.BlockID[:], + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(context, executeScriptAtBlockIDRequest) + if err != nil { + return nil, err + } + return executeScriptAtBlockIDResponse.Value, nil + } + + // default to sealed height + if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { + executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(context, executeScriptAtLatestBlockRequest) + if err != nil { + return nil, err + } + return executeScriptAtLatestBlockResponse.Value, nil + } + + if r.BlockHeight == request.FinalHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: false, + } + getLatestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return nil, err + } + r.BlockHeight = getLatestBlockHeaderResponse.Block.Height + } + + executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ + BlockHeight: r.BlockHeight, + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(context, executeScriptAtBlockHeightRequest) + if err != nil { + return nil, err + } + return executeScriptAtBlockHeightResponse.Value, nil +} + +// GetAccount handler retrieves account by address and returns the response. +func (f *RestForwarder) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + var response models.Account + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it + if r.Height == request.FinalHeight || r.Height == request.SealedHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: r.Height == request.SealedHeight, + } + blockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return response, err + } + r.Height = blockHeaderResponse.Block.Height + } + getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ + Address: r.Address.Bytes(), + BlockHeight: r.Height, + } + + accountResponse, err := upstream.GetAccountAtBlockHeight(context, getAccountAtBlockHeightRequest) + if err != nil { + return response, err + } + + flowAccount, err := convert.MessageToAccount(accountResponse.Account) + if err != nil { + return response, err + } + + err = response.Build(flowAccount, link, expandFields) + return response, err +} + +// GetEvents for the provided block range or list of block IDs filtered by type. +func (f *RestForwarder) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { + // if the request has block IDs provided then return events for block IDs + var blocksEvents models.BlocksEvents + + upstream, err := f.FaultTolerantClient() + if err != nil { + return blocksEvents, err + } + + if len(r.BlockIDs) > 0 { + var blockIds [][]byte + for _, id := range r.BlockIDs { + blockIds = append(blockIds, id[:]) + } + getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ + Type: r.Type, + BlockIds: blockIds, + } + eventsResponse, err := upstream.GetEventsForBlockIDs(context, getEventsForBlockIDsRequest) + if err != nil { + return nil, err + } + + blocksEvents.BuildFromGrpc(eventsResponse.Results) + + return blocksEvents, nil + } + + // if end height is provided with special values then load the height + if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: r.EndHeight == request.SealedHeight, + } + latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return nil, err + } + + r.EndHeight = latestBlockHeaderResponse.Block.Height + // special check after we resolve special height value + if r.StartHeight > r.EndHeight { + return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + } + } + + // if request provided block height range then return events for that range + getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ + Type: r.Type, + StartHeight: r.StartHeight, + EndHeight: r.EndHeight, + } + eventsResponse, err := upstream.GetEventsForHeightRange(context, getEventsForHeightRangeRequest) + if err != nil { + return nil, err + } + + blocksEvents.BuildFromGrpc(eventsResponse.Results) + return blocksEvents, nil +} + +// GetNetworkParameters returns network-wide parameters of the blockchain +func (f *RestForwarder) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { + var response models.NetworkParameters + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getNetworkParametersRequest := &accessproto.GetNetworkParametersRequest{} + getNetworkParametersResponse, err := upstream.GetNetworkParameters(r.Context(), getNetworkParametersRequest) + if err != nil { + return response, err + } + response.BuildFromGrpc(getNetworkParametersResponse) + return response, nil +} + +// GetNodeVersionInfo returns node version information +func (f *RestForwarder) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { + var response models.NodeVersionInfo + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getNodeVersionInfoRequest := &accessproto.GetNodeVersionInfoRequest{} + getNodeVersionInfoResponse, err := upstream.GetNodeVersionInfo(r.Context(), getNodeVersionInfoRequest) + if err != nil { + return response, err + } + + response.BuildFromGrpc(getNodeVersionInfoResponse.Info) + return response, nil +} diff --git a/module/forwarder/forwarder.go b/module/forwarder/forwarder.go new file mode 100644 index 00000000000..0201c3908cf --- /dev/null +++ b/module/forwarder/forwarder.go @@ -0,0 +1,144 @@ +package forwarder + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow/protobuf/go/flow/access" +) + +// Forwarder forwards all requests to a set of upstream access nodes or observers +type Forwarder struct { + lock sync.Mutex + roundRobin int + ids flow.IdentityList + upstream []access.AccessAPIClient + connections []*grpc.ClientConn + timeout time.Duration + maxMsgSize uint +} + +func NewForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (Forwarder, error) { + forwarder := Forwarder{maxMsgSize: maxMsgSize} + err := forwarder.setFlowAccessAPI(identities, timeout) + return forwarder, err +} + +// setFlowAccessAPI sets a backend access API that forwards some requests to an upstream node. +// It is used by Observer services, Blockchain Data Service, etc. +// Make sure that this is just for observation and not a staked participant in the flow network. +// This means that observers see a copy of the data but there is no interaction to ensure integrity from the root block. +func (f *Forwarder) setFlowAccessAPI(accessNodeAddressAndPort flow.IdentityList, timeout time.Duration) error { + f.timeout = timeout + f.ids = accessNodeAddressAndPort + f.upstream = make([]access.AccessAPIClient, accessNodeAddressAndPort.Count()) + f.connections = make([]*grpc.ClientConn, accessNodeAddressAndPort.Count()) + for i, identity := range accessNodeAddressAndPort { + // Store the faultTolerantClient setup parameters such as address, public, key and timeout, so that + // we can refresh the API on connection loss + f.ids[i] = identity + + // We fail on any single error on startup, so that + // we identify bootstrapping errors early + err := f.reconnectingClient(i) + if err != nil { + return err + } + } + + f.roundRobin = 0 + return nil +} + +// reconnectingClient returns an active client, or +// creates one, if the last one is not ready anymore. +func (f *Forwarder) reconnectingClient(i int) error { + timeout := f.timeout + + if f.connections[i] == nil || f.connections[i].GetState() != connectivity.Ready { + identity := f.ids[i] + var connection *grpc.ClientConn + var err error + if identity.NetworkPubKey == nil { + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(f.maxMsgSize))), + grpc.WithTransportCredentials(insecure.NewCredentials()), + backend.WithClientUnaryInterceptor(timeout)) + if err != nil { + return err + } + } else { + tlsConfig, err := grpcutils.DefaultClientTLSConfig(identity.NetworkPubKey) + if err != nil { + return fmt.Errorf("failed to get default TLS client config using public flow networking key %s %w", identity.NetworkPubKey.String(), err) + } + + connection, err = grpc.Dial( + identity.Address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(f.maxMsgSize))), + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + backend.WithClientUnaryInterceptor(timeout)) + if err != nil { + return fmt.Errorf("cannot connect to %s %w", identity.Address, err) + } + } + connection.Connect() + time.Sleep(1 * time.Second) + state := connection.GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + return fmt.Errorf("%v", state) + } + f.connections[i] = connection + f.upstream[i] = access.NewAccessAPIClient(connection) + } + + return nil +} + +// FaultTolerantClient implements an upstream connection that reconnects on errors +// a reasonable amount of time. +func (f *Forwarder) FaultTolerantClient() (access.AccessAPIClient, error) { + if f.upstream == nil || len(f.upstream) == 0 { + return nil, status.Errorf(codes.Unimplemented, "method not implemented") + } + + // Reasoning: A retry count of three gives an acceptable 5% failure ratio from a 37% failure ratio. + // A bigger number is problematic due to the DNS resolve and connection times, + // plus the need to log and debug each individual connection failure. + // + // This reasoning eliminates the need of making this parameter configurable. + // The logic works rolling over a single connection as well making clean code. + const retryMax = 3 + + f.lock.Lock() + defer f.lock.Unlock() + + var err error + for i := 0; i < retryMax; i++ { + f.roundRobin++ + f.roundRobin = f.roundRobin % len(f.upstream) + err = f.reconnectingClient(f.roundRobin) + if err != nil { + continue + } + state := f.connections[f.roundRobin].GetState() + if state != connectivity.Ready && state != connectivity.Connecting { + continue + } + return f.upstream[f.roundRobin], nil + } + + return nil, status.Errorf(codes.Unavailable, err.Error()) +} From a685db85e1d924beba8cb1cab66f784ae141f829 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 8 Jun 2023 13:19:57 +0300 Subject: [PATCH 027/169] Updated access api proxy test --- engine/access/apiproxy/access_api_proxy_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/engine/access/apiproxy/access_api_proxy_test.go b/engine/access/apiproxy/access_api_proxy_test.go index 9f5a5aa74b8..f8f2dce72e4 100644 --- a/engine/access/apiproxy/access_api_proxy_test.go +++ b/engine/access/apiproxy/access_api_proxy_test.go @@ -12,6 +12,7 @@ import ( grpcinsecure "google.golang.org/grpc/credentials/insecure" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/forwarder" "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/unittest" ) @@ -137,7 +138,8 @@ func TestNewFlowCachedAccessAPIProxy(t *testing.T) { // Prepare a proxy that fails due to the second connection being idle l := flow.IdentityList{{Address: unittest.IPPort("11634")}, {Address: unittest.IPPort("11635")}} c := FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) + c.Forwarder, err = forwarder.NewForwarder(l, time.Second, grpcutils.DefaultMaxMsgSize) + if err == nil { t.Fatal(fmt.Errorf("should not start with one connection ready")) } @@ -153,7 +155,7 @@ func TestNewFlowCachedAccessAPIProxy(t *testing.T) { // Prepare a proxy l = flow.IdentityList{{Address: unittest.IPPort("11634")}, {Address: unittest.IPPort("11635")}} c = FlowAccessAPIForwarder{} - err = c.setFlowAccessAPI(l, time.Second) + c.Forwarder, err = forwarder.NewForwarder(l, time.Second, grpcutils.DefaultMaxMsgSize) if err != nil { t.Fatal(err) } From 4ac6aa061090a3801d4526b8b0d2aced595d269f Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 8 Jun 2023 20:50:06 +0300 Subject: [PATCH 028/169] Fixed review remarks --- .../node_builder/access_node_builder.go | 16 +-- cmd/observer/node_builder/observer_builder.go | 15 +-- engine/access/rest_api_test.go | 14 +-- engine/access/rpc/rate_limit_test.go | 16 +-- engine/access/secure_grpcr_test.go | 15 +-- engine/access/state_stream/engine.go | 1 - module/grpcserver/server.go | 109 ++---------------- module/grpcserver/server_builder.go | 91 +++++++++++++++ 8 files changed, 122 insertions(+), 155 deletions(-) create mode 100644 module/grpcserver/server_builder.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index a79c1cf0ea1..be21ee337ac 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -995,23 +995,17 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("creating grpc servers", func(node *cmd.NodeConfig) error { - secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( - builder.rpcConf.SecureGRPCListenAddr, - builder.rpcConf.MaxMsgSize, grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) - builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, - secureGrpcServerConfig, + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, - builder.apiBurstlimits) + builder.apiBurstlimits, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) - unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, builder.rpcConf.UnsecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, - ) - - builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, - unsecureGrpcServerConfig, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 83fd51e94c9..53ae2cbcf30 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -847,24 +847,17 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { } func (builder *ObserverServiceBuilder) enqueueRPCServer() { - secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + secureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, builder.rpcConf.SecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, - grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) - - secureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, - secureGrpcServerConfig, builder.rpcMetricsEnabled, builder.apiRatelimits, - builder.apiBurstlimits) + builder.apiBurstlimits, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) - unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, builder.rpcConf.UnsecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, - ) - - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, - unsecureGrpcServerConfig, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits) diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 8191ac5fb65..f8d16172a45 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -135,23 +135,17 @@ func (suite *RestAPITestSuite) SetupTest() { // set the transport credentials for the server to use config.TransportCredentials = credentials.NewTLS(tlsConfig) - secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, - grpcserver.WithTransportCredentials(config.TransportCredentials)) - - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - secureGrpcServerConfig, false, nil, - nil) + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)) - unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, - ) - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - unsecureGrpcServerConfig, false, nil, nil) diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 45ce883b585..3f9558756e5 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -128,23 +128,17 @@ func (suite *RateLimitTestSuite) SetupTest() { "Ping": suite.rateLimit, } - secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, + false, + apiRateLimt, + apiBurstLimt, grpcserver.WithTransportCredentials(config.TransportCredentials)) - unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, - ) - - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - secureGrpcServerConfig, - false, - apiRateLimt, - apiBurstLimt) - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - unsecureGrpcServerConfig, false, apiRateLimt, apiBurstLimt) diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 3ef0fe5dd6b..9d2dde00cce 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -111,24 +111,17 @@ func (suite *SecureGRPCTestSuite) SetupTest() { // save the public key to use later in tests later suite.publicKey = networkingKey.PublicKey() - secureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, - grpcserver.WithTransportCredentials(config.TransportCredentials)) - - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - secureGrpcServerConfig, false, nil, - nil) + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)) - unsecureGrpcServerConfig := grpcserver.NewGrpcServerConfig( + unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, - ) - - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, - unsecureGrpcServerConfig, false, nil, nil) diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index fb6ed4419fc..c378e52c95b 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -107,7 +107,6 @@ func NewEng( e := &Engine{ log: logger, backend: backend, - server: server.Server(), headers: headers, chain: chainID.Chain(), config: config, diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index 25e483d986e..0b6368189c0 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -4,56 +4,22 @@ import ( "net" "sync" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/rs/zerolog" + "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" ) -// GrpcServerConfig defines the configurable options for the access node server -// GRPC server here implies a server that presents a self-signed TLS certificate and a client that authenticates -// the server via a pre-shared public key -type GrpcServerConfig struct { - GRPCListenAddr string // the GRPC server address as ip:port - TransportCredentials credentials.TransportCredentials // the GRPC credentials - MaxMsgSize uint // GRPC max message size -} - -// NewGrpcServerConfig initializes a new grpc server config. -func NewGrpcServerConfig(grpcListenAddr string, maxMsgSize uint, opts ...Option) GrpcServerConfig { - server := GrpcServerConfig{ - GRPCListenAddr: grpcListenAddr, - MaxMsgSize: maxMsgSize, - } - for _, applyOption := range opts { - applyOption(&server) - } - - return server -} - -type Option func(*GrpcServerConfig) - -// WithTransportCredentials sets the transport credentials parameters for a grpc server config. -func WithTransportCredentials(transportCredentials credentials.TransportCredentials) Option { - return func(c *GrpcServerConfig) { - c.TransportCredentials = transportCredentials - } -} - // GrpcServer defines a grpc server that starts once and uses in different Engines. // It makes it easy to configure the node to use the same port for both APIs. type GrpcServer struct { component.Component log zerolog.Logger - cm *component.ComponentManager grpcServer *grpc.Server - config GrpcServerConfig + grpcListenAddr string // the GRPC server address as ip:port addrLock sync.RWMutex grpcAddress net.Addr @@ -61,28 +27,27 @@ type GrpcServer struct { // NewGrpcServer returns a new grpc server. func NewGrpcServer(log zerolog.Logger, - config GrpcServerConfig, + grpcListenAddr string, grpcServer *grpc.Server, ) (*GrpcServer, error) { server := &GrpcServer{ - log: log, - grpcServer: grpcServer, - config: config, + log: log, + grpcServer: grpcServer, + grpcListenAddr: grpcListenAddr, } - server.cm = component.NewComponentManagerBuilder(). + server.Component = component.NewComponentManagerBuilder(). AddWorker(server.serveGRPCWorker). AddWorker(server.shutdownWorker). Build() - server.Component = server.cm return server, nil } // serveGRPCWorker is a worker routine which starts the gRPC server. // The ready callback is called after the server address is bound and set. func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - g.log.Info().Str("grpc_address", g.config.GRPCListenAddr).Msg("starting grpc server on address") + g.log.Info().Str("grpc_address", g.grpcListenAddr).Msg("starting grpc server on address") - l, err := net.Listen("tcp", g.config.GRPCListenAddr) + l, err := net.Listen("tcp", g.grpcListenAddr) if err != nil { g.log.Err(err).Msg("failed to start the grpc server") ctx.Throw(err) @@ -118,59 +83,3 @@ func (g *GrpcServer) shutdownWorker(ctx irrecoverable.SignalerContext, ready com <-ctx.Done() g.grpcServer.GracefulStop() } - -type GrpcServerBuilder struct { - log zerolog.Logger - config GrpcServerConfig - server *grpc.Server -} - -// NewGrpcServerBuilder helps to build a new grpc server. -func NewGrpcServerBuilder(log zerolog.Logger, - config GrpcServerConfig, - rpcMetricsEnabled bool, - apiRateLimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 - apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 -) *GrpcServerBuilder { - log = log.With().Str("component", "grpc_server").Logger() - // create a GRPC server to serve GRPC clients - grpcOpts := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(int(config.MaxMsgSize)), - grpc.MaxSendMsgSize(int(config.MaxMsgSize)), - } - var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors - // if rpc metrics is enabled, first create the grpc metrics interceptor - if rpcMetricsEnabled { - interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) - } - if len(apiRateLimits) > 0 { - // create a rate limit interceptor - rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRateLimits, apiBurstLimits).UnaryServerInterceptor - // append the rate limit interceptor to the list of interceptors - interceptors = append(interceptors, rateLimitInterceptor) - } - // add the logging interceptor, ensure it is innermost wrapper - interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) - // create a chained unary interceptor - chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) - // create an unsecured grpc server - grpcOpts = append(grpcOpts, chainedInterceptors) - if config.TransportCredentials != nil { - // create a secure server by using the secure grpc credentials that are passed in as part of config - grpcOpts = append(grpcOpts, grpc.Creds(config.TransportCredentials)) - } - - return &GrpcServerBuilder{ - log: log, - config: config, - server: grpc.NewServer(grpcOpts...), - } -} - -func (b *GrpcServerBuilder) Server() *grpc.Server { - return b.server -} - -func (b *GrpcServerBuilder) Build() (*GrpcServer, error) { - return NewGrpcServer(b.log, b.config, b.server) -} diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go new file mode 100644 index 00000000000..cf6f59838f7 --- /dev/null +++ b/module/grpcserver/server_builder.go @@ -0,0 +1,91 @@ +package grpcserver + +import ( + "github.com/rs/zerolog" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + + "github.com/onflow/flow-go/engine/common/rpc" +) + +type Option func(*GrpcServerBuilder) + +// WithTransportCredentials sets the transport credentials parameters for a grpc server config. +func WithTransportCredentials(transportCredentials credentials.TransportCredentials) Option { + return func(c *GrpcServerBuilder) { + c.transportCredentials = transportCredentials + } +} + +// GrpcServerBuilder created for separating the creation and starting GrpcServer, +// cause services need to be registered before the server starts. +type GrpcServerBuilder struct { + log zerolog.Logger + gRPCListenAddr string + server *grpc.Server + + transportCredentials credentials.TransportCredentials // the GRPC credentials +} + +// NewGrpcServerBuilder helps to build a new grpc server. +func NewGrpcServerBuilder(log zerolog.Logger, + gRPCListenAddr string, + maxMsgSize uint, + rpcMetricsEnabled bool, + apiRateLimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 + apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 + opts ...Option, +) *GrpcServerBuilder { + log = log.With().Str("component", "grpc_server").Logger() + + grpcServerBuilder := &GrpcServerBuilder{ + log: log, + gRPCListenAddr: gRPCListenAddr, + } + + for _, applyOption := range opts { + applyOption(grpcServerBuilder) + } + + // create a GRPC server to serve GRPC clients + grpcOpts := []grpc.ServerOption{ + grpc.MaxRecvMsgSize(int(maxMsgSize)), + grpc.MaxSendMsgSize(int(maxMsgSize)), + } + var interceptors []grpc.UnaryServerInterceptor // ordered list of interceptors + // if rpc metrics is enabled, first create the grpc metrics interceptor + if rpcMetricsEnabled { + interceptors = append(interceptors, grpc_prometheus.UnaryServerInterceptor) + } + if len(apiRateLimits) > 0 { + // create a rate limit interceptor + rateLimitInterceptor := rpc.NewRateLimiterInterceptor(log, apiRateLimits, apiBurstLimits).UnaryServerInterceptor + // append the rate limit interceptor to the list of interceptors + interceptors = append(interceptors, rateLimitInterceptor) + } + // add the logging interceptor, ensure it is innermost wrapper + interceptors = append(interceptors, rpc.LoggingInterceptor(log)...) + // create a chained unary interceptor + chainedInterceptors := grpc.ChainUnaryInterceptor(interceptors...) + // create an unsecured grpc server + grpcOpts = append(grpcOpts, chainedInterceptors) + + if grpcServerBuilder.transportCredentials != nil { + // create a secure server by using the secure grpc credentials that are passed in as part of config + grpcOpts = append(grpcOpts, grpc.Creds(grpcServerBuilder.transportCredentials)) + } + grpcServerBuilder.server = grpc.NewServer(grpcOpts...) + + return grpcServerBuilder +} + +func (b *GrpcServerBuilder) Server() *grpc.Server { + return b.server +} + +func (b *GrpcServerBuilder) Build() (*GrpcServer, error) { + return NewGrpcServer(b.log, b.gRPCListenAddr, b.server) +} From 9b2a245c02cefc9783bb1a4345e7b7aa1cbaddc2 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 8 Jun 2023 22:33:52 +0300 Subject: [PATCH 029/169] Updated last commit --- engine/access/state_stream/engine.go | 1 + module/grpcserver/server_builder.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index c378e52c95b..fb6ed4419fc 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -107,6 +107,7 @@ func NewEng( e := &Engine{ log: logger, backend: backend, + server: server.Server(), headers: headers, chain: chainID.Chain(), config: config, diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index cf6f59838f7..0079b781e13 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -13,7 +13,7 @@ import ( type Option func(*GrpcServerBuilder) -// WithTransportCredentials sets the transport credentials parameters for a grpc server config. +// WithTransportCredentials sets the transport credentials parameters for a grpc server builder. func WithTransportCredentials(transportCredentials credentials.TransportCredentials) Option { return func(c *GrpcServerBuilder) { c.transportCredentials = transportCredentials From 0a0e4e5486420ffc3df2d3bca1b5d83d5cdf5459 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 9 Jun 2023 12:44:11 -0600 Subject: [PATCH 030/169] remove one more usage of math/rand in engine/execution --- engine/execution/computation/query/executor.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/engine/execution/computation/query/executor.go b/engine/execution/computation/query/executor.go index 44f7ec69ab6..10a680475a0 100644 --- a/engine/execution/computation/query/executor.go +++ b/engine/execution/computation/query/executor.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "math/rand" "strings" "sync" "time" @@ -18,6 +17,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/utils/debug" + "github.com/onflow/flow-go/utils/rand" ) const ( @@ -71,7 +71,6 @@ type QueryExecutor struct { vmCtx fvm.Context derivedChainData *derived.DerivedChainData rngLock *sync.Mutex - rng *rand.Rand } var _ Executor = &QueryExecutor{} @@ -92,7 +91,6 @@ func NewQueryExecutor( vmCtx: vmCtx, derivedChainData: derivedChainData, rngLock: &sync.Mutex{}, - rng: rand.New(rand.NewSource(time.Now().UnixNano())), } } @@ -115,7 +113,10 @@ func (e *QueryExecutor) ExecuteScript( // TODO: this is a temporary measure, we could remove this in the future if e.logger.Debug().Enabled() { e.rngLock.Lock() - trackerID := e.rng.Uint32() + trackerID, err := rand.Uint32() + if err != nil { + return nil, fmt.Errorf("failed to generate trackerID: %w", err) + } e.rngLock.Unlock() trackedLogger := e.logger.With().Hex("script_hex", script).Uint32("trackerID", trackerID).Logger() From db663e33b9bf6b41c3636b3c15bf85ae0ca6b6fd Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 9 Jun 2023 14:34:39 -0600 Subject: [PATCH 031/169] avoid using math/rand in NewExponentialBackoff --- network/p2p/connection/connector_factory.go | 39 +++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/network/p2p/connection/connector_factory.go b/network/p2p/connection/connector_factory.go index a5c8be29704..caa16299df6 100644 --- a/network/p2p/connection/connector_factory.go +++ b/network/p2p/connection/connector_factory.go @@ -1,12 +1,14 @@ package connection import ( + "crypto/rand" "fmt" - "math/rand" "time" "github.com/libp2p/go-libp2p/core/host" discoveryBackoff "github.com/libp2p/go-libp2p/p2p/discovery/backoff" + + "github.com/onflow/flow-go/crypto/random" ) const ( @@ -34,7 +36,10 @@ const ( // (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) func DefaultLibp2pBackoffConnectorFactory(host host.Host) func() (*discoveryBackoff.BackoffConnector, error) { return func() (*discoveryBackoff.BackoffConnector, error) { - rngSrc := rand.NewSource(rand.Int63()) + rngSrc, err := newSource() + if err != nil { + return nil, fmt.Errorf("failed to generate a random source: %w", err) + } cacheSize := 100 dialTimeout := time.Minute * 2 @@ -54,3 +59,33 @@ func DefaultLibp2pBackoffConnectorFactory(host host.Host) func() (*discoveryBack return backoffConnector, nil } } + +// `source` implements math/rand.Source so it can be used +// by libp2p's `NewExponentialBackoff`. +// It is backed by a more secure randomness than math/rand's `NewSource`. +// `source` is only implemented to avoid using math/rand's `NewSource`. +type source struct { + prg random.Rand +} + +// Seed is not used by the backoff object from `NewExponentialBackoff` +func (src *source) Seed(seed int64) {} + +// Int63 is used by `NewExponentialBackoff` and is based on a crypto PRG +func (src *source) Int63() int64 { + return int64(src.prg.UintN(1 << 63)) +} + +// creates a source using a crypto PRG and secure random seed +func newSource() (*source, error) { + seed := make([]byte, random.Chacha20SeedLen) + _, err := rand.Read(seed) // checking err only is enough + if err != nil { + return nil, fmt.Errorf("failed to generate a seed: %w", err) + } + prg, err := random.NewChacha20PRG(seed, nil) + if err != nil { + return nil, fmt.Errorf("failed to generate a PRG: %w", err) + } + return &source{prg}, nil +} From 908212fca4c000ed0cadc49369718b416187619d Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 9 Jun 2023 17:58:42 -0600 Subject: [PATCH 032/169] clean up and comment updates --- engine/testutil/nodes.go | 1 - integration/tests/consensus/inclusion_test.go | 2 -- model/flow/address_test.go | 8 -------- model/flow/identity.go | 4 ++-- module/mempool/herocache/transactions.go | 12 +++++------- module/mempool/queue/heroQueue.go | 15 ++++++--------- module/mempool/queue/heroQueue_test.go | 1 - module/mempool/queue/heroStore.go | 4 +--- .../mempool/stdmap/backDataHeapBenchmark_test.go | 14 ++++++-------- network/p2p/connection/peerManager.go | 3 +++ state/protocol/badger/mutator_test.go | 4 ---- state/protocol/badger/snapshot_test.go | 4 ---- storage/badger/cleaner.go | 7 ++++++- storage/badger/operation/common_test.go | 4 ---- 14 files changed, 29 insertions(+), 54 deletions(-) diff --git a/engine/testutil/nodes.go b/engine/testutil/nodes.go index a3f4c75f15e..1ffd67b2e62 100644 --- a/engine/testutil/nodes.go +++ b/engine/testutil/nodes.go @@ -302,7 +302,6 @@ func CollectionNode(t *testing.T, hub *stub.Hub, identity bootstrap.NodeInfo, ro coll, err := collections.ByID(collID) return coll, err } - providerEngine, err := provider.New( node.Log, node.Metrics, diff --git a/integration/tests/consensus/inclusion_test.go b/integration/tests/consensus/inclusion_test.go index e36ef7dae8e..85ef3fd8046 100644 --- a/integration/tests/consensus/inclusion_test.go +++ b/integration/tests/consensus/inclusion_test.go @@ -43,8 +43,6 @@ func (is *InclusionSuite) SetupTest() { is.log = unittest.LoggerForTest(is.Suite.T(), zerolog.InfoLevel) is.log.Info().Msgf("================> SetupTest") - // seed random generator - // to collect node confiis... var nodeConfigs []testnet.NodeConfig diff --git a/model/flow/address_test.go b/model/flow/address_test.go index edb5b095218..28e99efa315 100644 --- a/model/flow/address_test.go +++ b/model/flow/address_test.go @@ -166,8 +166,6 @@ func testAddressConstants(t *testing.T) { const invalidCodeWord = uint64(0xab2ae42382900010) func testAddressGeneration(t *testing.T) { - // seed random generator - // loops in each test const loop = 50 @@ -258,8 +256,6 @@ func testAddressGeneration(t *testing.T) { } func testAddressesIntersection(t *testing.T) { - // seed random generator - // loops in each test const loop = 25 @@ -326,8 +322,6 @@ func testAddressesIntersection(t *testing.T) { } func testIndexFromAddress(t *testing.T) { - // seed random generator - // loops in each test const loop = 50 @@ -366,8 +360,6 @@ func testIndexFromAddress(t *testing.T) { } func TestUint48(t *testing.T) { - // seed random generator - const loop = 50 // test consistensy of putUint48 and uint48 for i := 0; i < loop; i++ { diff --git a/model/flow/identity.go b/model/flow/identity.go index 83d07236b79..7127858a379 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -480,8 +480,8 @@ func (il IdentityList) Sample(size uint) (IdentityList, error) { return dup[:size], nil } -// Shuffle non-deterministically randomly shuffles the identity -// list, returning the shuffled list without modifying the receiver. +// Shuffle randomly shuffles the identity list (non-deterministic), +// and returns the shuffled list without modifying the receiver. func (il IdentityList) Shuffle() (IdentityList, error) { n := uint(len(il)) dup := make([]*Identity, 0, n) diff --git a/module/mempool/herocache/transactions.go b/module/mempool/herocache/transactions.go index e8784c6a851..a052728de52 100644 --- a/module/mempool/herocache/transactions.go +++ b/module/mempool/herocache/transactions.go @@ -18,16 +18,14 @@ type Transactions struct { // NewTransactions implements a transactions mempool based on hero cache. func NewTransactions(limit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *Transactions { - cache := herocache.NewCache(limit, - herocache.DefaultOversizeFactor, - heropool.LRUEjection, - logger.With().Str("mempool", "transactions").Logger(), - collector) - t := &Transactions{ c: stdmap.NewBackend( stdmap.WithBackData( - cache)), + herocache.NewCache(limit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + logger.With().Str("mempool", "transactions").Logger(), + collector))), } return t diff --git a/module/mempool/queue/heroQueue.go b/module/mempool/queue/heroQueue.go index 52274fd4a81..ece206fec17 100644 --- a/module/mempool/queue/heroQueue.go +++ b/module/mempool/queue/heroQueue.go @@ -21,16 +21,13 @@ type HeroQueue struct { func NewHeroQueue(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, ) *HeroQueue { - - cache := herocache.NewCache( - sizeLimit, - herocache.DefaultOversizeFactor, - heropool.NoEjection, - logger.With().Str("mempool", "hero-queue").Logger(), - collector) - return &HeroQueue{ - cache: cache, + cache: herocache.NewCache( + sizeLimit, + herocache.DefaultOversizeFactor, + heropool.NoEjection, + logger.With().Str("mempool", "hero-queue").Logger(), + collector), sizeLimit: uint(sizeLimit), } } diff --git a/module/mempool/queue/heroQueue_test.go b/module/mempool/queue/heroQueue_test.go index 494dba7ae78..f0775a206c5 100644 --- a/module/mempool/queue/heroQueue_test.go +++ b/module/mempool/queue/heroQueue_test.go @@ -61,7 +61,6 @@ func TestHeroQueue_Concurrent(t *testing.T) { q := queue.NewHeroQueue(uint32(sizeLimit), unittest.Logger(), metrics.NewNoopCollector()) // initially queue must be zero require.Zero(t, q.Size()) - // initially there should be nothing to pop entity, ok := q.Pop() require.False(t, ok) diff --git a/module/mempool/queue/heroStore.go b/module/mempool/queue/heroStore.go index ddf6d176296..03c478e1893 100644 --- a/module/mempool/queue/heroStore.go +++ b/module/mempool/queue/heroStore.go @@ -35,10 +35,8 @@ type HeroStore struct { func NewHeroStore(sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics, ) *HeroStore { - queue := NewHeroQueue(sizeLimit, logger, collector) - return &HeroStore{ - q: queue, + q: NewHeroQueue(sizeLimit, logger, collector), } } diff --git a/module/mempool/stdmap/backDataHeapBenchmark_test.go b/module/mempool/stdmap/backDataHeapBenchmark_test.go index 4b9d7fc7c35..1a3fdbc7e17 100644 --- a/module/mempool/stdmap/backDataHeapBenchmark_test.go +++ b/module/mempool/stdmap/backDataHeapBenchmark_test.go @@ -46,16 +46,14 @@ func BenchmarkArrayBackDataLRU(b *testing.B) { defer debug.SetGCPercent(debug.SetGCPercent(-1)) // disable GC limit := uint(50_000) - cache := herocache.NewCache( - uint32(limit), - 8, - heropool.LRUEjection, - unittest.Logger(), - metrics.NewNoopCollector()) - backData := stdmap.NewBackend( stdmap.WithBackData( - cache), + herocache.NewCache( + uint32(limit), + 8, + heropool.LRUEjection, + unittest.Logger(), + metrics.NewNoopCollector())), stdmap.WithLimit(limit)) entities := unittest.EntityListFixture(uint(100_000_000)) diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index 4415d38aba5..d940a56beb9 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -89,6 +89,9 @@ func (pm *PeerManager) periodicLoop(ctx irrecoverable.SignalerContext) { r, _ := rand.Uint64n(uint64(pm.peerUpdateInterval.Nanoseconds())) // ignore the error here, if randomness fails `r` would be zero and there will be no delay // for the current node + // TODO: treat the error properly instead of swallowing it. In this specific case, `utils/rand` + // only errors if there is a randomness system issue. Such issue will cause errors in many + // other components. delay := time.Duration(r) ticker := time.NewTicker(pm.peerUpdateInterval) diff --git a/state/protocol/badger/mutator_test.go b/state/protocol/badger/mutator_test.go index ded41bd3340..53ecd1a6e79 100644 --- a/state/protocol/badger/mutator_test.go +++ b/state/protocol/badger/mutator_test.go @@ -40,10 +40,6 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func init() { - -} - var participants = unittest.IdentityListFixture(5, unittest.WithAllRoles()) func TestBootstrapValid(t *testing.T) { diff --git a/state/protocol/badger/snapshot_test.go b/state/protocol/badger/snapshot_test.go index 7eebf53dfcb..1e08715115d 100644 --- a/state/protocol/badger/snapshot_test.go +++ b/state/protocol/badger/snapshot_test.go @@ -26,10 +26,6 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func init() { - -} - // TestUnknownReferenceBlock tests queries for snapshots which should be unknown. // We use this fixture: // - Root height: 100 diff --git a/storage/badger/cleaner.go b/storage/badger/cleaner.go index b055751718e..b57668c825d 100644 --- a/storage/badger/cleaner.go +++ b/storage/badger/cleaner.go @@ -83,7 +83,12 @@ func (c *Cleaner) gcWorkerRoutine(ctx irrecoverable.SignalerContext, ready compo // Therefore GC is run every X seconds, where X is uniformly sampled from [interval, interval*1.2] func (c *Cleaner) nextWaitDuration() time.Duration { jitter, err := rand.Uint64n(uint64(c.interval.Nanoseconds() / 5)) - if err != nil { // if randomness fails, do not use a jitter for this instance. + if err != nil { + // if randomness fails, do not use a jitter for this instance. + // TODO: address the error properly and not swallow it. + // In this specific case, `utils/rand` only errors if the system randomness fails + // which is a symptom of a wider failure. Many other node components would catch such + // a failure. jitter = 0 } return time.Duration(c.interval.Nanoseconds() + int64(jitter)) diff --git a/storage/badger/operation/common_test.go b/storage/badger/operation/common_test.go index 5351518fda6..65f64fbd5cb 100644 --- a/storage/badger/operation/common_test.go +++ b/storage/badger/operation/common_test.go @@ -18,10 +18,6 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func init() { - -} - type Entity struct { ID uint64 } From 60a6b9d842d796c115c6726940c29eb7e6b63f51 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 1 Jun 2023 18:02:20 -0600 Subject: [PATCH 033/169] bump crypto module to 0.24.8 --- go.mod | 2 +- go.sum | 4 ++-- insecure/go.mod | 3 ++- insecure/go.sum | 20 ++++++++++++++++++-- integration/go.mod | 3 ++- integration/go.sum | 5 +++-- 6 files changed, 28 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 602fb4c15fd..a0b7088f56c 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.40.0 - github.com/onflow/flow-go/crypto v0.24.7 + github.com/onflow/flow-go/crypto v0.24.8 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 diff --git a/go.sum b/go.sum index ed305eed14f..e862d3b080a 100644 --- a/go.sum +++ b/go.sum @@ -1236,8 +1236,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtx github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-go/crypto v0.24.8 h1:VCzkDz+v6Fig+g+QcUVh9c+DyehjlZvXXquUMwOvCKM= +github.com/onflow/flow-go/crypto v0.24.8/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e h1:QYEd3KWTt309YGBch4IGK6vJ6b7cOGx2NStEnd5NeHM= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= diff --git a/insecure/go.mod b/insecure/go.mod index 73398c2b192..5cae139e6a0 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -10,7 +10,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.8.2 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/onflow/flow-go v0.29.8 - github.com/onflow/flow-go/crypto v0.24.7 + github.com/onflow/flow-go/crypto v0.24.8 github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.2 @@ -257,6 +257,7 @@ require ( golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gonum.org/v1/gonum v0.8.2 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 129d83cb596..6e46b705a0c 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -85,6 +85,7 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -304,6 +305,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -391,6 +393,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -723,6 +726,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= @@ -1184,8 +1188,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtx github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-go/crypto v0.24.8 h1:VCzkDz+v6Fig+g+QcUVh9c+DyehjlZvXXquUMwOvCKM= +github.com/onflow/flow-go/crypto v0.24.8/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e h1:QYEd3KWTt309YGBch4IGK6vJ6b7cOGx2NStEnd5NeHM= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= @@ -1590,7 +1594,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -1603,6 +1610,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1833,12 +1841,14 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1904,7 +1914,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -2090,6 +2105,7 @@ nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/integration/go.mod b/integration/go.mod index 478283c6530..f7373eaf5ae 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -23,7 +23,7 @@ require ( github.com/onflow/flow-emulator v0.48.1-0.20230502171545-1c91ebbf6870 github.com/onflow/flow-go v0.30.1-0.20230501182206-6a911be58b92 github.com/onflow/flow-go-sdk v0.40.0 - github.com/onflow/flow-go/crypto v0.24.7 + github.com/onflow/flow-go/crypto v0.24.8 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e github.com/plus3it/gorecurcopy v0.0.1 @@ -307,6 +307,7 @@ require ( golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gonum.org/v1/gonum v0.11.0 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect diff --git a/integration/go.sum b/integration/go.sum index 5aa4af7288b..93d189e7a98 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1316,8 +1316,8 @@ github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtx github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QEl4Xw= github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= -github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= -github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-go/crypto v0.24.8 h1:VCzkDz+v6Fig+g+QcUVh9c+DyehjlZvXXquUMwOvCKM= +github.com/onflow/flow-go/crypto v0.24.8/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e h1:QYEd3KWTt309YGBch4IGK6vJ6b7cOGx2NStEnd5NeHM= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= @@ -2141,6 +2141,7 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= From 234ff25c27d7b91f480aae94596d0535f0e880c1 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 1 Jun 2023 18:10:08 -0600 Subject: [PATCH 034/169] use flow-go/crypto randomness evaluation tools in flow-go --- fvm/environment/programs_test.go | 2 +- .../unsafe_random_generator_test.go | 37 +------------ utils/rand/rand_test.go | 55 ++++--------------- 3 files changed, 14 insertions(+), 80 deletions(-) diff --git a/fvm/environment/programs_test.go b/fvm/environment/programs_test.go index d6016f08dd0..55c3969f9a5 100644 --- a/fvm/environment/programs_test.go +++ b/fvm/environment/programs_test.go @@ -187,7 +187,7 @@ func Test_Programs(t *testing.T) { }) t.Run("register touches are captured for simple contract A", func(t *testing.T) { - fmt.Println("---------- Real transaction here ------------") + t.Log("---------- Real transaction here ------------") // run a TX using contract A diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/unsafe_random_generator_test.go index bb6f13b87e0..137d593db08 100644 --- a/fvm/environment/unsafe_random_generator_test.go +++ b/fvm/environment/unsafe_random_generator_test.go @@ -1,52 +1,19 @@ package environment_test import ( - "fmt" "math" mrand "math/rand" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gonum.org/v1/gonum/stat" + "github.com/onflow/flow-go/crypto/random" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) -// TODO: these functions are copied from flow-go/crypto/rand -// Once the new flow-go/crypto/ module version is tagged, flow-go would upgrade -// to the new version and import these functions -func BasicDistributionTest(t *testing.T, n uint64, classWidth uint64, randf func() (uint64, error)) { - // sample size should ideally be a high number multiple of `n` - // but if `n` is too small, we could use a small sample size so that the test - // isn't too slow - sampleSize := 1000 * n - if n < 100 { - sampleSize = (80000 / n) * n // highest multiple of n less than 80000 - } - distribution := make([]float64, n) - // populate the distribution - for i := uint64(0); i < sampleSize; i++ { - r, err := randf() - require.NoError(t, err) - if n*classWidth != 0 { - require.Less(t, r, n*classWidth) - } - distribution[r/classWidth] += 1.0 - } - EvaluateDistributionUniformity(t, distribution) -} - -func EvaluateDistributionUniformity(t *testing.T, distribution []float64) { - tolerance := 0.05 - stdev := stat.StdDev(distribution, nil) - mean := stat.Mean(distribution, nil) - assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed: n: %d, stdev: %v, mean: %v", len(distribution), stdev, mean)) -} - func TestUnsafeRandomGenerator(t *testing.T) { bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) @@ -78,7 +45,7 @@ func TestUnsafeRandomGenerator(t *testing.T) { // n is a random power of 2 (from 2 to 2^10) n := 1 << (1 + mrand.Intn(10)) classWidth := (math.MaxUint64 / uint64(n)) + 1 - BasicDistributionTest(t, uint64(n), uint64(classWidth), urg.UnsafeRandom) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), urg.UnsafeRandom) } }) diff --git a/utils/rand/rand_test.go b/utils/rand/rand_test.go index 14f00559d62..73d7ca539ca 100644 --- a/utils/rand/rand_test.go +++ b/utils/rand/rand_test.go @@ -1,49 +1,16 @@ package rand import ( - "fmt" "math" mrand "math/rand" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gonum.org/v1/gonum/stat" - _ "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/crypto/random" ) -// TODO: these functions are copied from flow-go/crypto/rand -// Once the new flow-go/crypto/ module version is tagged, flow-go would upgrade -// to the new version and import these functions -func BasicDistributionTest(t *testing.T, n uint64, classWidth uint64, randf func() (uint64, error)) { - // sample size should ideally be a high number multiple of `n` - // but if `n` is too small, we could use a small sample size so that the test - // isn't too slow - sampleSize := 1000 * n - if n < 100 { - sampleSize = (80000 / n) * n // highest multiple of n less than 80000 - } - distribution := make([]float64, n) - // populate the distribution - for i := uint64(0); i < sampleSize; i++ { - r, err := randf() - require.NoError(t, err) - if n*classWidth != 0 { - require.Less(t, r, n*classWidth) - } - distribution[r/classWidth] += 1.0 - } - EvaluateDistributionUniformity(t, distribution) -} - -func EvaluateDistributionUniformity(t *testing.T, distribution []float64) { - tolerance := 0.05 - stdev := stat.StdDev(distribution, nil) - mean := stat.Mean(distribution, nil) - assert.Greater(t, tolerance*mean, stdev, fmt.Sprintf("basic randomness test failed: n: %d, stdev: %v, mean: %v", len(distribution), stdev, mean)) -} - func TestRandomIntegers(t *testing.T) { t.Run("basic uniformity", func(t *testing.T) { @@ -56,7 +23,7 @@ func TestRandomIntegers(t *testing.T) { r, err := Uint() return uint64(r), err } - BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) }) t.Run("Uint64", func(t *testing.T) { @@ -64,7 +31,7 @@ func TestRandomIntegers(t *testing.T) { // n is a random power of 2 (from 2 to 2^10) n := 1 << (1 + mrand.Intn(10)) classWidth := (math.MaxUint64 / uint64(n)) + 1 - BasicDistributionTest(t, uint64(n), uint64(classWidth), Uint64) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), Uint64) }) t.Run("Uint32", func(t *testing.T) { @@ -76,7 +43,7 @@ func TestRandomIntegers(t *testing.T) { r, err := Uint32() return uint64(r), err } - BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(classWidth), uintf) }) t.Run("Uintn", func(t *testing.T) { @@ -86,7 +53,7 @@ func TestRandomIntegers(t *testing.T) { return uint64(r), err } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) t.Run("Uint64n", func(t *testing.T) { @@ -95,7 +62,7 @@ func TestRandomIntegers(t *testing.T) { return Uint64n(uint64(n)) } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) t.Run("Uint32n", func(t *testing.T) { @@ -105,7 +72,7 @@ func TestRandomIntegers(t *testing.T) { return uint64(r), err } // classWidth is 1 since `n` is small - BasicDistributionTest(t, uint64(n), uint64(1), uintf) + random.BasicDistributionTest(t, uint64(n), uint64(1), uintf) }) }) @@ -169,7 +136,7 @@ func TestShuffle(t *testing.T) { } // if the shuffle is uniform, the test element // should end up uniformly in all positions of the slice - EvaluateDistributionUniformity(t, distribution) + random.EvaluateDistributionUniformity(t, distribution) }) t.Run("shuffle a same permutation", func(t *testing.T) { @@ -182,7 +149,7 @@ func TestShuffle(t *testing.T) { } // if the shuffle is uniform, the test element // should end up uniformly in all positions of the slice - EvaluateDistributionUniformity(t, distribution) + random.EvaluateDistributionUniformity(t, distribution) }) }) @@ -232,10 +199,10 @@ func TestSamples(t *testing.T) { } // if the sampling is uniform, all elements // should end up being sampled an equivalent number of times - EvaluateDistributionUniformity(t, samplingDistribution) + random.EvaluateDistributionUniformity(t, samplingDistribution) // if the sampling is uniform, the test element // should end up uniformly in all positions of the sample slice - EvaluateDistributionUniformity(t, orderingDistribution) + random.EvaluateDistributionUniformity(t, orderingDistribution) }) t.Run("zero edge cases", func(t *testing.T) { From 2944aee0d304887382b8a8221e3df6a94b313f61 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 1 Jun 2023 20:25:56 -0600 Subject: [PATCH 035/169] replace tx index by tx ID in seed derivation --- fvm/environment/facade_env.go | 2 +- fvm/environment/unsafe_random_generator.go | 11 ++++----- .../unsafe_random_generator_test.go | 23 +++++++++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index d45fcdd5b6f..0e36b9ce8c0 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -77,7 +77,7 @@ func newFacadeEnvironment( UnsafeRandomGenerator: NewUnsafeRandomGenerator( tracer, params.BlockHeader, - params.TxIndex, + params.TxId, ), CryptoLibrary: NewCryptoLibrary(tracer, meter), diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index 548753d90ca..aa931905e54 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -27,7 +27,7 @@ type unsafeRandomGenerator struct { tracer tracing.TracerSpan blockHeader *flow.Header - txnIndex uint32 + txId flow.Identifier prg random.Rand createOnce sync.Once @@ -62,12 +62,12 @@ func (gen ParseRestrictedUnsafeRandomGenerator) UnsafeRandom() ( func NewUnsafeRandomGenerator( tracer tracing.TracerSpan, blockHeader *flow.Header, - txnIndex uint32, + txId flow.Identifier, ) UnsafeRandomGenerator { gen := &unsafeRandomGenerator{ tracer: tracer, blockHeader: blockHeader, - txnIndex: txnIndex, + txId: txId, } return gen @@ -86,9 +86,8 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( // source than the block ID) source := gen.blockHeader.ID() - // Provide additional randomness for each transaction. - salt := make([]byte, 4) - binary.LittleEndian.PutUint32(salt, gen.txnIndex) + // Diversify the seed per transaction ID + salt := gen.txId[:] // Extract the entropy from the source and expand it into the required // seed length. Note that we can use any implementation which provide diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/unsafe_random_generator_test.go index 137d593db08..8874fa4477a 100644 --- a/fvm/environment/unsafe_random_generator_test.go +++ b/fvm/environment/unsafe_random_generator_test.go @@ -17,12 +17,12 @@ import ( func TestUnsafeRandomGenerator(t *testing.T) { bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) - getRandoms := func(txnIndex uint32, N int) []uint64 { + getRandoms := func(txId flow.Identifier, N int) []uint64 { // seed the RG with the same block header urg := environment.NewUnsafeRandomGenerator( tracing.NewTracerSpan(), bh, - txnIndex) + txId) numbers := make([]uint64, N) for i := 0; i < N; i++ { u, err := urg.UnsafeRandom() @@ -35,11 +35,12 @@ func TestUnsafeRandomGenerator(t *testing.T) { // basic randomness test to check outputs are "uniformly" spread over the // output space t.Run("randomness test", func(t *testing.T) { - for txnIndex := uint32(0); txnIndex < 10; txnIndex++ { + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() urg := environment.NewUnsafeRandomGenerator( tracing.NewTracerSpan(), bh, - txnIndex) + txId) // make sure n is a power of 2 so that there is no bias in the last class // n is a random power of 2 (from 2 to 2^10) @@ -51,19 +52,21 @@ func TestUnsafeRandomGenerator(t *testing.T) { // tests that unsafeRandom is PRG based and hence has deterministic outputs. t.Run("PRG-based UnsafeRandom", func(t *testing.T) { - for txnIndex := uint32(0); txnIndex < 10; txnIndex++ { + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() N := 100 - r1 := getRandoms(txnIndex, N) - r2 := getRandoms(txnIndex, N) + r1 := getRandoms(txId, N) + r2 := getRandoms(txId, N) require.Equal(t, r1, r2) } }) t.Run("transaction specific randomness", func(t *testing.T) { txns := [][]uint64{} - for txnIndex := uint32(0); txnIndex < 10; txnIndex++ { - N := 100 - txns = append(txns, getRandoms(txnIndex, N)) + for i := 0; i < 10; i++ { + txId := unittest.TransactionFixture().ID() + N := 2 + txns = append(txns, getRandoms(txId, N)) } for i, txn := range txns { From 5a1bca33fd9eac7ef4b64ad3fabac9c1103472a9 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef <50252200+tarakby@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:09:18 -0600 Subject: [PATCH 036/169] use Throw to bubble the error up Co-authored-by: Peter Argue <89119817+peterargue@users.noreply.github.com> --- network/p2p/connection/peerManager.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index d940a56beb9..b8ad6c11f97 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -86,12 +86,10 @@ func (pm *PeerManager) updateLoop(ctx irrecoverable.SignalerContext) { func (pm *PeerManager) periodicLoop(ctx irrecoverable.SignalerContext) { // add a random delay to initial launch to avoid synchronizing this // potentially expensive operation across the network - r, _ := rand.Uint64n(uint64(pm.peerUpdateInterval.Nanoseconds())) - // ignore the error here, if randomness fails `r` would be zero and there will be no delay - // for the current node - // TODO: treat the error properly instead of swallowing it. In this specific case, `utils/rand` - // only errors if there is a randomness system issue. Such issue will cause errors in many - // other components. + r, err := rand.Uint64n(uint64(pm.peerUpdateInterval.Nanoseconds())) + if err != nil { + ctx.Throw(fmt.Errorf("unable to generate random interval: %w", err)) + } delay := time.Duration(r) ticker := time.NewTicker(pm.peerUpdateInterval) From 3c2f5bda231ad281ed7343769a3410692c59aca8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 12 Jun 2023 21:12:58 -0600 Subject: [PATCH 037/169] fix logic change in identity list sample --- model/flow/identity.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/model/flow/identity.go b/model/flow/identity.go index 7127858a379..c44c394cb06 100644 --- a/model/flow/identity.go +++ b/model/flow/identity.go @@ -466,16 +466,15 @@ func (il IdentityList) Sample(size uint) (IdentityList, error) { n := uint(len(il)) dup := make([]*Identity, 0, n) dup = append(dup, il...) - // if sample size is greater than total size, return all the elements - if n <= size { - return dup, nil + if n < size { + size = n } swap := func(i, j uint) { dup[i], dup[j] = dup[j], dup[i] } err := rand.Samples(n, size, swap) if err != nil { - return nil, fmt.Errorf("failed to generate randomness: %w", err) + return nil, fmt.Errorf("failed to sample identity list: %w", err) } return dup[:size], nil } @@ -483,17 +482,7 @@ func (il IdentityList) Sample(size uint) (IdentityList, error) { // Shuffle randomly shuffles the identity list (non-deterministic), // and returns the shuffled list without modifying the receiver. func (il IdentityList) Shuffle() (IdentityList, error) { - n := uint(len(il)) - dup := make([]*Identity, 0, n) - dup = append(dup, il...) - swap := func(i, j uint) { - dup[i], dup[j] = dup[j], dup[i] - } - err := rand.Shuffle(n, swap) - if err != nil { - return nil, fmt.Errorf("failed to generate randomness: %w", err) - } - return dup, nil + return il.Sample(uint(len(il))) } // SamplePct returns a random sample from the receiver identity list. The From 14df98fb47dc34a19ae6a2f03661404b0aacd96f Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 12 Jun 2023 21:14:47 -0600 Subject: [PATCH 038/169] fix returning before unlock and add warning log when randomness fails --- cmd/bootstrap/cmd/clusters.go | 3 +-- engine/execution/computation/query/executor.go | 2 +- storage/badger/cleaner.go | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/bootstrap/cmd/clusters.go b/cmd/bootstrap/cmd/clusters.go index 80ba74ad77a..441f573f429 100644 --- a/cmd/bootstrap/cmd/clusters.go +++ b/cmd/bootstrap/cmd/clusters.go @@ -38,8 +38,7 @@ func constructClusterAssignment(partnerNodes, internalNodes []model.NodeInfo) (f } // shuffle both collector lists based on a non-deterministic algorithm - var err error - partners, err = partners.Shuffle() + partners, err := partners.Shuffle() if err != nil { log.Fatal().Err(err).Msg("could not shuffle partners") } diff --git a/engine/execution/computation/query/executor.go b/engine/execution/computation/query/executor.go index 10a680475a0..7b5a7eb4b35 100644 --- a/engine/execution/computation/query/executor.go +++ b/engine/execution/computation/query/executor.go @@ -113,11 +113,11 @@ func (e *QueryExecutor) ExecuteScript( // TODO: this is a temporary measure, we could remove this in the future if e.logger.Debug().Enabled() { e.rngLock.Lock() + defer e.rngLock.Unlock() trackerID, err := rand.Uint32() if err != nil { return nil, fmt.Errorf("failed to generate trackerID: %w", err) } - e.rngLock.Unlock() trackedLogger := e.logger.With().Hex("script_hex", script).Uint32("trackerID", trackerID).Logger() trackedLogger.Debug().Msg("script is sent for execution") diff --git a/storage/badger/cleaner.go b/storage/badger/cleaner.go index b57668c825d..d9cd07997e7 100644 --- a/storage/badger/cleaner.go +++ b/storage/badger/cleaner.go @@ -89,6 +89,7 @@ func (c *Cleaner) nextWaitDuration() time.Duration { // In this specific case, `utils/rand` only errors if the system randomness fails // which is a symptom of a wider failure. Many other node components would catch such // a failure. + c.log.Warn().Msg("jitter is zero beacuse system randomness has failed") jitter = 0 } return time.Duration(c.interval.Nanoseconds() + int64(jitter)) From 557c6dd0426fb5634ceeb4cbaffcc47873d8dc56 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 16 Jun 2023 11:40:36 +0300 Subject: [PATCH 039/169] Added RestRouter, refactored engine and engine builder --- access/handler.go | 21 ++ apiproxy/access_api_proxy.go | 13 ++ .../node_builder/access_node_builder.go | 17 +- cmd/observer/node_builder/observer_builder.go | 46 ++++- engine/access/apiproxy/access_api_proxy.go | 27 ++- engine/access/backend.go | 93 +++++++++ engine/access/mock/access_api_client.go | 33 ++++ engine/access/mock/access_api_server.go | 26 +++ engine/access/rest/blocks.go | 22 +-- engine/access/rest/models/collection.go | 5 +- engine/access/rest/models/event.go | 1 + engine/access/rest/models/network.go | 1 + .../access/rest/models/node_version_info.go | 1 + engine/access/rest/rest_server_api.go | 182 ++++++++++++++++-- engine/access/rest_api_test.go | 13 +- engine/access/rpc/engine.go | 84 +------- engine/access/rpc/engine_builder.go | 52 +++-- go.mod | 3 + go.sum | 4 +- insecure/go.sum | 2 + integration/go.mod | 2 + integration/go.sum | 4 +- module/forwarder/forwarder.go | 1 + module/mock/grpc_connection_pool_metrics.go | 60 ++++++ 24 files changed, 571 insertions(+), 142 deletions(-) create mode 100644 engine/access/backend.go create mode 100644 module/mock/grpc_connection_pool_metrics.go diff --git a/access/handler.go b/access/handler.go index 404bfa81318..2417d4037ac 100644 --- a/access/handler.go +++ b/access/handler.go @@ -590,6 +590,27 @@ func (h *Handler) GetExecutionResultForBlockID(ctx context.Context, req *access. return executionResultToMessages(result, metadata) } +// GetExecutionResultByID returns the execution result for the given ID. +func (h *Handler) GetExecutionResultByID(ctx context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + metadata := h.buildMetadataResponse() + + blockID := convert.MessageToIdentifier(req.GetId()) + + result, err := h.api.GetExecutionResultByID(ctx, blockID) + if err != nil { + return nil, err + } + + execResult, err := convert.ExecutionResultToMessage(result) + if err != nil { + return nil, err + } + return &access.ExecutionResultByIDResponse{ + ExecutionResult: execResult, + Metadata: metadata, + }, nil +} + func (h *Handler) blockResponse(block *flow.Block, fullResponse bool, status flow.BlockStatus) (*access.BlockResponse, error) { metadata := h.buildMetadataResponse() diff --git a/apiproxy/access_api_proxy.go b/apiproxy/access_api_proxy.go index 8e0b781af5e..c61e8e52d69 100644 --- a/apiproxy/access_api_proxy.go +++ b/apiproxy/access_api_proxy.go @@ -248,6 +248,10 @@ func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Conte return h.upstream.GetExecutionResultForBlockID(context, req) } +func (h *FlowAccessAPIRouter) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + return h.upstream.GetExecutionResultByID(context, req) +} + // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { lock sync.Mutex @@ -466,3 +470,12 @@ func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Co } return upstream.GetExecutionResultForBlockID(context, req) } + +func (h *FlowAccessAPIForwarder) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + // This is a passthrough request + upstream, err := h.faultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetExecutionResultByID(context, req) +} diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 03625fe8f50..0482b600e66 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -35,6 +35,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" + accessengine "github.com/onflow/flow-go/engine/access" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" "github.com/onflow/flow-go/engine/access/rpc" @@ -991,8 +992,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - engineBuilder, err := rpc.NewBuilder( - node.Logger, + backend, err := accessengine.NewBackend(node.Logger, node.State, builder.rpcConf, builder.CollectionRPC, @@ -1007,11 +1007,22 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.AccessMetrics, builder.collectionGRPCPort, builder.executionGRPCPort, - builder.retryEnabled, + builder.retryEnabled) + if err != nil { + return nil, fmt.Errorf("could not initialize backend: %w", err) + } + + engineBuilder, err := rpc.NewBuilder( + node.Logger, + node.State, + builder.rpcConf, + node.RootChainID, + builder.AccessMetrics, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits, builder.Me, + backend, ) if err != nil { return nil, err diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 8b06825b52e..2b82705168d 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" @@ -27,7 +28,9 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" + accessengine "github.com/onflow/flow-go/engine/access" "github.com/onflow/flow-go/engine/access/apiproxy" + "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/follower" @@ -847,8 +850,8 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - engineBuilder, err := rpc.NewBuilder( - node.Logger, + accessMetrics := metrics.NewNoopCollector() + accessBackend, err := accessengine.NewBackend(node.Logger, node.State, builder.rpcConf, nil, @@ -860,14 +863,25 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { node.Storage.Receipts, node.Storage.Results, node.RootChainID, - metrics.NewNoopCollector(), + accessMetrics, 0, 0, - false, + false) + if err != nil { + return nil, fmt.Errorf("could not initialize backend: %w", err) + } + + engineBuilder, err := rpc.NewBuilder( + node.Logger, + node.State, + builder.rpcConf, + node.RootChainID, + accessMetrics, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits, builder.Me, + accessBackend, ) if err != nil { return nil, err @@ -879,9 +893,11 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return nil, err } - proxy := &apiproxy.FlowAccessAPIRouter{ + metrics := metrics.NewObserverCollector() + + rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, - Metrics: metrics.NewObserverCollector(), + Metrics: metrics, Upstream: forwarder, Observer: protocol.NewHandler(protocol.New( node.State, @@ -891,9 +907,25 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { )), } + restForwarder, err := rest.NewRestForwarder(builder.Logger, + builder.upstreamIdentities, + builder.apiTimeout, + builder.rpcConf.MaxMsgSize) + if err != nil { + return nil, err + } + + restHandler := &rest.RestRouter{ + Logger: builder.Logger, + Metrics: metrics, + Upstream: restForwarder, + Observer: rest.NewRequestHandler(builder.Logger, accessBackend), + } + // build the rpc engine builder.RpcEng, err = engineBuilder. - WithNewHandler(proxy). + WithRpcHandler(rpcHandler). + WithRestHandler(restHandler). WithLegacy(). Build() if err != nil { diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index 6f0925667ea..08136531b09 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -199,17 +199,27 @@ func (h *FlowAccessAPIRouter) GetExecutionResultForBlockID(context context.Conte return res, err } +func (h *FlowAccessAPIRouter) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + res, err := h.Upstream.GetExecutionResultByID(context, req) + h.log("upstream", "GetExecutionResultByID", err) + return res, err +} + // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { forwarder.Forwarder } func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*FlowAccessAPIForwarder, error) { - commonForwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + forwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + if err != nil { + return nil, err + } - forwarder := &FlowAccessAPIForwarder{} - forwarder.Forwarder = commonForwarder - return forwarder, err + accessApiForwarder := &FlowAccessAPIForwarder{ + Forwarder: forwarder, + } + return accessApiForwarder, nil } // Ping pings the service. It is special in the sense that it responds successful, @@ -446,3 +456,12 @@ func (h *FlowAccessAPIForwarder) GetExecutionResultForBlockID(context context.Co } return upstream.GetExecutionResultForBlockID(context, req) } + +func (h *FlowAccessAPIForwarder) GetExecutionResultByID(context context.Context, req *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + // This is a passthrough request + upstream, err := h.FaultTolerantClient() + if err != nil { + return nil, err + } + return upstream.GetExecutionResultByID(context, req) +} diff --git a/engine/access/backend.go b/engine/access/backend.go new file mode 100644 index 00000000000..92c8f932b2d --- /dev/null +++ b/engine/access/backend.go @@ -0,0 +1,93 @@ +package access + +import ( + "fmt" + + lru "github.com/hashicorp/golang-lru" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" +) + +func NewBackend( + log zerolog.Logger, + state protocol.State, + config rpc.Config, + collectionRPC accessproto.AccessAPIClient, + historicalAccessNodes []accessproto.AccessAPIClient, + blocks storage.Blocks, + headers storage.Headers, + collections storage.Collections, + transactions storage.Transactions, + executionReceipts storage.ExecutionReceipts, + executionResults storage.ExecutionResults, + chainID flow.ChainID, + accessMetrics module.AccessMetrics, + collectionGRPCPort uint, + executionGRPCPort uint, + retryEnabled bool) (*backend.Backend, error) { + + var cache *lru.Cache + cacheSize := config.ConnectionPoolSize + if cacheSize > 0 { + // TODO: remove this fallback after fixing issues with evictions + // It was observed that evictions cause connection errors for in flight requests. This works around + // the issue by forcing hte pool size to be greater than the number of ENs + LNs + if cacheSize < backend.DefaultConnectionPoolSize { + log.Warn().Msg("connection pool size below threshold, setting pool size to default value ") + cacheSize = backend.DefaultConnectionPoolSize + } + var err error + cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { + store := evictedValue.(*backend.CachedClient) + store.Close() + log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") + if accessMetrics != nil { + accessMetrics.ConnectionFromPoolEvicted() + } + }) + if err != nil { + return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) + } + } + + connectionFactory := &backend.ConnectionFactoryImpl{ + CollectionGRPCPort: collectionGRPCPort, + ExecutionGRPCPort: executionGRPCPort, + CollectionNodeGRPCTimeout: config.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, + ConnectionsCache: cache, + CacheSize: cacheSize, + MaxMsgSize: config.MaxMsgSize, + AccessMetrics: accessMetrics, + Log: log, + } + + return backend.New(state, + collectionRPC, + historicalAccessNodes, + blocks, + headers, + collections, + transactions, + executionReceipts, + executionResults, + chainID, + accessMetrics, + connectionFactory, + retryEnabled, + config.MaxHeightRange, + config.PreferredExecutionNodeIDs, + config.FixedExecutionNodeIDs, + log, + backend.DefaultSnapshotHistoryLimit, + config.ArchiveAddressList, + ), nil +} diff --git a/engine/access/mock/access_api_client.go b/engine/access/mock/access_api_client.go index 234e4ffcdee..4e2b1d065c7 100644 --- a/engine/access/mock/access_api_client.go +++ b/engine/access/mock/access_api_client.go @@ -446,6 +446,39 @@ func (_m *AccessAPIClient) GetEventsForHeightRange(ctx context.Context, in *acce return r0, r1 } +// GetExecutionResultByID provides a mock function with given fields: ctx, in, opts +func (_m *AccessAPIClient) GetExecutionResultByID(ctx context.Context, in *access.GetExecutionResultByIDRequest, opts ...grpc.CallOption) (*access.ExecutionResultByIDResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + var r0 *access.ExecutionResultByIDResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) (*access.ExecutionResultByIDResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) *access.ExecutionResultByIDResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.ExecutionResultByIDResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetExecutionResultByIDRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetExecutionResultForBlockID provides a mock function with given fields: ctx, in, opts func (_m *AccessAPIClient) GetExecutionResultForBlockID(ctx context.Context, in *access.GetExecutionResultForBlockIDRequest, opts ...grpc.CallOption) (*access.ExecutionResultForBlockIDResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/engine/access/mock/access_api_server.go b/engine/access/mock/access_api_server.go index 5515698eacd..1a2c3772e44 100644 --- a/engine/access/mock/access_api_server.go +++ b/engine/access/mock/access_api_server.go @@ -353,6 +353,32 @@ func (_m *AccessAPIServer) GetEventsForHeightRange(_a0 context.Context, _a1 *acc return r0, r1 } +// GetExecutionResultByID provides a mock function with given fields: _a0, _a1 +func (_m *AccessAPIServer) GetExecutionResultByID(_a0 context.Context, _a1 *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error) { + ret := _m.Called(_a0, _a1) + + var r0 *access.ExecutionResultByIDResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest) (*access.ExecutionResultByIDResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *access.GetExecutionResultByIDRequest) *access.ExecutionResultByIDResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.ExecutionResultByIDResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *access.GetExecutionResultByIDRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetExecutionResultForBlockID provides a mock function with given fields: _a0, _a1 func (_m *AccessAPIServer) GetExecutionResultForBlockID(_a0 context.Context, _a1 *access.GetExecutionResultForBlockIDRequest) (*access.ExecutionResultForBlockIDResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/engine/access/rest/blocks.go b/engine/access/rest/blocks.go index 1f24f44d717..9adcf0fec14 100644 --- a/engine/access/rest/blocks.go +++ b/engine/access/rest/blocks.go @@ -3,7 +3,6 @@ package rest import ( "context" "fmt" - "net/http" "google.golang.org/grpc/codes" @@ -14,6 +13,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" ) @@ -76,9 +76,9 @@ func getBlock(option blockRequestOption, context context.Context, expandFields m return &block, nil } -func getForwarderBlock(option blockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { +func getBlockFromGrpc(option blockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { // lookup block - blkProvider := NewBlockForwarderProvider(upstream, option) + blkProvider := NewBlockFromGrpcProvider(upstream, option) blk, blockStatus, err := blkProvider.getBlock(context) if err != nil { return nil, err @@ -164,7 +164,7 @@ func forFinalized(queryParam uint64) blockRequestOption { } } -// blockProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to +// blockRequestProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to // look up a block or a block header either by ID or by height type blockRequestProvider struct { blockRequest @@ -211,25 +211,25 @@ func (blkProvider *blockRequestProvider) getBlock(ctx context.Context) (*flow.Bl return blk, status, nil } -// blockProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to +// blockFromGrpcProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to // look up a block or a block header either by ID or by height -type blockForwarderProvider struct { +type blockFromGrpcProvider struct { blockRequest upstream accessproto.AccessAPIClient } -func NewBlockForwarderProvider(upstream accessproto.AccessAPIClient, options ...blockRequestOption) *blockForwarderProvider { - blockForwarderProvider := &blockForwarderProvider{ +func NewBlockFromGrpcProvider(upstream accessproto.AccessAPIClient, options ...blockRequestOption) *blockFromGrpcProvider { + blockFromGrpcProvider := &blockFromGrpcProvider{ upstream: upstream, } for _, o := range options { - o(&blockForwarderProvider.blockRequest) + o(&blockFromGrpcProvider.blockRequest) } - return blockForwarderProvider + return blockFromGrpcProvider } -func (blkProvider *blockForwarderProvider) getBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { +func (blkProvider *blockFromGrpcProvider) getBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { if blkProvider.id != nil { getBlockByIdRequest := &accessproto.GetBlockByIDRequest{ Id: []byte(blkProvider.id.String()), diff --git a/engine/access/rest/models/collection.go b/engine/access/rest/models/collection.go index b42982a4061..d4979146c7c 100644 --- a/engine/access/rest/models/collection.go +++ b/engine/access/rest/models/collection.go @@ -7,6 +7,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow/protobuf/go/flow/entities" ) @@ -69,10 +70,6 @@ func (c *Collection) BuildFromGrpc( var expandable CollectionExpandable var transactions Transactions if expand[ExpandsTransactions] { - var txIds []flow.Identifier - for _, id := range collection.TransactionIds { - txIds = append(txIds, convert.MessageToIdentifier(id)) - } transactions.Build(transactionsBody, link) } else { expandable.Transactions = make([]string, len(collection.TransactionIds)) diff --git a/engine/access/rest/models/event.go b/engine/access/rest/models/event.go index 8cfb3457dc0..d829ec862dc 100644 --- a/engine/access/rest/models/event.go +++ b/engine/access/rest/models/event.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) diff --git a/engine/access/rest/models/network.go b/engine/access/rest/models/network.go index 584e2a73e89..1a6dd9a9816 100644 --- a/engine/access/rest/models/network.go +++ b/engine/access/rest/models/network.go @@ -2,6 +2,7 @@ package models import ( "github.com/onflow/flow-go/access" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) diff --git a/engine/access/rest/models/node_version_info.go b/engine/access/rest/models/node_version_info.go index f51878f158d..782493c0ec9 100644 --- a/engine/access/rest/models/node_version_info.go +++ b/engine/access/rest/models/node_version_info.go @@ -5,6 +5,7 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow/protobuf/go/flow/entities" ) diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go index 9d1f3d1815f..19d6c00440e 100644 --- a/engine/access/rest/rest_server_api.go +++ b/engine/access/rest/rest_server_api.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "google.golang.org/grpc/status" + "github.com/rs/zerolog" "github.com/onflow/flow-go/access" @@ -13,11 +15,123 @@ import ( "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/forwarder" - "github.com/onflow/flow/protobuf/go/flow/entities" + "github.com/onflow/flow-go/module/metrics" accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" ) +// RestRouter is a structure that represents the routing proxy algorithm. +// It splits requests between a local and a remote rest service. +type RestRouter struct { + Logger zerolog.Logger + Metrics *metrics.ObserverCollector + Upstream *RestForwarder + Observer *RequestHandler +} + +func (r *RestRouter) log(handler, rpc string, err error) { + code := status.Code(err) + r.Metrics.RecordRPC(handler, rpc, code) + + logger := r.Logger.With(). + Str("handler", handler). + Str("rest_method", rpc). + Str("rest_code", code.String()). + Logger() + + if err != nil { + logger.Error().Err(err).Msg("request failed") + return + } + + logger.Info().Msg("request succeeded") +} + +func (r *RestRouter) GetTransactionByID(req request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { + res, err := r.Upstream.GetTransactionByID(req, context, link, chain) + r.log("upstream", "GetNodeVersionInfo", err) + return res, err +} + +func (r *RestRouter) CreateTransaction(req request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + res, err := r.Upstream.CreateTransaction(req, context, link) + r.log("upstream", "CreateTransaction", err) + return res, err +} + +func (r *RestRouter) GetTransactionResultByID(req request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + res, err := r.Upstream.GetTransactionResultByID(req, context, link) + r.log("upstream", "GetTransactionResultByID", err) + return res, err +} + +func (r *RestRouter) GetBlocksByIDs(req request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + res, err := r.Observer.GetBlocksByIDs(req, context, expandFields, link) + r.log("observer", "GetBlocksByIDs", err) + return res, err +} + +func (r *RestRouter) GetBlocksByHeight(req *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + res, err := r.Observer.GetBlocksByHeight(req, link) + r.log("observer", "GetBlocksByHeight", err) + return res, err +} + +func (r *RestRouter) GetBlockPayloadByID(req request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) { + res, err := r.Observer.GetBlockPayloadByID(req, context, link) + r.log("observer", "GetBlockPayloadByID", err) + return res, err +} + +func (r *RestRouter) GetExecutionResultByID(req request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + res, err := r.Upstream.GetExecutionResultByID(req, context, link) + r.log("upstream", "GetExecutionResultByID", err) + return res, err +} + +func (r *RestRouter) GetExecutionResultsByBlockIDs(req request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + res, err := r.Upstream.GetExecutionResultsByBlockIDs(req, context, link) + r.log("upstream", "GetExecutionResultsByBlockIDs", err) + return res, err +} + +func (r *RestRouter) GetCollectionByID(req request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { + res, err := r.Upstream.GetCollectionByID(req, context, expandFields, link, chain) + r.log("upstream", "GetCollectionByID", err) + return res, err +} + +func (r *RestRouter) ExecuteScript(req request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { + res, err := r.Upstream.ExecuteScript(req, context, link) + r.log("upstream", "ExecuteScript", err) + return res, err +} + +func (r *RestRouter) GetAccount(req request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + res, err := r.Upstream.GetAccount(req, context, expandFields, link) + r.log("upstream", "GetAccount", err) + return res, err +} + +func (r *RestRouter) GetEvents(req request.GetEvents, context context.Context) (models.BlocksEvents, error) { + res, err := r.Upstream.GetEvents(req, context) + r.log("upstream", "GetEvents", err) + return res, err +} + +func (r *RestRouter) GetNetworkParameters(req *request.Request) (models.NetworkParameters, error) { + res, err := r.Observer.GetNetworkParameters(req) + r.log("observer", "GetNetworkParameters", err) + return res, err +} + +func (r *RestRouter) GetNodeVersionInfo(req *request.Request) (models.NodeVersionInfo, error) { + res, err := r.Observer.GetNodeVersionInfo(req) + r.log("observer", "GetNodeVersionInfo", err) + return res, err +} + type RestServerApi interface { // GetTransactionByID gets a transaction by requested ID. GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) @@ -55,8 +169,16 @@ type RequestHandler struct { backend access.API } +//// NewRequestHandler returns new RequestHandler. +//func NewRequestHandler(log zerolog.Logger, backend access.API) RestServerApi { +// return &RequestHandler{ +// log: log, +// backend: backend, +// } +//} + // NewRequestHandler returns new RequestHandler. -func NewRequestHandler(log zerolog.Logger, backend access.API) RestServerApi { +func NewRequestHandler(log zerolog.Logger, backend access.API) *RequestHandler { return &RequestHandler{ log: log, backend: backend, @@ -384,14 +506,14 @@ type RestForwarder struct { } // NewRestForwarder returns new RestForwarder. -func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (RestServerApi, error) { - commonForwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) +func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*RestForwarder, error) { + forwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) - forwarder := &RestForwarder{ + restForwarder := &RestForwarder{ log: log, } - forwarder.Forwarder = commonForwarder - return forwarder, err + restForwarder.Forwarder = forwarder + return restForwarder, err } // GetTransactionByID gets a transaction by requested ID. @@ -492,7 +614,7 @@ func (f *RestForwarder) GetBlocksByIDs(r request.GetBlockByIDs, context context. } for i, id := range r.IDs { - block, err := getForwarderBlock(forID(&id), context, expandFields, upstream, link) + block, err := getBlockFromGrpc(forID(&id), context, expandFields, upstream, link) if err != nil { return nil, err } @@ -515,7 +637,7 @@ func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGe } if req.FinalHeight || req.SealedHeight { - block, err := getForwarderBlock(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) + block, err := getBlockFromGrpc(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) if err != nil { return nil, err } @@ -527,7 +649,7 @@ func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGe if req.HasHeights() { blocks := make([]*models.Block, len(req.Heights)) for i, height := range req.Heights { - block, err := getForwarderBlock(forHeight(height), r.Context(), r.ExpandFields, upstream, link) + block, err := getBlockFromGrpc(forHeight(height), r.Context(), r.ExpandFields, upstream, link) if err != nil { return nil, err } @@ -557,7 +679,7 @@ func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGe blocks := make([]*models.Block, 0) // start and end height inclusive for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := getForwarderBlock(forHeight(i), r.Context(), r.ExpandFields, upstream, link) + block, err := getBlockFromGrpc(forHeight(i), r.Context(), r.ExpandFields, upstream, link) if err != nil { return nil, err } @@ -576,7 +698,7 @@ func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context c return payload, err } - blkProvider := NewBlockForwarderProvider(upstream, forID(&r.ID)) + blkProvider := NewBlockFromGrpcProvider(upstream, forID(&r.ID)) block, _, statusErr := blkProvider.getBlock(context) if statusErr != nil { return payload, statusErr @@ -596,8 +718,38 @@ func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context c } // GetExecutionResultByID gets execution result by the ID. -func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { /**/ - panic("Need to be implemented after grpc call added") +func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + var response models.ExecutionResult + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ + Id: r.ID[:], + } + + executionResultByIDResponse, err := upstream.GetExecutionResultByID(context, executionResultByIDRequest) + if err != nil { + return response, err + } + + if executionResultByIDResponse == nil { + err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) + return response, NewNotFoundError(err.Error(), err) + } + + flowExecResult, err := convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) + if err != nil { + return response, err + } + err = response.Build(flowExecResult, link) + if err != nil { + return response, err + } + + return response, nil } // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. @@ -677,7 +829,7 @@ func (f *RestForwarder) GetCollectionByID(r request.GetCollection, context conte } // ExecuteScript handler sends the script from the request to be executed. -func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { +func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { upstream, err := f.FaultTolerantClient() if err != nil { return nil, err diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 5ee8f6d9730..4cb5c5d7c94 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -118,7 +118,7 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } - rpcEngBuilder, err := rpc.NewBuilder( + backend, err := NewBackend( suite.log, suite.state, config, @@ -134,11 +134,20 @@ func (suite *RestAPITestSuite) SetupTest() { suite.metrics, 0, 0, - false, + false) + require.NoError(suite.T(), err) + + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, false, nil, nil, suite.me, + backend, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index d4c812df997..4fde8fe1acf 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -9,13 +9,11 @@ import ( "sync" "time" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - lru "github.com/hashicorp/golang-lru" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/rs/zerolog" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "github.com/rs/zerolog" + "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc/backend" @@ -26,7 +24,8 @@ import ( "github.com/onflow/flow-go/module/events" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/storage" + + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" ) // Config defines the configurable options for the access node server @@ -74,31 +73,23 @@ type Engine struct { unsecureGrpcAddress net.Addr secureGrpcAddress net.Addr restAPIAddress net.Addr + + restHandler rest.RestServerApi } +type Option func(*RPCEngineBuilder) // NewBuilder returns a new RPC engine builder. func NewBuilder(log zerolog.Logger, state protocol.State, config Config, - collectionRPC accessproto.AccessAPIClient, - historicalAccessNodes []accessproto.AccessAPIClient, - blocks storage.Blocks, - headers storage.Headers, - collections storage.Collections, - transactions storage.Transactions, - executionReceipts storage.ExecutionReceipts, - executionResults storage.ExecutionResults, chainID flow.ChainID, accessMetrics module.AccessMetrics, - collectionGRPCPort uint, - executionGRPCPort uint, - retryEnabled bool, rpcMetricsEnabled bool, apiRatelimits map[string]int, // the api rate limit (max calls per second) for each of the Access API e.g. Ping->100, GetTransaction->300 apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 me module.Local, + backend *backend.Backend, ) (*RPCEngineBuilder, error) { - log = log.With().Str("engine", "rpc").Logger() // create a GRPC server to serve GRPC clients @@ -137,63 +128,6 @@ func NewBuilder(log zerolog.Logger, // wrap the unsecured server with an HTTP proxy server to serve HTTP clients httpServer := newHTTPProxyServer(unsecureGrpcServer) - var cache *lru.Cache - cacheSize := config.ConnectionPoolSize - if cacheSize > 0 { - // TODO: remove this fallback after fixing issues with evictions - // It was observed that evictions cause connection errors for in flight requests. This works around - // the issue by forcing hte pool size to be greater than the number of ENs + LNs - if cacheSize < backend.DefaultConnectionPoolSize { - log.Warn().Msg("connection pool size below threshold, setting pool size to default value ") - cacheSize = backend.DefaultConnectionPoolSize - } - var err error - cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { - store := evictedValue.(*backend.CachedClient) - store.Close() - log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") - if accessMetrics != nil { - accessMetrics.ConnectionFromPoolEvicted() - } - }) - if err != nil { - return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) - } - } - - connectionFactory := &backend.ConnectionFactoryImpl{ - CollectionGRPCPort: collectionGRPCPort, - ExecutionGRPCPort: executionGRPCPort, - CollectionNodeGRPCTimeout: config.CollectionClientTimeout, - ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, - ConnectionsCache: cache, - CacheSize: cacheSize, - MaxMsgSize: config.MaxMsgSize, - AccessMetrics: accessMetrics, - Log: log, - } - - backend := backend.New(state, - collectionRPC, - historicalAccessNodes, - blocks, - headers, - collections, - transactions, - executionReceipts, - executionResults, - chainID, - accessMetrics, - connectionFactory, - retryEnabled, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - log, - backend.DefaultSnapshotHistoryLimit, - config.ArchiveAddressList, - ) - finalizedCache, finalizedCacheWorker, err := events.NewFinalizedHeaderCache(state) if err != nil { return nil, fmt.Errorf("could not create header cache: %w", err) @@ -384,7 +318,7 @@ func (e *Engine) serveREST(ctx irrecoverable.SignalerContext, ready component.Re e.log.Info().Str("rest_api_address", e.config.RESTListenAddr).Msg("starting REST server on address") - r, err := rest.NewServer(e.backend, e.config.RESTListenAddr, e.log, e.chain, e.restCollector) + r, err := rest.NewServer(e.restHandler, e.config.RESTListenAddr, e.log, e.chain, e.restCollector) if err != nil { e.log.Err(err).Msg("failed to initialize the REST server") ctx.Throw(err) diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index a4694547b03..cdae487a525 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -5,13 +5,14 @@ import ( grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - legacyaccessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" - "github.com/onflow/flow-go/access" legacyaccess "github.com/onflow/flow-go/access/legacy" "github.com/onflow/flow-go/consensus/hotstuff" + "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/module" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + legacyaccessproto "github.com/onflow/flow/protobuf/go/flow/legacy/access" ) type RPCEngineBuilder struct { @@ -21,7 +22,7 @@ type RPCEngineBuilder struct { // optional parameters, only one can be set during build phase signerIndicesDecoder hotstuff.BlockSignerDecoder - handler accessproto.AccessAPIServer // Use the parent interface instead of implementation, so that we can assign it to proxy. + rpcHandler accessproto.AccessAPIServer // Use the parent interface instead of implementation, so that we can assign it to proxy. } // NewRPCEngineBuilder helps to build a new RPC engine. @@ -34,8 +35,12 @@ func NewRPCEngineBuilder(engine *Engine, me module.Local, finalizedHeaderCache m } } -func (builder *RPCEngineBuilder) Handler() accessproto.AccessAPIServer { - return builder.handler +func (builder *RPCEngineBuilder) RpcHandler() accessproto.AccessAPIServer { + return builder.rpcHandler +} + +func (builder *RPCEngineBuilder) RestHandler() rest.RestServerApi { + return builder.restHandler } // WithBlockSignerDecoder specifies that signer indices in block headers should be translated @@ -51,15 +56,21 @@ func (builder *RPCEngineBuilder) WithBlockSignerDecoder(signerIndicesDecoder hot return builder } -// WithNewHandler specifies that the given `AccessAPIServer` should be used for serving API queries. +// WithRpcHandler specifies that the given `AccessAPIServer` should be used for serving API queries. // Caution: // you can inject either a `BlockSignerDecoder` (via method `WithBlockSignerDecoder`) -// or an `AccessAPIServer` (via method `WithNewHandler`); but not both. If both are +// or an `AccessAPIServer` (via method `WithRpcHandler`); but not both. If both are // specified, the builder will error during the build step. // // Returns self-reference for chaining. -func (builder *RPCEngineBuilder) WithNewHandler(handler accessproto.AccessAPIServer) *RPCEngineBuilder { - builder.handler = handler +func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPIServer) *RPCEngineBuilder { + builder.rpcHandler = handler + return builder +} + +// WithRestHandler specifies that the given `RestServerApi` should be used for serving REST queries. +func (builder *RPCEngineBuilder) WithRestHandler(handler rest.RestServerApi) *RPCEngineBuilder { + builder.restHandler = handler return builder } @@ -89,18 +100,25 @@ func (builder *RPCEngineBuilder) WithMetrics() *RPCEngineBuilder { } func (builder *RPCEngineBuilder) Build() (*Engine, error) { - if builder.signerIndicesDecoder != nil && builder.handler != nil { + if builder.signerIndicesDecoder != nil && builder.rpcHandler != nil { return nil, fmt.Errorf("only BlockSignerDecoder (via method `WithBlockSignerDecoder`) or AccessAPIServer (via method `WithNewHandler`) can be specified but not both") } - handler := builder.handler - if handler == nil { + rpcHandler := builder.rpcHandler + if rpcHandler == nil { if builder.signerIndicesDecoder == nil { - handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me) } else { - handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) + rpcHandler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) } } - accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer, handler) - accessproto.RegisterAccessAPIServer(builder.secureGrpcServer, handler) + accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer, rpcHandler) + accessproto.RegisterAccessAPIServer(builder.secureGrpcServer, rpcHandler) + + restHandler := builder.Engine.restHandler + if restHandler == nil { + restHandler = rest.NewRequestHandler(builder.log, builder.backend) + } + builder.Engine.restHandler = restHandler + return builder.Engine, nil } diff --git a/go.mod b/go.mod index 602fb4c15fd..8a79bff8fd8 100644 --- a/go.mod +++ b/go.mod @@ -278,3 +278,6 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.6 // indirect ) + +//TODO: Remove when onflow/flow branch will be merged +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e => github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 diff --git a/go.sum b/go.sum index ed305eed14f..284f4b5d690 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,8 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 h1:8l0uZ9ut9TowB1qNKbPFt/ar/5mqxhqcp0r+HWv1zps= +github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -1238,8 +1240,6 @@ github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QE github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e h1:QYEd3KWTt309YGBch4IGK6vJ6b7cOGx2NStEnd5NeHM= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/insecure/go.sum b/insecure/go.sum index 129d83cb596..a2e659e29d8 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -92,6 +92,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -1176,6 +1177,7 @@ github.com/onflow/atree v0.5.0 h1:y3lh8hY2fUo8KVE2ALVcz0EiNTq0tXJ6YTXKYVDA+3E= github.com/onflow/atree v0.5.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.38.1 h1:8YpnE1ixAGB8hF3t+slkHGhjfIBJ95dqUS+sEHrM2kY= github.com/onflow/cadence v0.38.1/go.mod h1:SpfjNhPsJxGIHbOthE9JD/e8JFaFY73joYLPsov+PY4= +github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= diff --git a/integration/go.mod b/integration/go.mod index 478283c6530..e207633fdc7 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -325,3 +325,5 @@ require ( replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure + +replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e => github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 diff --git a/integration/go.sum b/integration/go.sum index 5aa4af7288b..d81044c4bbf 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -101,6 +101,8 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBY github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 h1:8l0uZ9ut9TowB1qNKbPFt/ar/5mqxhqcp0r+HWv1zps= +github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= @@ -1318,8 +1320,6 @@ github.com/onflow/flow-go-sdk v0.40.0 h1:s8uwoyTquN8tjdXpqGmNkXTjf79yUII8JExc5QE github.com/onflow/flow-go-sdk v0.40.0/go.mod h1:34dxXk9Hp/bQw6Zy6+H44Xo0kQU+aJyQoqdDxq00rJM= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e h1:QYEd3KWTt309YGBch4IGK6vJ6b7cOGx2NStEnd5NeHM= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230428213521-89bcc9e8517e/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/module/forwarder/forwarder.go b/module/forwarder/forwarder.go index 0201c3908cf..6ee4ae4ffd0 100644 --- a/module/forwarder/forwarder.go +++ b/module/forwarder/forwarder.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow/protobuf/go/flow/access" ) diff --git a/module/mock/grpc_connection_pool_metrics.go b/module/mock/grpc_connection_pool_metrics.go new file mode 100644 index 00000000000..2eddb3cf002 --- /dev/null +++ b/module/mock/grpc_connection_pool_metrics.go @@ -0,0 +1,60 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// GRPCConnectionPoolMetrics is an autogenerated mock type for the GRPCConnectionPoolMetrics type +type GRPCConnectionPoolMetrics struct { + mock.Mock +} + +// ConnectionAddedToPool provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionAddedToPool() { + _m.Called() +} + +// ConnectionFromPoolEvicted provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolEvicted() { + _m.Called() +} + +// ConnectionFromPoolInvalidated provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolInvalidated() { + _m.Called() +} + +// ConnectionFromPoolReused provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolReused() { + _m.Called() +} + +// ConnectionFromPoolUpdated provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) ConnectionFromPoolUpdated() { + _m.Called() +} + +// NewConnectionEstablished provides a mock function with given fields: +func (_m *GRPCConnectionPoolMetrics) NewConnectionEstablished() { + _m.Called() +} + +// TotalConnectionsInPool provides a mock function with given fields: connectionCount, connectionPoolSize +func (_m *GRPCConnectionPoolMetrics) TotalConnectionsInPool(connectionCount uint, connectionPoolSize uint) { + _m.Called(connectionCount, connectionPoolSize) +} + +type mockConstructorTestingTNewGRPCConnectionPoolMetrics interface { + mock.TestingT + Cleanup(func()) +} + +// NewGRPCConnectionPoolMetrics creates a new instance of GRPCConnectionPoolMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewGRPCConnectionPoolMetrics(t mockConstructorTestingTNewGRPCConnectionPoolMetrics) *GRPCConnectionPoolMetrics { + mock := &GRPCConnectionPoolMetrics{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From 0a8aa0d90c1f829a349d1e9c5da935fd15d2d615 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 16 Jun 2023 12:46:53 +0300 Subject: [PATCH 040/169] Updated tests --- engine/access/rest/accounts_test.go | 211 ++++++++- engine/access/rest/blocks_test.go | 65 ++- engine/access/rest/collections_test.go | 7 +- engine/access/rest/events_test.go | 3 +- engine/access/rest/execution_result_test.go | 18 +- engine/access/rest/mock/rest_server_api.go | 380 +++++++++++++++ engine/access/rest/network_test.go | 3 +- engine/access/rest/node_version_info_test.go | 3 +- engine/access/rest/rest_server_api.go | 2 +- engine/access/rest/scripts_test.go | 17 +- engine/access/rest/test_helpers.go | 80 +++- engine/access/rest/transactions_test.go | 38 +- engine/access/rpc/rate_limit_test.go | 463 ++++++++++--------- engine/access/secure_grpcr_test.go | 14 +- 14 files changed, 1021 insertions(+), 283 deletions(-) create mode 100644 engine/access/rest/mock/rest_server_api.go diff --git a/engine/access/rest/accounts_test.go b/engine/access/rest/accounts_test.go index 61982ff5f9c..0446fc4958d 100644 --- a/engine/access/rest/accounts_test.go +++ b/engine/access/rest/accounts_test.go @@ -35,6 +35,7 @@ func accountURL(t *testing.T, address string, height string) string { func TestGetAccount(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get by address at latest sealed block", func(t *testing.T) { account := accountFixture(t) @@ -53,7 +54,7 @@ func TestGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -73,7 +74,7 @@ func TestGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -88,7 +89,7 @@ func TestGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -103,7 +104,7 @@ func TestGetAccount(t *testing.T) { expected := expectedCondensedResponse(account) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -118,7 +119,7 @@ func TestGetAccount(t *testing.T) { for i, test := range tests { req, _ := http.NewRequest("GET", test.url, nil) - rr, err := executeRequest(req, backend) + rr, err := executeRequest(req, restHandler) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -127,6 +128,206 @@ func TestGetAccount(t *testing.T) { }) } +//func TestObserverGetAccount(t *testing.T) { +// backend := &mock.API{} +// address := unittest.IPPort("11632") +// +// t.Run("get by address at latest sealed block", func(t *testing.T) { +// account := accountFixture(t) +// entityAccount, err := convert.AccountToMessage(account) +// assert.NoError(t, err) +// +// var height uint64 = 100 +// blockHeader, err := convert.BlockHeaderToMessage(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)), unittest.IdentifierListFixture(4)) //? +// assert.NoError(t, err) +// +// req := getAccountRequest(t, account, sealedHeightQueryParam, expandableFieldKeys, expandableFieldContracts) +// +// done := make(chan int) +// // Bring up 1st upstream server +// mockServer := new(engineaccessmock.AccessAPIServer) +// mockServer.Mock. +// On("GetLatestBlockHeader", mocktestify.Anything, +// &access.GetLatestBlockHeaderRequest{ +// IsSealed: true, +// }). +// Return(&access.BlockHeaderResponse{ +// Block: blockHeader, +// BlockStatus: entities.BlockStatus_BLOCK_SEALED, +// Metadata: nil, +// }, nil) +// +// mockServer.Mock. +// On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ +// Address: account.Address.Bytes(), +// BlockHeight: height, +// }). +// Return(&access.AccountResponse{ +// Account: entityAccount, +// Metadata: nil, +// }, nil) +// expected := expectedExpandedResponse(account) +// +// server, _, err := newGrpcServer(mockServer, "tcp", address, done) +// assert.NoError(t, err) +// +// restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) +// assert.NoError(t, err) +// +// assertOKResponse(t, req, expected, restHandler) +// mocktestify.AssertExpectationsForObjects(t, backend) +// +// server.Stop() +// <-done +// }) +// +// //t.Run("get by address at latest finalized block", func(t *testing.T) { +// // var height uint64 = 100 +// // blockHeader, err := convert.BlockHeaderToMessage(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)), unittest.IdentifierListFixture(4)) +// // assert.NoError(t, err) +// // account := accountFixture(t) +// // entityAccount, err := convert.AccountToMessage(account) +// // assert.NoError(t, err) +// // +// // req := getAccountRequest(t, account, finalHeightQueryParam, expandableFieldKeys, expandableFieldContracts) +// // +// // done := make(chan int) +// // // Bring up 1st upstream server +// // mockServer := new(engineaccessmock.AccessAPIServer) +// // mockServer.Mock. +// // On("GetLatestBlockHeader", mocktestify.Anything, +// // &access.GetLatestBlockHeaderRequest{ +// // IsSealed: false, +// // }). +// // Return(&access.BlockHeaderResponse{ +// // Block: blockHeader, +// // BlockStatus: entities.BlockStatus_BLOCK_FINALIZED, +// // Metadata: nil, +// // }, nil) +// // mockServer.Mock. +// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ +// // Address: account.Address.Bytes(), +// // BlockHeight: height, +// // }). +// // Return(&access.AccountResponse{ +// // Account: entityAccount, +// // Metadata: nil, +// // }, nil) +// // expected := expectedExpandedResponse(account) +// // +// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) +// // assert.NoError(t, err) +// // +// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) +// // assert.NoError(t, err) +// // +// // assertOKResponse(t, req, expected, restHandler) +// // mocktestify.AssertExpectationsForObjects(t, backend) +// // +// // server.Stop() +// // <-done +// //}) +// // +// //t.Run("get by address at height", func(t *testing.T) { +// // var height uint64 = 1337 +// // account := accountFixture(t) +// // entityAccount, err := convert.AccountToMessage(account) +// // assert.NoError(t, err) +// // +// // req := getAccountRequest(t, account, fmt.Sprintf("%d", height), expandableFieldKeys, expandableFieldContracts) +// // +// // done := make(chan int) +// // // Bring up 1st upstream server +// // mockServer := new(engineaccessmock.AccessAPIServer) +// // mockServer.Mock. +// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ +// // Address: account.Address.Bytes(), +// // BlockHeight: height, +// // }). +// // Return(&access.AccountResponse{ +// // Account: entityAccount, +// // Metadata: nil, +// // }, nil) +// // expected := expectedExpandedResponse(account) +// // +// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) +// // assert.NoError(t, err) +// // +// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) +// // assert.NoError(t, err) +// // +// // assertOKResponse(t, req, expected, restHandler) +// // mocktestify.AssertExpectationsForObjects(t, backend) +// // +// // server.Stop() +// // <-done +// //}) +// // +// //t.Run("get by address at height condensed", func(t *testing.T) { +// // var height uint64 = 1337 +// // account := accountFixture(t) +// // entityAccount, err := convert.AccountToMessage(account) +// // assert.NoError(t, err) +// // +// // req := getAccountRequest(t, account, fmt.Sprintf("%d", height)) +// // +// // done := make(chan int) +// // // Bring up 1st upstream server +// // mockServer := new(engineaccessmock.AccessAPIServer) +// // mockServer.Mock. +// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ +// // Address: account.Address.Bytes(), +// // BlockHeight: height, +// // }). +// // Return(&access.AccountResponse{ +// // Account: entityAccount, +// // Metadata: nil, +// // }, nil) +// // expected := expectedCondensedResponse(account) +// // +// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) +// // assert.NoError(t, err) +// // +// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) +// // assert.NoError(t, err) +// // +// // assertOKResponse(t, req, expected, restHandler) +// // mocktestify.AssertExpectationsForObjects(t, backend) +// // +// // server.Stop() +// // <-done +// //}) +// // +// //t.Run("get invalid", func(t *testing.T) { +// // tests := []struct { +// // url string +// // out string +// // }{ +// // {accountURL(t, "123", ""), `{"code":400, "message":"invalid address"}`}, +// // {accountURL(t, unittest.AddressFixture().String(), "foo"), `{"code":400, "message":"invalid height format"}`}, +// // } +// // +// // done := make(chan int) +// // // Bring up 1st upstream server +// // server, _, err := newGrpcServer(new(engineaccessmock.AccessAPIServer), "tcp", address, done) +// // assert.NoError(t, err) +// // +// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) +// // assert.NoError(t, err) +// // for i, test := range tests { +// // req, _ := http.NewRequest("GET", test.url, nil) +// // rr, err := executeRequest(req, restHandler) +// // assert.NoError(t, err) +// // +// // assert.Equal(t, http.StatusBadRequest, rr.Code) +// // assert.JSONEq(t, test.out, rr.Body.String(), fmt.Sprintf("test #%d failed: %v", i, test)) +// // } +// // +// // server.Stop() +// // <-done +// //}) +//} + func expectedExpandedResponse(account *flow.Account) string { return fmt.Sprintf(`{ "address":"%s", diff --git a/engine/access/rest/blocks_test.go b/engine/access/rest/blocks_test.go index 7f977b06d69..536b2b50296 100644 --- a/engine/access/rest/blocks_test.go +++ b/engine/access/rest/blocks_test.go @@ -31,12 +31,13 @@ type testVector struct { expectedResponse string } -// TestGetBlocks tests the get blocks by ID and get blocks by heights API -func TestGetBlocks(t *testing.T) { - backend := &mock.API{} - - blkCnt := 10 - blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) +// ? +func prepareTestVectors(t *testing.T, + blockIDs []string, + heights []string, + blocks []*flow.Block, + executionResults []*flow.ExecutionResult, + blkCnt int) []testVector { singleBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusUnknown) singleSealedBlockExpandedResponse := expectedBlockResponsesExpanded(blocks[:1], executionResults[:1], true, flow.BlockStatusSealed) @@ -137,9 +138,20 @@ func TestGetBlocks(t *testing.T) { expectedResponse: fmt.Sprintf(`{"code":400, "message": "at most %d IDs can be requested at a time"}`, request.MaxBlockRequestHeightRange), }, } + return testVectors +} + +// TestGetBlocks tests the get blocks by ID and get blocks by heights API +func TestGetBlocks(t *testing.T) { + backend := &mock.API{} + + blkCnt := 10 + blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) + testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) + restHandler := newAccessRestHandler(backend) for _, tv := range testVectors { - responseRec, err := executeRequest(tv.request, backend) + responseRec, err := executeRequest(tv.request, restHandler) assert.NoError(t, err) require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) actualResp := responseRec.Body.String() @@ -147,6 +159,45 @@ func TestGetBlocks(t *testing.T) { } } +// ? +//func TestGetBlockFromObserver(t *testing.T) { +// backend := &mock.API{} +// +// // Bring up upstream server +// blkCnt := 10 +// blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) +// address := unittest.IPPort("11633") +// +// done := make(chan int) +// server, _, err := newGrpcServer(new(engineaccessmock.AccessAPIServer), "tcp", address, done) +// assert.NoError(t, err) +// +// testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) +// +// var b bytes.Buffer +// logger := zerolog.New(&b) +// +// restForwarder, err := NewRestForwarder(logger, +// flow.IdentityList{{Address: address}}, +// time.Second, +// grpcutils.DefaultMaxMsgSize) +// assert.NoError(t, err) +// +// restHandler, err := newObserverRestHandler_v2(backend, ) +// assert.NoError(t, err) +// +// for _, tv := range testVectors { +// responseRec, err := executeRequest(tv.request, restHandler) +// assert.NoError(t, err) +// require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) +// actualResp := responseRec.Body.String() +// require.JSONEq(t, tv.expectedResponse, actualResp, "Failed: %s: incorrect response body", tv.description) +// +// } +// server.Stop() +// <-done +//} + func requestURL(t *testing.T, ids []string, start string, end string, expandResponse bool, heights ...string) *http.Request { u, _ := url.Parse("/v1/blocks") q := u.Query() diff --git a/engine/access/rest/collections_test.go b/engine/access/rest/collections_test.go index 3981541f3a7..a033fb8a453 100644 --- a/engine/access/rest/collections_test.go +++ b/engine/access/rest/collections_test.go @@ -31,6 +31,7 @@ func getCollectionReq(id string, expandTransactions bool) *http.Request { func TestGetCollections(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get by ID", func(t *testing.T) { inputs := []flow.LightCollection{ @@ -62,7 +63,7 @@ func TestGetCollections(t *testing.T) { }`, col.ID(), col.ID(), transactionsStr) req := getCollectionReq(col.ID().String(), false) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocks.AssertExpectationsForObjects(t, backend) } }) @@ -87,7 +88,7 @@ func TestGetCollections(t *testing.T) { Once() req := getCollectionReq(col.ID().String(), true) - rr, err := executeRequest(req, backend) + rr, err := executeRequest(req, restHandler) assert.NoError(t, err) assert.Equal(t, http.StatusOK, rr.Code) @@ -146,7 +147,7 @@ func TestGetCollections(t *testing.T) { Return(test.mockValue, test.mockErr) } req := getCollectionReq(test.id, false) - assertResponse(t, req, test.status, test.response, backend) + assertResponse(t, req, test.status, test.response, restHandler) } }) } diff --git a/engine/access/rest/events_test.go b/engine/access/rest/events_test.go index 9f0fede2c6c..c5b9be0c00b 100644 --- a/engine/access/rest/events_test.go +++ b/engine/access/rest/events_test.go @@ -23,6 +23,7 @@ import ( func TestGetEvents(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) events := generateEventsMocks(backend, 5) allBlockIDs := make([]string, len(events)) @@ -125,7 +126,7 @@ func TestGetEvents(t *testing.T) { for _, test := range testVectors { t.Run(test.description, func(t *testing.T) { - assertResponse(t, test.request, test.expectedStatus, test.expectedResponse, backend) + assertResponse(t, test.request, test.expectedStatus, test.expectedResponse, restHandler) }) } diff --git a/engine/access/rest/execution_result_test.go b/engine/access/rest/execution_result_test.go index adb3852c668..241f14f6b59 100644 --- a/engine/access/rest/execution_result_test.go +++ b/engine/access/rest/execution_result_test.go @@ -37,9 +37,10 @@ func getResultByIDReq(id string, blockIDs []string) *http.Request { } func TestGetResultByID(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get by ID", func(t *testing.T) { - backend := &mock.API{} result := unittest.ExecutionResultFixture() id := unittest.IdentifierFixture() backend.Mock. @@ -49,12 +50,11 @@ func TestGetResultByID(t *testing.T) { req := getResultByIDReq(id.String(), nil) expected := executionResultExpectedStr(result) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocks.AssertExpectationsForObjects(t, backend) }) t.Run("get by ID not found", func(t *testing.T) { - backend := &mock.API{} id := unittest.IdentifierFixture() backend.Mock. On("GetExecutionResultByID", mocks.Anything, id). @@ -62,14 +62,17 @@ func TestGetResultByID(t *testing.T) { Once() req := getResultByIDReq(id.String(), nil) - assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, backend) + assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, restHandler) mocks.AssertExpectationsForObjects(t, backend) }) } func TestGetResultBlockID(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + t.Run("get by block ID", func(t *testing.T) { - backend := &mock.API{} + blockID := unittest.IdentifierFixture() result := unittest.ExecutionResultFixture(unittest.WithExecutionResultBlockID(blockID)) @@ -81,12 +84,11 @@ func TestGetResultBlockID(t *testing.T) { req := getResultByIDReq("", []string{blockID.String()}) expected := fmt.Sprintf(`[%s]`, executionResultExpectedStr(result)) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocks.AssertExpectationsForObjects(t, backend) }) t.Run("get by block ID not found", func(t *testing.T) { - backend := &mock.API{} blockID := unittest.IdentifierFixture() backend.Mock. On("GetExecutionResultForBlockID", mocks.Anything, blockID). @@ -94,7 +96,7 @@ func TestGetResultBlockID(t *testing.T) { Once() req := getResultByIDReq("", []string{blockID.String()}) - assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, backend) + assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, restHandler) mocks.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/mock/rest_server_api.go b/engine/access/rest/mock/rest_server_api.go new file mode 100644 index 00000000000..fb87307189c --- /dev/null +++ b/engine/access/rest/mock/rest_server_api.go @@ -0,0 +1,380 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + context "context" + + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" + + models "github.com/onflow/flow-go/engine/access/rest/models" + + request "github.com/onflow/flow-go/engine/access/rest/request" +) + +// RestServerApi is an autogenerated mock type for the RestServerApi type +type RestServerApi struct { + mock.Mock +} + +// CreateTransaction provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) CreateTransaction(r request.CreateTransaction, _a1 context.Context, link models.LinkGenerator) (models.Transaction, error) { + ret := _m.Called(r, _a1, link) + + var r0 models.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(request.CreateTransaction, context.Context, models.LinkGenerator) (models.Transaction, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.CreateTransaction, context.Context, models.LinkGenerator) models.Transaction); ok { + r0 = rf(r, _a1, link) + } else { + r0 = ret.Get(0).(models.Transaction) + } + + if rf, ok := ret.Get(1).(func(request.CreateTransaction, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ExecuteScript provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) ExecuteScript(r request.GetScript, _a1 context.Context, link models.LinkGenerator) ([]byte, error) { + ret := _m.Called(r, _a1, link) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(request.GetScript, context.Context, models.LinkGenerator) ([]byte, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.GetScript, context.Context, models.LinkGenerator) []byte); ok { + r0 = rf(r, _a1, link) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(request.GetScript, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAccount provides a mock function with given fields: r, _a1, expandFields, link +func (_m *RestServerApi) GetAccount(r request.GetAccount, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + ret := _m.Called(r, _a1, expandFields, link) + + var r0 models.Account + var r1 error + if rf, ok := ret.Get(0).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) (models.Account, error)); ok { + return rf(r, _a1, expandFields, link) + } + if rf, ok := ret.Get(0).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) models.Account); ok { + r0 = rf(r, _a1, expandFields, link) + } else { + r0 = ret.Get(0).(models.Account) + } + + if rf, ok := ret.Get(1).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, expandFields, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetBlockPayloadByID provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) GetBlockPayloadByID(r request.GetBlockPayload, _a1 context.Context, link models.LinkGenerator) (models.BlockPayload, error) { + ret := _m.Called(r, _a1, link) + + var r0 models.BlockPayload + var r1 error + if rf, ok := ret.Get(0).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) (models.BlockPayload, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) models.BlockPayload); ok { + r0 = rf(r, _a1, link) + } else { + r0 = ret.Get(0).(models.BlockPayload) + } + + if rf, ok := ret.Get(1).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetBlocksByHeight provides a mock function with given fields: r, link +func (_m *RestServerApi) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + ret := _m.Called(r, link) + + var r0 []*models.Block + var r1 error + if rf, ok := ret.Get(0).(func(*request.Request, models.LinkGenerator) ([]*models.Block, error)); ok { + return rf(r, link) + } + if rf, ok := ret.Get(0).(func(*request.Request, models.LinkGenerator) []*models.Block); ok { + r0 = rf(r, link) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Block) + } + } + + if rf, ok := ret.Get(1).(func(*request.Request, models.LinkGenerator) error); ok { + r1 = rf(r, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetBlocksByIDs provides a mock function with given fields: r, _a1, expandFields, link +func (_m *RestServerApi) GetBlocksByIDs(r request.GetBlockByIDs, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + ret := _m.Called(r, _a1, expandFields, link) + + var r0 []*models.Block + var r1 error + if rf, ok := ret.Get(0).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) ([]*models.Block, error)); ok { + return rf(r, _a1, expandFields, link) + } + if rf, ok := ret.Get(0).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) []*models.Block); ok { + r0 = rf(r, _a1, expandFields, link) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.Block) + } + } + + if rf, ok := ret.Get(1).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, expandFields, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetCollectionByID provides a mock function with given fields: r, _a1, expandFields, link, chain +func (_m *RestServerApi) GetCollectionByID(r request.GetCollection, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { + ret := _m.Called(r, _a1, expandFields, link, chain) + + var r0 models.Collection + var r1 error + if rf, ok := ret.Get(0).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) (models.Collection, error)); ok { + return rf(r, _a1, expandFields, link, chain) + } + if rf, ok := ret.Get(0).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) models.Collection); ok { + r0 = rf(r, _a1, expandFields, link, chain) + } else { + r0 = ret.Get(0).(models.Collection) + } + + if rf, ok := ret.Get(1).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) error); ok { + r1 = rf(r, _a1, expandFields, link, chain) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetEvents provides a mock function with given fields: r, _a1 +func (_m *RestServerApi) GetEvents(r request.GetEvents, _a1 context.Context) (models.BlocksEvents, error) { + ret := _m.Called(r, _a1) + + var r0 models.BlocksEvents + var r1 error + if rf, ok := ret.Get(0).(func(request.GetEvents, context.Context) (models.BlocksEvents, error)); ok { + return rf(r, _a1) + } + if rf, ok := ret.Get(0).(func(request.GetEvents, context.Context) models.BlocksEvents); ok { + r0 = rf(r, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(models.BlocksEvents) + } + } + + if rf, ok := ret.Get(1).(func(request.GetEvents, context.Context) error); ok { + r1 = rf(r, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetExecutionResultByID provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) GetExecutionResultByID(r request.GetExecutionResult, _a1 context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + ret := _m.Called(r, _a1, link) + + var r0 models.ExecutionResult + var r1 error + if rf, ok := ret.Get(0).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) (models.ExecutionResult, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) models.ExecutionResult); ok { + r0 = rf(r, _a1, link) + } else { + r0 = ret.Get(0).(models.ExecutionResult) + } + + if rf, ok := ret.Get(1).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetExecutionResultsByBlockIDs provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, _a1 context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + ret := _m.Called(r, _a1, link) + + var r0 []models.ExecutionResult + var r1 error + if rf, ok := ret.Get(0).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) ([]models.ExecutionResult, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) []models.ExecutionResult); ok { + r0 = rf(r, _a1, link) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]models.ExecutionResult) + } + } + + if rf, ok := ret.Get(1).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetNetworkParameters provides a mock function with given fields: r +func (_m *RestServerApi) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { + ret := _m.Called(r) + + var r0 models.NetworkParameters + var r1 error + if rf, ok := ret.Get(0).(func(*request.Request) (models.NetworkParameters, error)); ok { + return rf(r) + } + if rf, ok := ret.Get(0).(func(*request.Request) models.NetworkParameters); ok { + r0 = rf(r) + } else { + r0 = ret.Get(0).(models.NetworkParameters) + } + + if rf, ok := ret.Get(1).(func(*request.Request) error); ok { + r1 = rf(r) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetNodeVersionInfo provides a mock function with given fields: r +func (_m *RestServerApi) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { + ret := _m.Called(r) + + var r0 models.NodeVersionInfo + var r1 error + if rf, ok := ret.Get(0).(func(*request.Request) (models.NodeVersionInfo, error)); ok { + return rf(r) + } + if rf, ok := ret.Get(0).(func(*request.Request) models.NodeVersionInfo); ok { + r0 = rf(r) + } else { + r0 = ret.Get(0).(models.NodeVersionInfo) + } + + if rf, ok := ret.Get(1).(func(*request.Request) error); ok { + r1 = rf(r) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTransactionByID provides a mock function with given fields: r, _a1, link, chain +func (_m *RestServerApi) GetTransactionByID(r request.GetTransaction, _a1 context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { + ret := _m.Called(r, _a1, link, chain) + + var r0 models.Transaction + var r1 error + if rf, ok := ret.Get(0).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) (models.Transaction, error)); ok { + return rf(r, _a1, link, chain) + } + if rf, ok := ret.Get(0).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) models.Transaction); ok { + r0 = rf(r, _a1, link, chain) + } else { + r0 = ret.Get(0).(models.Transaction) + } + + if rf, ok := ret.Get(1).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) error); ok { + r1 = rf(r, _a1, link, chain) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTransactionResultByID provides a mock function with given fields: r, _a1, link +func (_m *RestServerApi) GetTransactionResultByID(r request.GetTransactionResult, _a1 context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + ret := _m.Called(r, _a1, link) + + var r0 models.TransactionResult + var r1 error + if rf, ok := ret.Get(0).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) (models.TransactionResult, error)); ok { + return rf(r, _a1, link) + } + if rf, ok := ret.Get(0).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) models.TransactionResult); ok { + r0 = rf(r, _a1, link) + } else { + r0 = ret.Get(0).(models.TransactionResult) + } + + if rf, ok := ret.Get(1).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) error); ok { + r1 = rf(r, _a1, link) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewRestServerApi interface { + mock.TestingT + Cleanup(func()) +} + +// NewRestServerApi creates a new instance of RestServerApi. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRestServerApi(t mockConstructorTestingTNewRestServerApi) *RestServerApi { + mock := &RestServerApi{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/rest/network_test.go b/engine/access/rest/network_test.go index c4ce7492476..f8013c86685 100644 --- a/engine/access/rest/network_test.go +++ b/engine/access/rest/network_test.go @@ -23,6 +23,7 @@ func networkURL(t *testing.T) string { func TestGetNetworkParameters(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get network parameters on mainnet", func(t *testing.T) { @@ -38,7 +39,7 @@ func TestGetNetworkParameters(t *testing.T) { expected := networkParametersExpectedStr(flow.Mainnet) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/node_version_info_test.go b/engine/access/rest/node_version_info_test.go index 4140089a280..b2543950ea0 100644 --- a/engine/access/rest/node_version_info_test.go +++ b/engine/access/rest/node_version_info_test.go @@ -24,6 +24,7 @@ func nodeVersionInfoURL(t *testing.T) string { func TestGetNodeVersionInfo(t *testing.T) { backend := mock.NewAPI(t) + restHandler := newAccessRestHandler(backend) t.Run("get node version info", func(t *testing.T) { req := getNodeVersionInfoRequest(t) @@ -41,7 +42,7 @@ func TestGetNodeVersionInfo(t *testing.T) { expected := nodeVersionInfoExpectedStr(params) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) mocktestify.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go index 19d6c00440e..ed29d90b6aa 100644 --- a/engine/access/rest/rest_server_api.go +++ b/engine/access/rest/rest_server_api.go @@ -26,7 +26,7 @@ import ( type RestRouter struct { Logger zerolog.Logger Metrics *metrics.ObserverCollector - Upstream *RestForwarder + Upstream RestServerApi Observer *RequestHandler } diff --git a/engine/access/rest/scripts_test.go b/engine/access/rest/scripts_test.go index 7e3271c1d81..d69886e1d20 100644 --- a/engine/access/rest/scripts_test.go +++ b/engine/access/rest/scripts_test.go @@ -45,9 +45,10 @@ func TestScripts(t *testing.T) { "script": util.ToBase64(validCode), "arguments": []string{util.ToBase64(validArgs)}, } + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get by Latest height", func(t *testing.T) { - backend := &mock.API{} backend.Mock. On("ExecuteScriptAtLatestBlock", mocks.Anything, validCode, [][]byte{validArgs}). Return([]byte("hello world"), nil) @@ -56,11 +57,10 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), backend) + ), restHandler) }) t.Run("get by height", func(t *testing.T) { - backend := &mock.API{} height := uint64(1337) backend.Mock. @@ -71,11 +71,10 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), backend) + ), restHandler) }) t.Run("get by ID", func(t *testing.T) { - backend := &mock.API{} id, _ := flow.HexStringToIdentifier("222dc5dd51b9e4910f687e475f892f495f3352362ba318b53e318b4d78131312") backend.Mock. @@ -86,11 +85,10 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), backend) + ), restHandler) }) t.Run("get error", func(t *testing.T) { - backend := &mock.API{} backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, uint64(1337), validCode, [][]byte{validArgs}). Return(nil, status.Error(codes.Internal, "internal server error")) @@ -101,12 +99,11 @@ func TestScripts(t *testing.T) { req, http.StatusBadRequest, `{"code":400, "message":"Invalid Flow request: internal server error"}`, - backend, + restHandler, ) }) t.Run("get invalid", func(t *testing.T) { - backend := &mock.API{} backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, mocks.Anything, mocks.Anything, mocks.Anything). Return(nil, nil) @@ -126,7 +123,7 @@ func TestScripts(t *testing.T) { for _, test := range tests { req := scriptReq(test.id, test.height, test.body) - assertResponse(t, req, http.StatusBadRequest, test.out, backend) + assertResponse(t, req, http.StatusBadRequest, test.out, restHandler) } }) } diff --git a/engine/access/rest/test_helpers.go b/engine/access/rest/test_helpers.go index 88170769c99..fc62c8b6feb 100644 --- a/engine/access/rest/test_helpers.go +++ b/engine/access/rest/test_helpers.go @@ -3,17 +3,26 @@ package rest import ( "bytes" "fmt" + "net" "net/http" "net/http/httptest" "testing" + "time" + + "google.golang.org/grpc" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/access/mock" + engineaccessmock "github.com/onflow/flow-go/engine/access/mock" + restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/utils/grpcutils" + + "github.com/onflow/flow/protobuf/go/flow/access" ) const ( @@ -26,11 +35,12 @@ const ( heightQueryParam = "height" ) -func executeRequest(req *http.Request, backend *mock.API) (*httptest.ResponseRecorder, error) { +func executeRequest(req *http.Request, restHandler RestServerApi) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - restCollector := metrics.NewNoopCollector() - router, err := newRouter(backend, logger, flow.Testnet.Chain(), restCollector) + metrics := metrics.NewNoopCollector() + + router, err := newRouter(restHandler, logger, flow.Testnet.Chain(), metrics) if err != nil { return nil, err } @@ -40,14 +50,68 @@ func executeRequest(req *http.Request, backend *mock.API) (*httptest.ResponseRec return rr, nil } -func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend *mock.API) { - assertResponse(t, req, http.StatusOK, expectedRespBody, backend) +func newAccessRestHandler(backend *mock.API) RestServerApi { + var b bytes.Buffer + logger := zerolog.New(&b) + + return NewRequestHandler(logger, backend) } -func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend *mock.API) { - rr, err := executeRequest(req, backend) - assert.NoError(t, err) +func newObserverRestHandler_v2(backend *mock.API, restForwarder *restmock.RestServerApi) (RestServerApi, error) { + var b bytes.Buffer + logger := zerolog.New(&b) + observerCollector := metrics.NewObserverCollector() // + + return &RestRouter{ + Logger: logger, + Metrics: observerCollector, + Upstream: restForwarder, + Observer: NewRequestHandler(logger, backend), + }, nil +} + +func newObserverRestHandler(backend *mock.API, identities flow.IdentityList) (RestServerApi, error) { + var b bytes.Buffer + logger := zerolog.New(&b) + observerCollector := metrics.NewObserverCollector() + restForwarder, err := NewRestForwarder(logger, + identities, + time.Second, + grpcutils.DefaultMaxMsgSize) + if err != nil { + return nil, err + } + + return &RestRouter{ + Logger: logger, + Metrics: observerCollector, + Upstream: restForwarder, + Observer: NewRequestHandler(logger, backend), + }, nil +} + +func newGrpcServer(mockServer *engineaccessmock.AccessAPIServer, network string, address string, done chan int) (*grpc.Server, *net.Listener, error) { + l, err := net.Listen(network, address) + if err != nil { + return nil, nil, err + } + s := grpc.NewServer() + go func(done chan int) { + access.RegisterAccessAPIServer(s, mockServer) + _ = s.Serve(l) + done <- 1 + }(done) + return s, &l, nil +} + +func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, restHandler RestServerApi) { + assertResponse(t, req, http.StatusOK, expectedRespBody, restHandler) +} + +func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, restHandler RestServerApi) { + rr, err := executeRequest(req, restHandler) + assert.NoError(t, err) actualResponseBody := rr.Body.String() require.JSONEq(t, expectedRespBody, diff --git a/engine/access/rest/transactions_test.go b/engine/access/rest/transactions_test.go index 26710c747e5..c35980d4347 100644 --- a/engine/access/rest/transactions_test.go +++ b/engine/access/rest/transactions_test.go @@ -102,9 +102,11 @@ func validCreateBody(tx flow.TransactionBody) map[string]interface{} { } func TestGetTransactions(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("get by ID without results", func(t *testing.T) { - backend := &mock.API{} + tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -145,7 +147,7 @@ func TestGetTransactions(t *testing.T) { }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("Get by ID with results", func(t *testing.T) { @@ -214,19 +216,16 @@ func TestGetTransactions(t *testing.T) { } }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ReferenceBlockID, txr.CollectionID, tx.ID(), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("get by ID Invalid", func(t *testing.T) { - backend := &mock.API{} - req := getTransactionReq("invalid", false, "", "") expected := `{"code":400, "message":"invalid ID format"}` - assertResponse(t, req, http.StatusBadRequest, expected, backend) + assertResponse(t, req, http.StatusBadRequest, expected, restHandler) }) t.Run("get by ID non-existing", func(t *testing.T) { - backend := &mock.API{} tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -235,7 +234,7 @@ func TestGetTransactions(t *testing.T) { Return(nil, status.Error(codes.NotFound, "transaction not found")) expected := `{"code":404, "message":"Flow resource not found: transaction not found"}` - assertResponse(t, req, http.StatusNotFound, expected, backend) + assertResponse(t, req, http.StatusNotFound, expected, restHandler) }) } @@ -276,15 +275,17 @@ func TestGetTransactionResult(t *testing.T) { } }`, bid.String(), cid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + t.Run("get by transaction ID", func(t *testing.T) { - backend := &mock.API{} req := getTransactionResultReq(id.String(), "", "") backend.Mock. On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). Return(txr, nil) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("get by block ID", func(t *testing.T) { @@ -295,7 +296,7 @@ func TestGetTransactionResult(t *testing.T) { On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID). Return(txr, nil) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("get by collection ID", func(t *testing.T) { @@ -306,7 +307,7 @@ func TestGetTransactionResult(t *testing.T) { On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid). Return(txr, nil) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("get execution statuses", func(t *testing.T) { @@ -353,23 +354,23 @@ func TestGetTransactionResult(t *testing.T) { "_self": "/v1/transaction_results/%s" } }`, bid.String(), cid.String(), err, cases.Title(language.English).String(strings.ToLower(txResult.Status.String())), txResult.ErrorMessage, id.String()) - assertOKResponse(t, req, expectedResp, backend) + assertOKResponse(t, req, expectedResp, restHandler) } }) t.Run("get by ID Invalid", func(t *testing.T) { - backend := &mock.API{} req := getTransactionResultReq("invalid", "", "") expected := `{"code":400, "message":"invalid ID format"}` - assertResponse(t, req, http.StatusBadRequest, expected, backend) + assertResponse(t, req, http.StatusBadRequest, expected, restHandler) }) } func TestCreateTransaction(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) t.Run("create", func(t *testing.T) { - backend := &mock.API{} tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} tx.Arguments = [][]uint8{} @@ -417,11 +418,10 @@ func TestCreateTransaction(t *testing.T) { } }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.PayloadSignatures[0].Signature), util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, backend) + assertOKResponse(t, req, expected, restHandler) }) t.Run("post invalid transaction", func(t *testing.T) { - backend := &mock.API{} tests := []struct { inputField string inputValue string @@ -445,7 +445,7 @@ func TestCreateTransaction(t *testing.T) { testTx[test.inputField] = test.inputValue req := createTransactionReq(testTx) - assertResponse(t, req, http.StatusBadRequest, test.output, backend) + assertResponse(t, req, http.StatusBadRequest, test.output, restHandler) } }) } diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 3cce6e97fda..4cbe5a6e495 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -1,219 +1,248 @@ package rpc -import ( - "context" - "fmt" - "io" - "os" - "testing" - "time" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" - - accessmock "github.com/onflow/flow-go/engine/access/mock" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/irrecoverable" - "github.com/onflow/flow-go/module/metrics" - module "github.com/onflow/flow-go/module/mock" - "github.com/onflow/flow-go/network" - protocol "github.com/onflow/flow-go/state/protocol/mock" - storagemock "github.com/onflow/flow-go/storage/mock" - "github.com/onflow/flow-go/utils/grpcutils" - "github.com/onflow/flow-go/utils/unittest" -) - -type RateLimitTestSuite struct { - suite.Suite - state *protocol.State - snapshot *protocol.Snapshot - epochQuery *protocol.EpochQuery - log zerolog.Logger - net *network.Network - request *module.Requester - collClient *accessmock.AccessAPIClient - execClient *accessmock.ExecutionAPIClient - me *module.Local - chainID flow.ChainID - metrics *metrics.NoopCollector - rpcEng *Engine - client accessproto.AccessAPIClient - closer io.Closer - - // storage - blocks *storagemock.Blocks - headers *storagemock.Headers - collections *storagemock.Collections - transactions *storagemock.Transactions - receipts *storagemock.ExecutionReceipts - - // test rate limit - rateLimit int - burstLimit int - - ctx irrecoverable.SignalerContext - cancel context.CancelFunc -} - -func (suite *RateLimitTestSuite) SetupTest() { - suite.log = zerolog.New(os.Stdout) - suite.net = new(network.Network) - suite.state = new(protocol.State) - suite.snapshot = new(protocol.Snapshot) - - suite.epochQuery = new(protocol.EpochQuery) - suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() - suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() - suite.blocks = new(storagemock.Blocks) - suite.headers = new(storagemock.Headers) - suite.transactions = new(storagemock.Transactions) - suite.collections = new(storagemock.Collections) - suite.receipts = new(storagemock.ExecutionReceipts) - - suite.collClient = new(accessmock.AccessAPIClient) - suite.execClient = new(accessmock.ExecutionAPIClient) - - suite.request = new(module.Requester) - suite.request.On("EntityByID", mock.Anything, mock.Anything) - - suite.me = new(module.Local) - - accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) - suite.me. - On("NodeID"). - Return(accessIdentity.NodeID) - - suite.chainID = flow.Testnet - suite.metrics = metrics.NewNoopCollector() - - config := Config{ - UnsecureGRPCListenAddr: unittest.DefaultAddress, - SecureGRPCListenAddr: unittest.DefaultAddress, - HTTPListenAddr: unittest.DefaultAddress, - } - - // set the rate limit to test with - suite.rateLimit = 2 - // set the burst limit to test with - suite.burstLimit = 2 - - apiRateLimt := map[string]int{ - "Ping": suite.rateLimit, - } - - apiBurstLimt := map[string]int{ - "Ping": suite.rateLimit, - } - - block := unittest.BlockHeaderFixture() - suite.snapshot.On("Head").Return(block, nil) - - rpcEngBuilder, err := NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, 0, 0, false, false, apiRateLimt, apiBurstLimt, suite.me) - require.NoError(suite.T(), err) - suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() - require.NoError(suite.T(), err) - suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) - suite.rpcEng.Start(suite.ctx) - // wait for the server to startup - unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") - - // create the access api client - suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) - require.NoError(suite.T(), err) -} - -func (suite *RateLimitTestSuite) TearDownTest() { - if suite.cancel != nil { - suite.cancel() - } - // close the client - if suite.closer != nil { - suite.closer.Close() - } - // close the server - unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) -} - -func TestRateLimit(t *testing.T) { - suite.Run(t, new(RateLimitTestSuite)) -} - -// TestRatelimitingWithoutBurst tests that rate limit is correctly applied to an Access API call -func (suite *RateLimitTestSuite) TestRatelimitingWithoutBurst() { - - req := &accessproto.PingRequest{} - ctx := context.Background() - - // expect 2 upstream calls - suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) - suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) - - requestCnt := 0 - // requests within the burst should succeed - for requestCnt < suite.rateLimit { - resp, err := suite.client.Ping(ctx, req) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), resp) - // sleep to prevent burst - time.Sleep(100 * time.Millisecond) - requestCnt++ - } - - // request more than the limit should fail - _, err := suite.client.Ping(ctx, req) - suite.assertRateLimitError(err) -} - -// TestRatelimitingWithBurst tests that burst limit is correctly applied to an Access API call -func (suite *RateLimitTestSuite) TestRatelimitingWithBurst() { - - req := &accessproto.PingRequest{} - ctx := context.Background() - - // expect rpc.defaultBurst number of upstream calls - suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) - suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) - - requestCnt := 0 - // generate a permissible burst of request and assert that they succeed - for requestCnt < suite.burstLimit { - resp, err := suite.client.Ping(ctx, req) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), resp) - requestCnt++ - } - - // request more than the permissible burst and assert that it fails - _, err := suite.client.Ping(ctx, req) - suite.assertRateLimitError(err) -} - -func (suite *RateLimitTestSuite) assertRateLimitError(err error) { - assert.Error(suite.T(), err) - status, ok := status.FromError(err) - assert.True(suite.T(), ok) - assert.Equal(suite.T(), codes.ResourceExhausted, status.Code()) -} - -func accessAPIClient(address string) (accessproto.AccessAPIClient, io.Closer, error) { - conn, err := grpc.Dial( - address, - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), - grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - return nil, nil, fmt.Errorf("failed to connect to address %s: %w", address, err) - } - client := accessproto.NewAccessAPIClient(conn) - closer := io.Closer(conn) - return client, closer, nil -} +// import ( +// "context" +// "fmt" +// "io" +// "os" +// "testing" +// "time" + +// "github.com/rs/zerolog" +// "github.com/stretchr/testify/assert" +// "github.com/stretchr/testify/mock" +// "github.com/stretchr/testify/require" +// "github.com/stretchr/testify/suite" +// "google.golang.org/grpc" +// "google.golang.org/grpc/codes" +// "google.golang.org/grpc/credentials/insecure" +// "google.golang.org/grpc/status" + +// accessengine "github.com/onflow/flow-go/engine/access" +// accessmock "github.com/onflow/flow-go/engine/access/mock" +// "github.com/onflow/flow-go/model/flow" +// "github.com/onflow/flow-go/module/irrecoverable" +// "github.com/onflow/flow-go/module/metrics" +// module "github.com/onflow/flow-go/module/mock" +// "github.com/onflow/flow-go/network" +// protocol "github.com/onflow/flow-go/state/protocol/mock" +// storagemock "github.com/onflow/flow-go/storage/mock" +// "github.com/onflow/flow-go/utils/grpcutils" +// "github.com/onflow/flow-go/utils/unittest" +// accessproto "github.com/onflow/flow/protobuf/go/flow/access" +// ) + +// type RateLimitTestSuite struct { +// suite.Suite +// state *protocol.State +// snapshot *protocol.Snapshot +// epochQuery *protocol.EpochQuery +// log zerolog.Logger +// net *network.Network +// request *module.Requester +// collClient *accessmock.AccessAPIClient +// execClient *accessmock.ExecutionAPIClient +// me *module.Local +// chainID flow.ChainID +// metrics *metrics.NoopCollector +// rpcEng *Engine +// client accessproto.AccessAPIClient +// closer io.Closer + +// // storage +// blocks *storagemock.Blocks +// headers *storagemock.Headers +// collections *storagemock.Collections +// transactions *storagemock.Transactions +// receipts *storagemock.ExecutionReceipts + +// // test rate limit +// rateLimit int +// burstLimit int + +// ctx irrecoverable.SignalerContext +// cancel context.CancelFunc +// } + +// func (suite *RateLimitTestSuite) SetupTest() { +// suite.log = zerolog.New(os.Stdout) +// suite.net = new(network.Network) +// suite.state = new(protocol.State) +// suite.snapshot = new(protocol.Snapshot) + +// suite.epochQuery = new(protocol.EpochQuery) +// suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() +// suite.state.On("Final").Return(suite.snapshot, nil).Maybe() +// suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() +// suite.blocks = new(storagemock.Blocks) +// suite.headers = new(storagemock.Headers) +// suite.transactions = new(storagemock.Transactions) +// suite.collections = new(storagemock.Collections) +// suite.receipts = new(storagemock.ExecutionReceipts) + +// suite.collClient = new(accessmock.AccessAPIClient) +// suite.execClient = new(accessmock.ExecutionAPIClient) + +// suite.request = new(module.Requester) +// suite.request.On("EntityByID", mock.Anything, mock.Anything) + +// suite.me = new(module.Local) + +// accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) +// suite.me. +// On("NodeID"). +// Return(accessIdentity.NodeID) + +// suite.chainID = flow.Testnet +// suite.metrics = metrics.NewNoopCollector() + +// config := Config{ +// UnsecureGRPCListenAddr: unittest.DefaultAddress, +// SecureGRPCListenAddr: unittest.DefaultAddress, +// HTTPListenAddr: unittest.DefaultAddress, +// } + +// // set the rate limit to test with +// suite.rateLimit = 2 +// // set the burst limit to test with +// suite.burstLimit = 2 + +// apiRateLimt := map[string]int{ +// "Ping": suite.rateLimit, +// } + +// apiBurstLimt := map[string]int{ +// "Ping": suite.rateLimit, +// } + +// block := unittest.BlockHeaderFixture() +// suite.snapshot.On("Head").Return(block, nil) + +// backend, err := accessengine.NewBackend( +// suite.log, +// suite.state, +// config, +// suite.collClient, +// nil, +// suite.blocks, +// suite.headers, +// suite.collections, +// suite.transactions, +// nil, +// nil, +// suite.chainID, +// suite.metrics, +// 0, +// 0, +// false) +// require.NoError(suite.T(), err) + +// rpcEngBuilder, err := NewBuilder( +// suite.log, +// suite.state, +// config, +// suite.chainID, +// suite.metrics, +// false, +// apiRateLimt, +// apiBurstLimt, +// suite.me, +// backend) +// require.NoError(suite.T(), err) +// suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() +// require.NoError(suite.T(), err) +// suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) +// suite.rpcEng.Start(suite.ctx) +// // wait for the server to startup +// unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") + +// // create the access api client +// suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) +// require.NoError(suite.T(), err) +// } + +// func (suite *RateLimitTestSuite) TearDownTest() { +// if suite.cancel != nil { +// suite.cancel() +// } +// // close the client +// if suite.closer != nil { +// suite.closer.Close() +// } +// // close the server +// unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) +// } + +// func TestRateLimit(t *testing.T) { +// suite.Run(t, new(RateLimitTestSuite)) +// } + +// // TestRatelimitingWithoutBurst tests that rate limit is correctly applied to an Access API call +// func (suite *RateLimitTestSuite) TestRatelimitingWithoutBurst() { + +// req := &accessproto.PingRequest{} +// ctx := context.Background() + +// // expect 2 upstream calls +// suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) +// suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) + +// requestCnt := 0 +// // requests within the burst should succeed +// for requestCnt < suite.rateLimit { +// resp, err := suite.client.Ping(ctx, req) +// assert.NoError(suite.T(), err) +// assert.NotNil(suite.T(), resp) +// // sleep to prevent burst +// time.Sleep(100 * time.Millisecond) +// requestCnt++ +// } + +// // request more than the limit should fail +// _, err := suite.client.Ping(ctx, req) +// suite.assertRateLimitError(err) +// } + +// // TestRatelimitingWithBurst tests that burst limit is correctly applied to an Access API call +// func (suite *RateLimitTestSuite) TestRatelimitingWithBurst() { + +// req := &accessproto.PingRequest{} +// ctx := context.Background() + +// // expect rpc.defaultBurst number of upstream calls +// suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) +// suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) + +// requestCnt := 0 +// // generate a permissible burst of request and assert that they succeed +// for requestCnt < suite.burstLimit { +// resp, err := suite.client.Ping(ctx, req) +// assert.NoError(suite.T(), err) +// assert.NotNil(suite.T(), resp) +// requestCnt++ +// } + +// // request more than the permissible burst and assert that it fails +// _, err := suite.client.Ping(ctx, req) +// suite.assertRateLimitError(err) +// } + +// func (suite *RateLimitTestSuite) assertRateLimitError(err error) { +// assert.Error(suite.T(), err) +// status, ok := status.FromError(err) +// assert.True(suite.T(), ok) +// assert.Equal(suite.T(), codes.ResourceExhausted, status.Code()) +// } + +// func accessAPIClient(address string) (accessproto.AccessAPIClient, io.Closer, error) { +// conn, err := grpc.Dial( +// address, +// grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), +// grpc.WithTransportCredentials(insecure.NewCredentials())) +// if err != nil { +// return nil, nil, fmt.Errorf("failed to connect to address %s: %w", address, err) +// } +// client := accessproto.NewAccessAPIClient(conn) +// closer := io.Closer(conn) +// return client, closer, nil +// } diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index b82160668db..3c96ca2740c 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -11,6 +11,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -108,7 +109,7 @@ func (suite *SecureGRPCTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - rpcEngBuilder, err := rpc.NewBuilder( + backend, err := NewBackend( suite.log, suite.state, config, @@ -124,11 +125,20 @@ func (suite *SecureGRPCTestSuite) SetupTest() { suite.metrics, 0, 0, - false, + false) + require.NoError(suite.T(), err) + + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, false, nil, nil, suite.me, + backend, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() From 8cac7afaa3cb16cd66427864feb96c1abbfaf526 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 16 Jun 2023 14:38:54 -0600 Subject: [PATCH 041/169] replace block header ID by beacon source in execution PRG generation --- engine/execution/computation/manager.go | 1 + fvm/context.go | 15 +++++++-- fvm/environment/env.go | 3 ++ fvm/environment/facade_env.go | 8 ++++- fvm/environment/unsafe_random_generator.go | 32 ++++++++++++------- .../unsafe_random_generator_test.go | 9 ++++-- fvm/fvm_blockcontext_test.go | 11 +++++-- 7 files changed, 59 insertions(+), 20 deletions(-) diff --git a/engine/execution/computation/manager.go b/engine/execution/computation/manager.go index ae45c80fd89..a4331ca31bd 100644 --- a/engine/execution/computation/manager.go +++ b/engine/execution/computation/manager.go @@ -118,6 +118,7 @@ func New( }, ), ), + fvm.WithProtocolState(protoState), } if params.ExtensiveTracing { options = append(options, fvm.WithExtensiveTracing()) diff --git a/fvm/context.go b/fvm/context.go index a1c25541360..1ce1eb297eb 100644 --- a/fvm/context.go +++ b/fvm/context.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/state/protocol" ) const ( @@ -160,10 +161,20 @@ func WithEventCollectionSizeLimit(limit uint64) Option { } } +// WithProtocolState sets the protocol state for a virtual machine context. +// +// The VM uses the protocol state to provide protocol information to the Cadence runtime, +// including the source of the pseudorandom number generator. +func WithProtocolState(protocolState protocol.State) Option { + return func(ctx Context) Context { + ctx.State = protocolState + return ctx + } +} + // WithBlockHeader sets the block header for a virtual machine context. // -// The VM uses the header to provide current block information to the Cadence runtime, -// as well as to seed the pseudorandom number generator. +// The VM uses the header to provide current block information to the Cadence runtime. func WithBlockHeader(header *flow.Header) Option { return func(ctx Context) Context { ctx.BlockHeader = header diff --git a/fvm/environment/env.go b/fvm/environment/env.go index 886a82be701..97e23b060c2 100644 --- a/fvm/environment/env.go +++ b/fvm/environment/env.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/protocol" ) // Environment implements the accounts business logic and exposes cadence @@ -98,6 +99,8 @@ type EnvironmentParams struct { BlockInfoParams TransactionInfoParams + protocol.State + ContractUpdaterParams } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 0e36b9ce8c0..51e94fb8f05 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" + "github.com/onflow/flow-go/state/protocol" ) var _ Environment = &facadeEnvironment{} @@ -65,6 +66,11 @@ func newFacadeEnvironment( logger, runtime) + var protocolSnapshot protocol.Snapshot + if params.State != nil && params.BlockHeader != nil { + protocolSnapshot = params.State.AtBlockID(params.BlockHeader.ID()) + } + env := &facadeEnvironment{ Runtime: runtime, @@ -76,7 +82,7 @@ func newFacadeEnvironment( UnsafeRandomGenerator: NewUnsafeRandomGenerator( tracer, - params.BlockHeader, + protocolSnapshot, params.TxId, ), CryptoLibrary: NewCryptoLibrary(tracer, meter), diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index aa931905e54..2591640c808 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/protocol" ) type UnsafeRandomGenerator interface { @@ -26,8 +27,8 @@ type UnsafeRandomGenerator interface { type unsafeRandomGenerator struct { tracer tracing.TracerSpan - blockHeader *flow.Header - txId flow.Identifier + stateSnapshot protocol.Snapshot + txId flow.Identifier prg random.Rand createOnce sync.Once @@ -61,13 +62,13 @@ func (gen ParseRestrictedUnsafeRandomGenerator) UnsafeRandom() ( func NewUnsafeRandomGenerator( tracer tracing.TracerSpan, - blockHeader *flow.Header, + stateSnapshot protocol.Snapshot, txId flow.Identifier, ) UnsafeRandomGenerator { gen := &unsafeRandomGenerator{ - tracer: tracer, - blockHeader: blockHeader, - txId: txId, + tracer: tracer, + stateSnapshot: stateSnapshot, + txId: txId, } return gen @@ -77,14 +78,21 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( random.Rand, error, ) { - if gen.blockHeader == nil { + if gen.stateSnapshot == nil { return nil, nil } - // The block header ID is currently used as the entropy source. - // This should evolve to become the beacon signature (safer entropy - // source than the block ID) - source := gen.blockHeader.ID() + // Use the protocol state source of randomness for the currrent block + // execution (which is the randmoness beacon output for thiss block) + source, err := gen.stateSnapshot.RandomSource() + // expected errors of RandomSource() are : + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations + if err != nil { + return nil, fmt.Errorf("reading random source from state failed: %w", err) + } // Diversify the seed per transaction ID salt := gen.txId[:] @@ -98,7 +106,7 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( salt, nil) seed := make([]byte, random.Chacha20SeedLen) - _, err := io.ReadFull(hkdf, seed) + _, err = io.ReadFull(hkdf, seed) if err != nil { return nil, fmt.Errorf("extracting seed with HKDF failed: %w", err) } diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/unsafe_random_generator_test.go index 8874fa4477a..83089129d07 100644 --- a/fvm/environment/unsafe_random_generator_test.go +++ b/fvm/environment/unsafe_random_generator_test.go @@ -11,17 +11,20 @@ import ( "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" + pmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) func TestUnsafeRandomGenerator(t *testing.T) { - bh := unittest.BlockHeaderFixtureOnChain(flow.Mainnet.Chain().ChainID()) + // protocol snapshot mock + snapshot := &pmock.Snapshot{} + snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) getRandoms := func(txId flow.Identifier, N int) []uint64 { // seed the RG with the same block header urg := environment.NewUnsafeRandomGenerator( tracing.NewTracerSpan(), - bh, + snapshot, txId) numbers := make([]uint64, N) for i := 0; i < N; i++ { @@ -39,7 +42,7 @@ func TestUnsafeRandomGenerator(t *testing.T) { txId := unittest.TransactionFixture().ID() urg := environment.NewUnsafeRandomGenerator( tracing.NewTracerSpan(), - bh, + snapshot, txId) // make sure n is a power of 2 so that there is no bias in the last class diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index bb94ad2abb9..7d3f9f5ca73 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -24,6 +24,7 @@ import ( errors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" + pmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -1670,10 +1671,15 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { chain, vm := createChainAndVm(flow.Mainnet) header := &flow.Header{Height: 42} + snapshot := pmock.Snapshot{} + state := pmock.State{} + state.On("AtBlockID", mock.Anything).Return(&snapshot) + snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) ctx := fvm.NewContext( fvm.WithChain(chain), fvm.WithBlockHeader(header), + fvm.WithProtocolState(&state), fvm.WithCadenceLogging(true), ) @@ -1700,9 +1706,10 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { require.Len(t, output.Logs, 1) - num, err := strconv.ParseUint(output.Logs[0], 10, 64) + // output cannot be deterministic because transaction signature is not deterministic + // (which makes the tx hash and the PRG seed used by the execution not deterministic) + _, err = strconv.ParseUint(output.Logs[0], 10, 64) require.NoError(t, err) - require.Equal(t, uint64(0x7515f254adc6f8af), num) }) } From 7c940d2b82c9fe32d63eb22e7f508df063898a02 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 16 Jun 2023 18:18:51 -0600 Subject: [PATCH 042/169] use state/protocol/prg package to instanciate a CSPRG --- fvm/environment/unsafe_random_generator.go | 43 +++++++--------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index 2591640c808..15f89f7d45b 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -1,15 +1,10 @@ package environment import ( - "crypto/sha256" "encoding/binary" "fmt" - "hash" - "io" "sync" - "golang.org/x/crypto/hkdf" - "github.com/onflow/flow-go/crypto/random" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/storage/state" @@ -17,6 +12,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/state/protocol/prg" ) type UnsafeRandomGenerator interface { @@ -82,8 +78,8 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( return nil, nil } - // Use the protocol state source of randomness for the currrent block - // execution (which is the randmoness beacon output for thiss block) + // Use the protocol state source of randomness [SoR] for the current block + // execution source, err := gen.stateSnapshot.RandomSource() // expected errors of RandomSource() are : // - storage.ErrNotFound if the QC is unknown. @@ -94,36 +90,23 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( return nil, fmt.Errorf("reading random source from state failed: %w", err) } - // Diversify the seed per transaction ID - salt := gen.txId[:] - - // Extract the entropy from the source and expand it into the required - // seed length. Note that we can use any implementation which provide - // similar properties. - hkdf := hkdf.New( - func() hash.Hash { return sha256.New() }, - source[:], - salt, - nil) - seed := make([]byte, random.Chacha20SeedLen) - _, err = io.ReadFull(hkdf, seed) - if err != nil { - return nil, fmt.Errorf("extracting seed with HKDF failed: %w", err) - } - - // initialize a fresh crypto-secure PRG with the seed (here ChaCha20) - // This PRG provides all outputs of Cadence UnsafeRandom. - prg, err := random.NewChacha20PRG(seed, []byte{}) + // Use the state/protocol PRG derivation from the source of randomness: + // - for the transaction execution case, the PRG used must be a CSPRG + // - use the state/protocol/prg customizer defined for the execution environment + // - use the transaction ID as an extra diversifier of the CSPRG. Although this + // does not add any extra entropy to the output, it allows creating an independent + // PRG for each transaction. + csprg, err := prg.FromRandomSource(source, prg.ExecutionEnvironment, gen.txId[:]) if err != nil { - return nil, fmt.Errorf("creating random generator failed: %w", err) + return nil, fmt.Errorf("failed to create a CSPRG from source: %w", err) } - return prg, nil + return csprg, nil } // maybeCreateRandomGenerator seeds the pseudo-random number generator using the // block header ID and transaction index as an entropy source. The seed -// function is currently called for each tranaction, the PRG is used to +// function is currently called for each transaction, the PRG is used to // provide all the randoms the transaction needs through UnsafeRandom. // // This allows lazy seeding of the random number generator, since not a lot of From ee5b7c561178f3a745a7fb53ffce5b8e5b55960c Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 19 Jun 2023 15:18:14 +0300 Subject: [PATCH 043/169] Refactored tests ande added new tests for rest servise. Linted. --- .../node_builder/access_node_builder.go | 17 +- cmd/observer/node_builder/observer_builder.go | 28 +- engine/access/apiproxy/access_api_proxy.go | 7 +- engine/access/backend.go | 93 ---- engine/access/rest/accounts_test.go | 409 +++++++------- engine/access/rest/blocks_test.go | 67 +-- engine/access/rest/handler.go | 9 +- engine/access/rest/rest_server_api.go | 19 +- engine/access/rest/scripts_test.go | 17 +- engine/access/rest/test_helpers.go | 48 +- engine/access/rest/transactions_test.go | 28 +- engine/access/rest_api_test.go | 15 +- engine/access/rpc/backend/backend.go | 84 +++ engine/access/rpc/engine_builder.go | 2 +- engine/access/rpc/rate_limit_test.go | 500 +++++++++--------- engine/access/secure_grpcr_test.go | 14 +- module/forwarder/forwarder.go | 4 +- 17 files changed, 684 insertions(+), 677 deletions(-) delete mode 100644 engine/access/backend.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 0482b600e66..12e6e03a683 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -35,7 +35,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" - accessengine "github.com/onflow/flow-go/engine/access" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" "github.com/onflow/flow-go/engine/access/rpc" @@ -992,9 +991,9 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - backend, err := accessengine.NewBackend(node.Logger, + config := builder.rpcConf + backend, err := backend.NewBackend(node.Logger, node.State, - builder.rpcConf, builder.CollectionRPC, builder.HistoricalAccessRPCs, node.Storage.Blocks, @@ -1007,7 +1006,15 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.AccessMetrics, builder.collectionGRPCPort, builder.executionGRPCPort, - builder.retryEnabled) + builder.retryEnabled, + config.MaxHeightRange, + config.ExecutionClientTimeout, + config.CollectionClientTimeout, + config.ConnectionPoolSize, + config.MaxHeightRange, + config.PreferredExecutionNodeIDs, + config.FixedExecutionNodeIDs, + config.ArchiveAddressList) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) } @@ -1015,7 +1022,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, - builder.rpcConf, + config, node.RootChainID, builder.AccessMetrics, builder.rpcMetricsEnabled, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 2b82705168d..ddef152feda 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -28,7 +28,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/verification" recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" - accessengine "github.com/onflow/flow-go/engine/access" "github.com/onflow/flow-go/engine/access/apiproxy" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc" @@ -851,9 +850,9 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { accessMetrics := metrics.NewNoopCollector() - accessBackend, err := accessengine.NewBackend(node.Logger, + config := builder.rpcConf + accessBackend, err := backend.NewBackend(node.Logger, node.State, - builder.rpcConf, nil, nil, node.Storage.Blocks, @@ -866,7 +865,16 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { accessMetrics, 0, 0, - false) + false, + config.MaxHeightRange, + config.ExecutionClientTimeout, + config.CollectionClientTimeout, + config.ConnectionPoolSize, + config.MaxHeightRange, + config.PreferredExecutionNodeIDs, + config.FixedExecutionNodeIDs, + config.ArchiveAddressList) + if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) } @@ -874,7 +882,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, - builder.rpcConf, + config, node.RootChainID, accessMetrics, builder.rpcMetricsEnabled, @@ -888,16 +896,16 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { } // upstream access node forwarder - forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, builder.rpcConf.MaxMsgSize) + forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) if err != nil { return nil, err } - metrics := metrics.NewObserverCollector() + observerCollector := metrics.NewObserverCollector() rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, - Metrics: metrics, + Metrics: observerCollector, Upstream: forwarder, Observer: protocol.NewHandler(protocol.New( node.State, @@ -910,14 +918,14 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { restForwarder, err := rest.NewRestForwarder(builder.Logger, builder.upstreamIdentities, builder.apiTimeout, - builder.rpcConf.MaxMsgSize) + config.MaxMsgSize) if err != nil { return nil, err } restHandler := &rest.RestRouter{ Logger: builder.Logger, - Metrics: metrics, + Metrics: observerCollector, Upstream: restForwarder, Observer: rest.NewRequestHandler(builder.Logger, accessBackend), } diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index 08136531b09..2b345420229 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -207,7 +207,7 @@ func (h *FlowAccessAPIRouter) GetExecutionResultByID(context context.Context, re // FlowAccessAPIForwarder forwards all requests to a set of upstream access nodes or observers type FlowAccessAPIForwarder struct { - forwarder.Forwarder + *forwarder.Forwarder } func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*FlowAccessAPIForwarder, error) { @@ -216,10 +216,9 @@ func NewFlowAccessAPIForwarder(identities flow.IdentityList, timeout time.Durati return nil, err } - accessApiForwarder := &FlowAccessAPIForwarder{ + return &FlowAccessAPIForwarder{ Forwarder: forwarder, - } - return accessApiForwarder, nil + }, nil } // Ping pings the service. It is special in the sense that it responds successful, diff --git a/engine/access/backend.go b/engine/access/backend.go deleted file mode 100644 index 92c8f932b2d..00000000000 --- a/engine/access/backend.go +++ /dev/null @@ -1,93 +0,0 @@ -package access - -import ( - "fmt" - - lru "github.com/hashicorp/golang-lru" - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine/access/rpc" - "github.com/onflow/flow-go/engine/access/rpc/backend" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/state/protocol" - "github.com/onflow/flow-go/storage" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" -) - -func NewBackend( - log zerolog.Logger, - state protocol.State, - config rpc.Config, - collectionRPC accessproto.AccessAPIClient, - historicalAccessNodes []accessproto.AccessAPIClient, - blocks storage.Blocks, - headers storage.Headers, - collections storage.Collections, - transactions storage.Transactions, - executionReceipts storage.ExecutionReceipts, - executionResults storage.ExecutionResults, - chainID flow.ChainID, - accessMetrics module.AccessMetrics, - collectionGRPCPort uint, - executionGRPCPort uint, - retryEnabled bool) (*backend.Backend, error) { - - var cache *lru.Cache - cacheSize := config.ConnectionPoolSize - if cacheSize > 0 { - // TODO: remove this fallback after fixing issues with evictions - // It was observed that evictions cause connection errors for in flight requests. This works around - // the issue by forcing hte pool size to be greater than the number of ENs + LNs - if cacheSize < backend.DefaultConnectionPoolSize { - log.Warn().Msg("connection pool size below threshold, setting pool size to default value ") - cacheSize = backend.DefaultConnectionPoolSize - } - var err error - cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { - store := evictedValue.(*backend.CachedClient) - store.Close() - log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") - if accessMetrics != nil { - accessMetrics.ConnectionFromPoolEvicted() - } - }) - if err != nil { - return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) - } - } - - connectionFactory := &backend.ConnectionFactoryImpl{ - CollectionGRPCPort: collectionGRPCPort, - ExecutionGRPCPort: executionGRPCPort, - CollectionNodeGRPCTimeout: config.CollectionClientTimeout, - ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, - ConnectionsCache: cache, - CacheSize: cacheSize, - MaxMsgSize: config.MaxMsgSize, - AccessMetrics: accessMetrics, - Log: log, - } - - return backend.New(state, - collectionRPC, - historicalAccessNodes, - blocks, - headers, - collections, - transactions, - executionReceipts, - executionResults, - chainID, - accessMetrics, - connectionFactory, - retryEnabled, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - log, - backend.DefaultSnapshotHistoryLimit, - config.ArchiveAddressList, - ), nil -} diff --git a/engine/access/rest/accounts_test.go b/engine/access/rest/accounts_test.go index 0446fc4958d..d248bd312c8 100644 --- a/engine/access/rest/accounts_test.go +++ b/engine/access/rest/accounts_test.go @@ -13,6 +13,9 @@ import ( "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/middleware" + restmock "github.com/onflow/flow-go/engine/access/rest/mock" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -33,7 +36,7 @@ func accountURL(t *testing.T, address string, height string) string { return u.String() } -func TestGetAccount(t *testing.T) { +func TestAccessGetAccount(t *testing.T) { backend := &mock.API{} restHandler := newAccessRestHandler(backend) @@ -128,205 +131,210 @@ func TestGetAccount(t *testing.T) { }) } -//func TestObserverGetAccount(t *testing.T) { -// backend := &mock.API{} -// address := unittest.IPPort("11632") -// -// t.Run("get by address at latest sealed block", func(t *testing.T) { -// account := accountFixture(t) -// entityAccount, err := convert.AccountToMessage(account) -// assert.NoError(t, err) -// -// var height uint64 = 100 -// blockHeader, err := convert.BlockHeaderToMessage(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)), unittest.IdentifierListFixture(4)) //? -// assert.NoError(t, err) -// -// req := getAccountRequest(t, account, sealedHeightQueryParam, expandableFieldKeys, expandableFieldContracts) -// -// done := make(chan int) -// // Bring up 1st upstream server -// mockServer := new(engineaccessmock.AccessAPIServer) -// mockServer.Mock. -// On("GetLatestBlockHeader", mocktestify.Anything, -// &access.GetLatestBlockHeaderRequest{ -// IsSealed: true, -// }). -// Return(&access.BlockHeaderResponse{ -// Block: blockHeader, -// BlockStatus: entities.BlockStatus_BLOCK_SEALED, -// Metadata: nil, -// }, nil) -// -// mockServer.Mock. -// On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ -// Address: account.Address.Bytes(), -// BlockHeight: height, -// }). -// Return(&access.AccountResponse{ -// Account: entityAccount, -// Metadata: nil, -// }, nil) -// expected := expectedExpandedResponse(account) -// -// server, _, err := newGrpcServer(mockServer, "tcp", address, done) -// assert.NoError(t, err) -// -// restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) -// assert.NoError(t, err) -// -// assertOKResponse(t, req, expected, restHandler) -// mocktestify.AssertExpectationsForObjects(t, backend) -// -// server.Stop() -// <-done -// }) -// -// //t.Run("get by address at latest finalized block", func(t *testing.T) { -// // var height uint64 = 100 -// // blockHeader, err := convert.BlockHeaderToMessage(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)), unittest.IdentifierListFixture(4)) -// // assert.NoError(t, err) -// // account := accountFixture(t) -// // entityAccount, err := convert.AccountToMessage(account) -// // assert.NoError(t, err) -// // -// // req := getAccountRequest(t, account, finalHeightQueryParam, expandableFieldKeys, expandableFieldContracts) -// // -// // done := make(chan int) -// // // Bring up 1st upstream server -// // mockServer := new(engineaccessmock.AccessAPIServer) -// // mockServer.Mock. -// // On("GetLatestBlockHeader", mocktestify.Anything, -// // &access.GetLatestBlockHeaderRequest{ -// // IsSealed: false, -// // }). -// // Return(&access.BlockHeaderResponse{ -// // Block: blockHeader, -// // BlockStatus: entities.BlockStatus_BLOCK_FINALIZED, -// // Metadata: nil, -// // }, nil) -// // mockServer.Mock. -// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ -// // Address: account.Address.Bytes(), -// // BlockHeight: height, -// // }). -// // Return(&access.AccountResponse{ -// // Account: entityAccount, -// // Metadata: nil, -// // }, nil) -// // expected := expectedExpandedResponse(account) -// // -// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) -// // assert.NoError(t, err) -// // -// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) -// // assert.NoError(t, err) -// // -// // assertOKResponse(t, req, expected, restHandler) -// // mocktestify.AssertExpectationsForObjects(t, backend) -// // -// // server.Stop() -// // <-done -// //}) -// // -// //t.Run("get by address at height", func(t *testing.T) { -// // var height uint64 = 1337 -// // account := accountFixture(t) -// // entityAccount, err := convert.AccountToMessage(account) -// // assert.NoError(t, err) -// // -// // req := getAccountRequest(t, account, fmt.Sprintf("%d", height), expandableFieldKeys, expandableFieldContracts) -// // -// // done := make(chan int) -// // // Bring up 1st upstream server -// // mockServer := new(engineaccessmock.AccessAPIServer) -// // mockServer.Mock. -// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ -// // Address: account.Address.Bytes(), -// // BlockHeight: height, -// // }). -// // Return(&access.AccountResponse{ -// // Account: entityAccount, -// // Metadata: nil, -// // }, nil) -// // expected := expectedExpandedResponse(account) -// // -// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) -// // assert.NoError(t, err) -// // -// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) -// // assert.NoError(t, err) -// // -// // assertOKResponse(t, req, expected, restHandler) -// // mocktestify.AssertExpectationsForObjects(t, backend) -// // -// // server.Stop() -// // <-done -// //}) -// // -// //t.Run("get by address at height condensed", func(t *testing.T) { -// // var height uint64 = 1337 -// // account := accountFixture(t) -// // entityAccount, err := convert.AccountToMessage(account) -// // assert.NoError(t, err) -// // -// // req := getAccountRequest(t, account, fmt.Sprintf("%d", height)) -// // -// // done := make(chan int) -// // // Bring up 1st upstream server -// // mockServer := new(engineaccessmock.AccessAPIServer) -// // mockServer.Mock. -// // On("GetAccountAtBlockHeight", mocktestify.Anything, &access.GetAccountAtBlockHeightRequest{ -// // Address: account.Address.Bytes(), -// // BlockHeight: height, -// // }). -// // Return(&access.AccountResponse{ -// // Account: entityAccount, -// // Metadata: nil, -// // }, nil) -// // expected := expectedCondensedResponse(account) -// // -// // server, _, err := newGrpcServer(mockServer, "tcp", address, done) -// // assert.NoError(t, err) -// // -// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) -// // assert.NoError(t, err) -// // -// // assertOKResponse(t, req, expected, restHandler) -// // mocktestify.AssertExpectationsForObjects(t, backend) -// // -// // server.Stop() -// // <-done -// //}) -// // -// //t.Run("get invalid", func(t *testing.T) { -// // tests := []struct { -// // url string -// // out string -// // }{ -// // {accountURL(t, "123", ""), `{"code":400, "message":"invalid address"}`}, -// // {accountURL(t, unittest.AddressFixture().String(), "foo"), `{"code":400, "message":"invalid height format"}`}, -// // } -// // -// // done := make(chan int) -// // // Bring up 1st upstream server -// // server, _, err := newGrpcServer(new(engineaccessmock.AccessAPIServer), "tcp", address, done) -// // assert.NoError(t, err) -// // -// // restHandler, err := newObserverRestHandler(backend, flow.IdentityList{{Address: address}}) -// // assert.NoError(t, err) -// // for i, test := range tests { -// // req, _ := http.NewRequest("GET", test.url, nil) -// // rr, err := executeRequest(req, restHandler) -// // assert.NoError(t, err) -// // -// // assert.Equal(t, http.StatusBadRequest, rr.Code) -// // assert.JSONEq(t, test.out, rr.Body.String(), fmt.Sprintf("test #%d failed: %v", i, test)) -// // } -// // -// // server.Stop() -// // <-done -// //}) -//} +// TestObserverGetAccount tests the get account from observer node +func TestObserverGetAccount(t *testing.T) { + backend := &mock.API{} + restForwarder := &restmock.RestServerApi{} + + restHandler, err := newObserverRestHandler(backend, restForwarder) + assert.NoError(t, err) + + t.Run("get by address at latest sealed block", func(t *testing.T) { + account := accountFixture(t) + + req := getAccountRequest(t, account, sealedHeightQueryParam, expandableFieldKeys, expandableFieldContracts) + + accountKeys := make([]models.AccountPublicKey, 1) + + sigAlgo := models.SigningAlgorithm("ECDSA_P256") + hashAlgo := models.HashingAlgorithm("SHA3_256") + + accountKeys[0] = models.AccountPublicKey{ + Index: "0", + PublicKey: account.Keys[0].PublicKey.String(), + SigningAlgorithm: &sigAlgo, + HashingAlgorithm: &hashAlgo, + SequenceNumber: "0", + Weight: "1000", + Revoked: false, + } + + restForwarder.Mock.On("GetAccount", + request.GetAccount{ + Address: account.Address, + Height: request.SealedHeight, + }, + mocktestify.Anything, + mocktestify.Anything, + mocktestify.Anything). + Return(models.Account{ + Address: account.Address.String(), + Balance: fmt.Sprintf("%d", account.Balance), + Keys: accountKeys, + Contracts: map[string]string{ + "contract1": "Y29udHJhY3Qx", + "contract2": "Y29udHJhY3Qy", + }, + Expandable: &models.AccountExpandable{}, + Links: &models.Links{ + Self: fmt.Sprintf("/v1/accounts/%s", account.Address), + }, + }, nil) + + expected := expectedExpandedResponse(account) + + assertOKResponse(t, req, expected, restHandler) + mocktestify.AssertExpectationsForObjects(t, backend) + }) + + t.Run("get by address at latest finalized block", func(t *testing.T) { + account := accountFixture(t) + + req := getAccountRequest(t, account, finalHeightQueryParam, expandableFieldKeys, expandableFieldContracts) + + accountKeys := make([]models.AccountPublicKey, 1) + + sigAlgo := models.SigningAlgorithm("ECDSA_P256") + hashAlgo := models.HashingAlgorithm("SHA3_256") + + accountKeys[0] = models.AccountPublicKey{ + Index: "0", + PublicKey: account.Keys[0].PublicKey.String(), + SigningAlgorithm: &sigAlgo, + HashingAlgorithm: &hashAlgo, + SequenceNumber: "0", + Weight: "1000", + Revoked: false, + } + + restForwarder.Mock.On("GetAccount", + request.GetAccount{ + Address: account.Address, + Height: request.FinalHeight, + }, + mocktestify.Anything, + mocktestify.Anything, + mocktestify.Anything). + Return(models.Account{ + Address: account.Address.String(), + Balance: fmt.Sprintf("%d", account.Balance), + Keys: accountKeys, + Contracts: map[string]string{ + "contract1": "Y29udHJhY3Qx", + "contract2": "Y29udHJhY3Qy", + }, + Expandable: &models.AccountExpandable{}, + Links: &models.Links{ + Self: fmt.Sprintf("/v1/accounts/%s", account.Address), + }, + }, nil) + + expected := expectedExpandedResponse(account) + + assertOKResponse(t, req, expected, restHandler) + mocktestify.AssertExpectationsForObjects(t, backend) + }) + + t.Run("get by address at height", func(t *testing.T) { + var height uint64 = 1337 + account := accountFixture(t) + + req := getAccountRequest(t, account, fmt.Sprintf("%d", height), expandableFieldKeys, expandableFieldContracts) + accountKeys := make([]models.AccountPublicKey, 1) + + sigAlgo := models.SigningAlgorithm("ECDSA_P256") + hashAlgo := models.HashingAlgorithm("SHA3_256") + + accountKeys[0] = models.AccountPublicKey{ + Index: "0", + PublicKey: account.Keys[0].PublicKey.String(), + SigningAlgorithm: &sigAlgo, + HashingAlgorithm: &hashAlgo, + SequenceNumber: "0", + Weight: "1000", + Revoked: false, + } + + restForwarder.Mock.On("GetAccount", + request.GetAccount{ + Address: account.Address, + Height: height, + }, + mocktestify.Anything, + mocktestify.Anything, + mocktestify.Anything). + Return(models.Account{ + Address: account.Address.String(), + Balance: fmt.Sprintf("%d", account.Balance), + Keys: accountKeys, + Contracts: map[string]string{ + "contract1": "Y29udHJhY3Qx", + "contract2": "Y29udHJhY3Qy", + }, + Expandable: &models.AccountExpandable{}, + Links: &models.Links{ + Self: fmt.Sprintf("/v1/accounts/%s", account.Address), + }, + }, nil) + + expected := expectedExpandedResponse(account) + + assertOKResponse(t, req, expected, restHandler) + mocktestify.AssertExpectationsForObjects(t, backend) + }) + + t.Run("get by address at height condensed", func(t *testing.T) { + var height uint64 = 1337 + account := accountFixture(t) + + req := getAccountRequest(t, account, fmt.Sprintf("%d", height)) + + restForwarder.Mock.On("GetAccount", + request.GetAccount{ + Address: account.Address, + Height: height, + }, + mocktestify.Anything, + mocktestify.Anything, + mocktestify.Anything). + Return(models.Account{ + Address: account.Address.String(), + Balance: fmt.Sprintf("%d", account.Balance), + Contracts: map[string]string{}, + Expandable: &models.AccountExpandable{ + Keys: expandableFieldKeys, + Contracts: expandableFieldContracts, + }, + Links: &models.Links{ + Self: fmt.Sprintf("/v1/accounts/%s", account.Address), + }, + }, nil) + + expected := expectedCondensedResponse(account) + + assertOKResponse(t, req, expected, restHandler) + mocktestify.AssertExpectationsForObjects(t, backend) + }) + + t.Run("get invalid", func(t *testing.T) { + tests := []struct { + url string + out string + }{ + {accountURL(t, "123", ""), `{"code":400, "message":"invalid address"}`}, + {accountURL(t, unittest.AddressFixture().String(), "foo"), `{"code":400, "message":"invalid height format"}`}, + } + + for i, test := range tests { + req, _ := http.NewRequest("GET", test.url, nil) + rr, err := executeRequest(req, restHandler) + assert.NoError(t, err) + + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.JSONEq(t, test.out, rr.Body.String(), fmt.Sprintf("test #%d failed: %v", i, test)) + } + }) +} func expectedExpandedResponse(account *flow.Account) string { return fmt.Sprintf(`{ @@ -366,6 +374,7 @@ func getAccountRequest(t *testing.T, account *flow.Account, height string, expan q.Add(middleware.ExpandQueryParam, fieldParam) req.URL.RawQuery = q.Encode() } + require.NoError(t, err) return req } diff --git a/engine/access/rest/blocks_test.go b/engine/access/rest/blocks_test.go index 536b2b50296..2ea9a1f3f6b 100644 --- a/engine/access/rest/blocks_test.go +++ b/engine/access/rest/blocks_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" + restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/util" @@ -31,7 +32,6 @@ type testVector struct { expectedResponse string } -// ? func prepareTestVectors(t *testing.T, blockIDs []string, heights []string, @@ -141,8 +141,8 @@ func prepareTestVectors(t *testing.T, return testVectors } -// TestGetBlocks tests the get blocks by ID and get blocks by heights API -func TestGetBlocks(t *testing.T) { +// TestGetBlocks tests the get blocks by ID and get blocks by heights API from access node +func TestAccessGetBlocks(t *testing.T) { backend := &mock.API{} blkCnt := 10 @@ -159,44 +159,29 @@ func TestGetBlocks(t *testing.T) { } } -// ? -//func TestGetBlockFromObserver(t *testing.T) { -// backend := &mock.API{} -// -// // Bring up upstream server -// blkCnt := 10 -// blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) -// address := unittest.IPPort("11633") -// -// done := make(chan int) -// server, _, err := newGrpcServer(new(engineaccessmock.AccessAPIServer), "tcp", address, done) -// assert.NoError(t, err) -// -// testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) -// -// var b bytes.Buffer -// logger := zerolog.New(&b) -// -// restForwarder, err := NewRestForwarder(logger, -// flow.IdentityList{{Address: address}}, -// time.Second, -// grpcutils.DefaultMaxMsgSize) -// assert.NoError(t, err) -// -// restHandler, err := newObserverRestHandler_v2(backend, ) -// assert.NoError(t, err) -// -// for _, tv := range testVectors { -// responseRec, err := executeRequest(tv.request, restHandler) -// assert.NoError(t, err) -// require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) -// actualResp := responseRec.Body.String() -// require.JSONEq(t, tv.expectedResponse, actualResp, "Failed: %s: incorrect response body", tv.description) -// -// } -// server.Stop() -// <-done -//} +// TestObserverGetBlocks tests the get blocks by ID and get blocks by heights API from observer node +func TestObserverGetBlocks(t *testing.T) { + backend := &mock.API{} + restForwarder := &restmock.RestServerApi{} + + // Bring up upstream server + blkCnt := 10 + blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) + + testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) + + restHandler, err := newObserverRestHandler(backend, restForwarder) + assert.NoError(t, err) + + for _, tv := range testVectors { + responseRec, err := executeRequest(tv.request, restHandler) + assert.NoError(t, err) + require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) + actualResp := responseRec.Body.String() + require.JSONEq(t, tv.expectedResponse, actualResp, "Failed: %s: incorrect response body", tv.description) + + } +} func requestURL(t *testing.T, ids []string, start string, end string, expandResponse bool, heights ...string) *http.Request { u, _ := url.Parse("/v1/blocks") diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index 74e5663172f..025e9eb0d01 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -6,15 +6,16 @@ import ( "fmt" "net/http" + "github.com/rs/zerolog" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/util" fvmErrors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" - - "github.com/rs/zerolog" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) const MaxRequestSize = 2 << 20 // 2MB diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go index ed29d90b6aa..d37edf7f731 100644 --- a/engine/access/rest/rest_server_api.go +++ b/engine/access/rest/rest_server_api.go @@ -132,6 +132,7 @@ func (r *RestRouter) GetNodeVersionInfo(req *request.Request) (models.NodeVersio return res, err } +// RestServerApi is the server API for REST service. type RestServerApi interface { // GetTransactionByID gets a transaction by requested ID. GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) @@ -163,20 +164,13 @@ type RestServerApi interface { GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) } +// RequestHandler is a structure that represents local requests type RequestHandler struct { RestServerApi log zerolog.Logger backend access.API } -//// NewRequestHandler returns new RequestHandler. -//func NewRequestHandler(log zerolog.Logger, backend access.API) RestServerApi { -// return &RequestHandler{ -// log: log, -// backend: backend, -// } -//} - // NewRequestHandler returns new RequestHandler. func NewRequestHandler(log zerolog.Logger, backend access.API) *RequestHandler { return &RequestHandler{ @@ -500,19 +494,20 @@ func (h *RequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVers return response, nil } +// RestForwarder handles the request forwarding to upstream type RestForwarder struct { log zerolog.Logger - forwarder.Forwarder + *forwarder.Forwarder } // NewRestForwarder returns new RestForwarder. func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*RestForwarder, error) { - forwarder, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + f, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) restForwarder := &RestForwarder{ - log: log, + log: log, + Forwarder: f, } - restForwarder.Forwarder = forwarder return restForwarder, err } diff --git a/engine/access/rest/scripts_test.go b/engine/access/rest/scripts_test.go index d69886e1d20..8cb19415ae3 100644 --- a/engine/access/rest/scripts_test.go +++ b/engine/access/rest/scripts_test.go @@ -45,10 +45,11 @@ func TestScripts(t *testing.T) { "script": util.ToBase64(validCode), "arguments": []string{util.ToBase64(validArgs)}, } - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("get by Latest height", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + backend.Mock. On("ExecuteScriptAtLatestBlock", mocks.Anything, validCode, [][]byte{validArgs}). Return([]byte("hello world"), nil) @@ -63,6 +64,9 @@ func TestScripts(t *testing.T) { t.Run("get by height", func(t *testing.T) { height := uint64(1337) + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, height, validCode, [][]byte{validArgs}). Return([]byte("hello world"), nil) @@ -77,6 +81,9 @@ func TestScripts(t *testing.T) { t.Run("get by ID", func(t *testing.T) { id, _ := flow.HexStringToIdentifier("222dc5dd51b9e4910f687e475f892f495f3352362ba318b53e318b4d78131312") + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + backend.Mock. On("ExecuteScriptAtBlockID", mocks.Anything, id, validCode, [][]byte{validArgs}). Return([]byte("hello world"), nil) @@ -89,6 +96,9 @@ func TestScripts(t *testing.T) { }) t.Run("get error", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, uint64(1337), validCode, [][]byte{validArgs}). Return(nil, status.Error(codes.Internal, "internal server error")) @@ -104,6 +114,9 @@ func TestScripts(t *testing.T) { }) t.Run("get invalid", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, mocks.Anything, mocks.Anything, mocks.Anything). Return(nil, nil) diff --git a/engine/access/rest/test_helpers.go b/engine/access/rest/test_helpers.go index fc62c8b6feb..a9723728a42 100644 --- a/engine/access/rest/test_helpers.go +++ b/engine/access/rest/test_helpers.go @@ -3,26 +3,18 @@ package rest import ( "bytes" "fmt" - "net" "net/http" "net/http/httptest" "testing" - "time" - - "google.golang.org/grpc" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/access/mock" - engineaccessmock "github.com/onflow/flow-go/engine/access/mock" restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/utils/grpcutils" - - "github.com/onflow/flow/protobuf/go/flow/access" ) const ( @@ -57,10 +49,11 @@ func newAccessRestHandler(backend *mock.API) RestServerApi { return NewRequestHandler(logger, backend) } -func newObserverRestHandler_v2(backend *mock.API, restForwarder *restmock.RestServerApi) (RestServerApi, error) { +func newObserverRestHandler(backend *mock.API, restForwarder *restmock.RestServerApi) (RestServerApi, error) { var b bytes.Buffer logger := zerolog.New(&b) - observerCollector := metrics.NewObserverCollector() // + observerCollector := metrics.NewObserverCollector() //TODO: + //metrics := metrics.NewNoopCollector() return &RestRouter{ Logger: logger, @@ -70,41 +63,6 @@ func newObserverRestHandler_v2(backend *mock.API, restForwarder *restmock.RestSe }, nil } -func newObserverRestHandler(backend *mock.API, identities flow.IdentityList) (RestServerApi, error) { - var b bytes.Buffer - logger := zerolog.New(&b) - observerCollector := metrics.NewObserverCollector() - - restForwarder, err := NewRestForwarder(logger, - identities, - time.Second, - grpcutils.DefaultMaxMsgSize) - if err != nil { - return nil, err - } - - return &RestRouter{ - Logger: logger, - Metrics: observerCollector, - Upstream: restForwarder, - Observer: NewRequestHandler(logger, backend), - }, nil -} - -func newGrpcServer(mockServer *engineaccessmock.AccessAPIServer, network string, address string, done chan int) (*grpc.Server, *net.Listener, error) { - l, err := net.Listen(network, address) - if err != nil { - return nil, nil, err - } - s := grpc.NewServer() - go func(done chan int) { - access.RegisterAccessAPIServer(s, mockServer) - _ = s.Serve(l) - done <- 1 - }(done) - return s, &l, nil -} - func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, restHandler RestServerApi) { assertResponse(t, req, http.StatusOK, expectedRespBody, restHandler) } diff --git a/engine/access/rest/transactions_test.go b/engine/access/rest/transactions_test.go index c35980d4347..36a0cfe9885 100644 --- a/engine/access/rest/transactions_test.go +++ b/engine/access/rest/transactions_test.go @@ -102,10 +102,9 @@ func validCreateBody(tx flow.TransactionBody) map[string]interface{} { } func TestGetTransactions(t *testing.T) { - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) - t.Run("get by ID without results", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -152,6 +151,8 @@ func TestGetTransactions(t *testing.T) { t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -220,12 +221,18 @@ func TestGetTransactions(t *testing.T) { }) t.Run("get by ID Invalid", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + req := getTransactionReq("invalid", false, "", "") expected := `{"code":400, "message":"invalid ID format"}` assertResponse(t, req, http.StatusBadRequest, expected, restHandler) }) t.Run("get by ID non-existing", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -275,10 +282,10 @@ func TestGetTransactionResult(t *testing.T) { } }`, bid.String(), cid.String(), id.String(), util.ToBase64(txr.Events[0].Payload), id.String()) - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) - t.Run("get by transaction ID", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + req := getTransactionResultReq(id.String(), "", "") backend.Mock. @@ -290,6 +297,8 @@ func TestGetTransactionResult(t *testing.T) { t.Run("get by block ID", func(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + req := getTransactionResultReq(id.String(), bid.String(), "") backend.Mock. @@ -301,6 +310,8 @@ func TestGetTransactionResult(t *testing.T) { t.Run("get by collection ID", func(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + req := getTransactionResultReq(id.String(), "", cid.String()) backend.Mock. @@ -312,6 +323,8 @@ func TestGetTransactionResult(t *testing.T) { t.Run("get execution statuses", func(t *testing.T) { backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + testVectors := map[*access.TransactionResult]string{{ Status: flow.TransactionStatusExpired, ErrorMessage: "", @@ -359,6 +372,9 @@ func TestGetTransactionResult(t *testing.T) { }) t.Run("get by ID Invalid", func(t *testing.T) { + backend := &mock.API{} + restHandler := newAccessRestHandler(backend) + req := getTransactionResultReq("invalid", "", "") expected := `{"code":400, "message":"invalid ID format"}` diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 4cb5c5d7c94..fafb33f7d6b 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" @@ -118,10 +119,9 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } - backend, err := NewBackend( + backend, err := backend.NewBackend( suite.log, suite.state, - config, suite.collClient, nil, suite.blocks, @@ -134,7 +134,16 @@ func (suite *RestAPITestSuite) SetupTest() { suite.metrics, 0, 0, - false) + false, + 0, + 0, + 0, + 0, + 0, + nil, + nil, + nil) + require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder( diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index bde0ceffda6..5ebdf139f47 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -189,6 +189,90 @@ func New( return b } +func NewBackend( + log zerolog.Logger, + state protocol.State, + collectionRPC accessproto.AccessAPIClient, + historicalAccessNodes []accessproto.AccessAPIClient, + blocks storage.Blocks, + headers storage.Headers, + collections storage.Collections, + transactions storage.Transactions, + executionReceipts storage.ExecutionReceipts, + executionResults storage.ExecutionResults, + chainID flow.ChainID, + accessMetrics module.AccessMetrics, + collectionGRPCPort uint, + executionGRPCPort uint, + retryEnabled bool, + maxMsgSize uint, + executionClientTimeout time.Duration, + collectionClientTimeout time.Duration, + connectionPoolSize uint, + maxHeightRange uint, + preferredExecutionNodeIDs []string, + fixedExecutionNodeIDs, + archiveAddressList []string, +) (*Backend, error) { + + var cache *lru.Cache + cacheSize := connectionPoolSize + if cacheSize > 0 { + // TODO: remove this fallback after fixing issues with evictions + // It was observed that evictions cause connection errors for in flight requests. This works around + // the issue by forcing hte pool size to be greater than the number of ENs + LNs + if cacheSize < DefaultConnectionPoolSize { + log.Warn().Msg("connection pool size below threshold, setting pool size to default value ") + cacheSize = DefaultConnectionPoolSize + } + var err error + cache, err = lru.NewWithEvict(int(cacheSize), func(_, evictedValue interface{}) { + store := evictedValue.(*CachedClient) + store.Close() + log.Debug().Str("grpc_conn_evicted", store.Address).Msg("closing grpc connection evicted from pool") + if accessMetrics != nil { + accessMetrics.ConnectionFromPoolEvicted() + } + }) + if err != nil { + return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) + } + } + + connectionFactory := &ConnectionFactoryImpl{ + CollectionGRPCPort: collectionGRPCPort, + ExecutionGRPCPort: executionGRPCPort, + CollectionNodeGRPCTimeout: collectionClientTimeout, + ExecutionNodeGRPCTimeout: executionClientTimeout, + ConnectionsCache: cache, + CacheSize: cacheSize, + MaxMsgSize: maxMsgSize, + AccessMetrics: accessMetrics, + Log: log, + } + + return New(state, + collectionRPC, + historicalAccessNodes, + blocks, + headers, + collections, + transactions, + executionReceipts, + executionResults, + chainID, + accessMetrics, + connectionFactory, + retryEnabled, + maxHeightRange, + preferredExecutionNodeIDs, + fixedExecutionNodeIDs, + log, + DefaultSnapshotHistoryLimit, + archiveAddressList, + ), nil +} + func identifierList(ids []string) (flow.IdentifierList, error) { idList := make(flow.IdentifierList, len(ids)) for i, idStr := range ids { diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index cdae487a525..fbd925cee68 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -68,7 +68,7 @@ func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPISer return builder } -// WithRestHandler specifies that the given `RestServerApi` should be used for serving REST queries. +// WithRestHandler specifies that the given `RestServerApi` should be used for REST. func (builder *RPCEngineBuilder) WithRestHandler(handler rest.RestServerApi) *RPCEngineBuilder { builder.restHandler = handler return builder diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 4cbe5a6e495..a9dadb34839 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -1,248 +1,256 @@ package rpc -// import ( -// "context" -// "fmt" -// "io" -// "os" -// "testing" -// "time" - -// "github.com/rs/zerolog" -// "github.com/stretchr/testify/assert" -// "github.com/stretchr/testify/mock" -// "github.com/stretchr/testify/require" -// "github.com/stretchr/testify/suite" -// "google.golang.org/grpc" -// "google.golang.org/grpc/codes" -// "google.golang.org/grpc/credentials/insecure" -// "google.golang.org/grpc/status" - -// accessengine "github.com/onflow/flow-go/engine/access" -// accessmock "github.com/onflow/flow-go/engine/access/mock" -// "github.com/onflow/flow-go/model/flow" -// "github.com/onflow/flow-go/module/irrecoverable" -// "github.com/onflow/flow-go/module/metrics" -// module "github.com/onflow/flow-go/module/mock" -// "github.com/onflow/flow-go/network" -// protocol "github.com/onflow/flow-go/state/protocol/mock" -// storagemock "github.com/onflow/flow-go/storage/mock" -// "github.com/onflow/flow-go/utils/grpcutils" -// "github.com/onflow/flow-go/utils/unittest" -// accessproto "github.com/onflow/flow/protobuf/go/flow/access" -// ) - -// type RateLimitTestSuite struct { -// suite.Suite -// state *protocol.State -// snapshot *protocol.Snapshot -// epochQuery *protocol.EpochQuery -// log zerolog.Logger -// net *network.Network -// request *module.Requester -// collClient *accessmock.AccessAPIClient -// execClient *accessmock.ExecutionAPIClient -// me *module.Local -// chainID flow.ChainID -// metrics *metrics.NoopCollector -// rpcEng *Engine -// client accessproto.AccessAPIClient -// closer io.Closer - -// // storage -// blocks *storagemock.Blocks -// headers *storagemock.Headers -// collections *storagemock.Collections -// transactions *storagemock.Transactions -// receipts *storagemock.ExecutionReceipts - -// // test rate limit -// rateLimit int -// burstLimit int - -// ctx irrecoverable.SignalerContext -// cancel context.CancelFunc -// } - -// func (suite *RateLimitTestSuite) SetupTest() { -// suite.log = zerolog.New(os.Stdout) -// suite.net = new(network.Network) -// suite.state = new(protocol.State) -// suite.snapshot = new(protocol.Snapshot) - -// suite.epochQuery = new(protocol.EpochQuery) -// suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() -// suite.state.On("Final").Return(suite.snapshot, nil).Maybe() -// suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() -// suite.blocks = new(storagemock.Blocks) -// suite.headers = new(storagemock.Headers) -// suite.transactions = new(storagemock.Transactions) -// suite.collections = new(storagemock.Collections) -// suite.receipts = new(storagemock.ExecutionReceipts) - -// suite.collClient = new(accessmock.AccessAPIClient) -// suite.execClient = new(accessmock.ExecutionAPIClient) - -// suite.request = new(module.Requester) -// suite.request.On("EntityByID", mock.Anything, mock.Anything) - -// suite.me = new(module.Local) - -// accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) -// suite.me. -// On("NodeID"). -// Return(accessIdentity.NodeID) - -// suite.chainID = flow.Testnet -// suite.metrics = metrics.NewNoopCollector() - -// config := Config{ -// UnsecureGRPCListenAddr: unittest.DefaultAddress, -// SecureGRPCListenAddr: unittest.DefaultAddress, -// HTTPListenAddr: unittest.DefaultAddress, -// } - -// // set the rate limit to test with -// suite.rateLimit = 2 -// // set the burst limit to test with -// suite.burstLimit = 2 - -// apiRateLimt := map[string]int{ -// "Ping": suite.rateLimit, -// } - -// apiBurstLimt := map[string]int{ -// "Ping": suite.rateLimit, -// } - -// block := unittest.BlockHeaderFixture() -// suite.snapshot.On("Head").Return(block, nil) - -// backend, err := accessengine.NewBackend( -// suite.log, -// suite.state, -// config, -// suite.collClient, -// nil, -// suite.blocks, -// suite.headers, -// suite.collections, -// suite.transactions, -// nil, -// nil, -// suite.chainID, -// suite.metrics, -// 0, -// 0, -// false) -// require.NoError(suite.T(), err) - -// rpcEngBuilder, err := NewBuilder( -// suite.log, -// suite.state, -// config, -// suite.chainID, -// suite.metrics, -// false, -// apiRateLimt, -// apiBurstLimt, -// suite.me, -// backend) -// require.NoError(suite.T(), err) -// suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() -// require.NoError(suite.T(), err) -// suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) -// suite.rpcEng.Start(suite.ctx) -// // wait for the server to startup -// unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") - -// // create the access api client -// suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) -// require.NoError(suite.T(), err) -// } - -// func (suite *RateLimitTestSuite) TearDownTest() { -// if suite.cancel != nil { -// suite.cancel() -// } -// // close the client -// if suite.closer != nil { -// suite.closer.Close() -// } -// // close the server -// unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) -// } - -// func TestRateLimit(t *testing.T) { -// suite.Run(t, new(RateLimitTestSuite)) -// } - -// // TestRatelimitingWithoutBurst tests that rate limit is correctly applied to an Access API call -// func (suite *RateLimitTestSuite) TestRatelimitingWithoutBurst() { - -// req := &accessproto.PingRequest{} -// ctx := context.Background() - -// // expect 2 upstream calls -// suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) -// suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) - -// requestCnt := 0 -// // requests within the burst should succeed -// for requestCnt < suite.rateLimit { -// resp, err := suite.client.Ping(ctx, req) -// assert.NoError(suite.T(), err) -// assert.NotNil(suite.T(), resp) -// // sleep to prevent burst -// time.Sleep(100 * time.Millisecond) -// requestCnt++ -// } - -// // request more than the limit should fail -// _, err := suite.client.Ping(ctx, req) -// suite.assertRateLimitError(err) -// } - -// // TestRatelimitingWithBurst tests that burst limit is correctly applied to an Access API call -// func (suite *RateLimitTestSuite) TestRatelimitingWithBurst() { - -// req := &accessproto.PingRequest{} -// ctx := context.Background() - -// // expect rpc.defaultBurst number of upstream calls -// suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) -// suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) - -// requestCnt := 0 -// // generate a permissible burst of request and assert that they succeed -// for requestCnt < suite.burstLimit { -// resp, err := suite.client.Ping(ctx, req) -// assert.NoError(suite.T(), err) -// assert.NotNil(suite.T(), resp) -// requestCnt++ -// } - -// // request more than the permissible burst and assert that it fails -// _, err := suite.client.Ping(ctx, req) -// suite.assertRateLimitError(err) -// } - -// func (suite *RateLimitTestSuite) assertRateLimitError(err error) { -// assert.Error(suite.T(), err) -// status, ok := status.FromError(err) -// assert.True(suite.T(), ok) -// assert.Equal(suite.T(), codes.ResourceExhausted, status.Code()) -// } - -// func accessAPIClient(address string) (accessproto.AccessAPIClient, io.Closer, error) { -// conn, err := grpc.Dial( -// address, -// grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), -// grpc.WithTransportCredentials(insecure.NewCredentials())) -// if err != nil { -// return nil, nil, fmt.Errorf("failed to connect to address %s: %w", address, err) -// } -// client := accessproto.NewAccessAPIClient(conn) -// closer := io.Closer(conn) -// return client, closer, nil -// } +import ( + "context" + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc/backend" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow-go/utils/unittest" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" +) + +type RateLimitTestSuite struct { + suite.Suite + state *protocol.State + snapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + log zerolog.Logger + net *network.Network + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + chainID flow.ChainID + metrics *metrics.NoopCollector + rpcEng *Engine + client accessproto.AccessAPIClient + closer io.Closer + + // storage + blocks *storagemock.Blocks + headers *storagemock.Headers + collections *storagemock.Collections + transactions *storagemock.Transactions + receipts *storagemock.ExecutionReceipts + + // test rate limit + rateLimit int + burstLimit int + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc +} + +func (suite *RateLimitTestSuite) SetupTest() { + suite.log = zerolog.New(os.Stdout) + suite.net = new(network.Network) + suite.state = new(protocol.State) + suite.snapshot = new(protocol.Snapshot) + + suite.epochQuery = new(protocol.EpochQuery) + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.blocks = new(storagemock.Blocks) + suite.headers = new(storagemock.Headers) + suite.transactions = new(storagemock.Transactions) + suite.collections = new(storagemock.Collections) + suite.receipts = new(storagemock.ExecutionReceipts) + + suite.collClient = new(accessmock.AccessAPIClient) + suite.execClient = new(accessmock.ExecutionAPIClient) + + suite.request = new(module.Requester) + suite.request.On("EntityByID", mock.Anything, mock.Anything) + + suite.me = new(module.Local) + + accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) + suite.me. + On("NodeID"). + Return(accessIdentity.NodeID) + + suite.chainID = flow.Testnet + suite.metrics = metrics.NewNoopCollector() + + config := Config{ + UnsecureGRPCListenAddr: unittest.DefaultAddress, + SecureGRPCListenAddr: unittest.DefaultAddress, + HTTPListenAddr: unittest.DefaultAddress, + } + + // set the rate limit to test with + suite.rateLimit = 2 + // set the burst limit to test with + suite.burstLimit = 2 + + apiRateLimt := map[string]int{ + "Ping": suite.rateLimit, + } + + apiBurstLimt := map[string]int{ + "Ping": suite.rateLimit, + } + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil) + + backend, err := backend.NewBackend( + suite.log, + suite.state, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + nil, + suite.chainID, + suite.metrics, + 0, + 0, + false, + 0, + 0, + 0, + 0, + 0, + nil, + nil, + nil) + require.NoError(suite.T(), err) + + rpcEngBuilder, err := NewBuilder( + suite.log, + suite.state, + config, + suite.chainID, + suite.metrics, + false, + apiRateLimt, + apiBurstLimt, + suite.me, + backend) + require.NoError(suite.T(), err) + suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() + require.NoError(suite.T(), err) + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.rpcEng.Start(suite.ctx) + // wait for the server to startup + unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") + + // create the access api client + suite.client, suite.closer, err = accessAPIClient(suite.rpcEng.UnsecureGRPCAddress().String()) + require.NoError(suite.T(), err) +} + +func (suite *RateLimitTestSuite) TearDownTest() { + if suite.cancel != nil { + suite.cancel() + } + // close the client + if suite.closer != nil { + suite.closer.Close() + } + // close the server + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Done(), 2*time.Second) +} + +func TestRateLimit(t *testing.T) { + suite.Run(t, new(RateLimitTestSuite)) +} + +// TestRatelimitingWithoutBurst tests that rate limit is correctly applied to an Access API call +func (suite *RateLimitTestSuite) TestRatelimitingWithoutBurst() { + + req := &accessproto.PingRequest{} + ctx := context.Background() + + // expect 2 upstream calls + suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) + suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.rateLimit) + + requestCnt := 0 + // requests within the burst should succeed + for requestCnt < suite.rateLimit { + resp, err := suite.client.Ping(ctx, req) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), resp) + // sleep to prevent burst + time.Sleep(100 * time.Millisecond) + requestCnt++ + } + + // request more than the limit should fail + _, err := suite.client.Ping(ctx, req) + suite.assertRateLimitError(err) +} + +// TestRatelimitingWithBurst tests that burst limit is correctly applied to an Access API call +func (suite *RateLimitTestSuite) TestRatelimitingWithBurst() { + + req := &accessproto.PingRequest{} + ctx := context.Background() + + // expect rpc.defaultBurst number of upstream calls + suite.execClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) + suite.collClient.On("Ping", mock.Anything, mock.Anything).Return(nil, nil).Times(suite.burstLimit) + + requestCnt := 0 + // generate a permissible burst of request and assert that they succeed + for requestCnt < suite.burstLimit { + resp, err := suite.client.Ping(ctx, req) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), resp) + requestCnt++ + } + + // request more than the permissible burst and assert that it fails + _, err := suite.client.Ping(ctx, req) + suite.assertRateLimitError(err) +} + +func (suite *RateLimitTestSuite) assertRateLimitError(err error) { + assert.Error(suite.T(), err) + status, ok := status.FromError(err) + assert.True(suite.T(), ok) + assert.Equal(suite.T(), codes.ResourceExhausted, status.Code()) +} + +func accessAPIClient(address string) (accessproto.AccessAPIClient, io.Closer, error) { + conn, err := grpc.Dial( + address, + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, nil, fmt.Errorf("failed to connect to address %s: %w", address, err) + } + client := accessproto.NewAccessAPIClient(conn) + closer := io.Closer(conn) + return client, closer, nil +} diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 3c96ca2740c..73b6d32349a 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/flow-go/crypto" accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" @@ -109,10 +110,9 @@ func (suite *SecureGRPCTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - backend, err := NewBackend( + backend, err := backend.NewBackend( suite.log, suite.state, - config, suite.collClient, nil, suite.blocks, @@ -125,7 +125,15 @@ func (suite *SecureGRPCTestSuite) SetupTest() { suite.metrics, 0, 0, - false) + false, + 0, + 0, + 0, + 0, + 0, + nil, + nil, + nil) require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder( diff --git a/module/forwarder/forwarder.go b/module/forwarder/forwarder.go index 6ee4ae4ffd0..b5cf6244d44 100644 --- a/module/forwarder/forwarder.go +++ b/module/forwarder/forwarder.go @@ -30,8 +30,8 @@ type Forwarder struct { maxMsgSize uint } -func NewForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (Forwarder, error) { - forwarder := Forwarder{maxMsgSize: maxMsgSize} +func NewForwarder(identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*Forwarder, error) { + forwarder := &Forwarder{maxMsgSize: maxMsgSize} err := forwarder.setFlowAccessAPI(identities, timeout) return forwarder, err } From 0fef81459cbf570671a69c35274857fc65a24163 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 19 Jun 2023 12:56:59 -0600 Subject: [PATCH 044/169] add dummy unsafeRNG instance for script execution and separate it from transaction execution --- fvm/environment/facade_env.go | 21 ++++++++-------- fvm/environment/unsafe_random_generator.go | 28 +++++++++++++++------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 51e94fb8f05..9df083b90af 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -10,7 +10,6 @@ import ( "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/storage/state" "github.com/onflow/flow-go/fvm/tracing" - "github.com/onflow/flow-go/state/protocol" ) var _ Environment = &facadeEnvironment{} @@ -66,11 +65,6 @@ func newFacadeEnvironment( logger, runtime) - var protocolSnapshot protocol.Snapshot - if params.State != nil && params.BlockHeader != nil { - protocolSnapshot = params.State.AtBlockID(params.BlockHeader.ID()) - } - env := &facadeEnvironment{ Runtime: runtime, @@ -80,11 +74,6 @@ func newFacadeEnvironment( ProgramLogger: logger, EventEmitter: NoEventEmitter{}, - UnsafeRandomGenerator: NewUnsafeRandomGenerator( - tracer, - protocolSnapshot, - params.TxId, - ), CryptoLibrary: NewCryptoLibrary(tracer, meter), BlockInfo: NewBlockInfo( @@ -173,6 +162,8 @@ func NewScriptEnv( txnState, NewCancellableMeter(ctx, txnState)) + env.UnsafeRandomGenerator = NewDummyRandomGenerator() + env.addParseRestrictedChecks() return env @@ -229,6 +220,14 @@ func NewTransactionEnvironment( txnState, env) + if params.State != nil && params.BlockHeader != nil { + env.UnsafeRandomGenerator = NewUnsafeRandomGenerator( + tracer, + params.State.AtBlockID(params.BlockHeader.ID()), + params.TxId, + ) + } + env.addParseRestrictedChecks() return env diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/unsafe_random_generator.go index 15f89f7d45b..702c63e65d4 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/unsafe_random_generator.go @@ -20,6 +20,10 @@ type UnsafeRandomGenerator interface { UnsafeRandom() (uint64, error) } +var _ UnsafeRandomGenerator = (*unsafeRandomGenerator)(nil) + +// unsafeRandomGenerator implements UnsafeRandomGenerator and is used +// for the transactions execution environment type unsafeRandomGenerator struct { tracer tracing.TracerSpan @@ -74,10 +78,6 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( random.Rand, error, ) { - if gen.stateSnapshot == nil { - return nil, nil - } - // Use the protocol state source of randomness [SoR] for the current block // execution source, err := gen.stateSnapshot.RandomSource() @@ -136,11 +136,23 @@ func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { return 0, err } - if gen.prg == nil { - return 0, errors.NewOperationNotSupportedError("UnsafeRandom") - } - buf := make([]byte, 8) gen.prg.Read(buf) // Note: prg.Read does not return error return binary.LittleEndian.Uint64(buf), nil } + +var _ UnsafeRandomGenerator = (*dummyRandomGenerator)(nil) + +// dummyRandomGenerator implements UnsafeRandomGenerator and is used +// for the scripts execution environment +type dummyRandomGenerator struct{} + +func NewDummyRandomGenerator() UnsafeRandomGenerator { + return &dummyRandomGenerator{} +} + +// UnsafeRandom() returns an error because executing scripts +// does not support randomness APIs. +func (gen *dummyRandomGenerator) UnsafeRandom() (uint64, error) { + return 0, errors.NewOperationNotSupportedError("UnsafeRandom") +} From 2b13c420af30d1483f736846861b653af9337efa Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 19 Jun 2023 17:13:49 -0600 Subject: [PATCH 045/169] use protocol state as BlockComputer attribute and protocol snapshot as fvm Context attribute --- .../computation/computer/computer.go | 12 +++++++-- .../computation/computer/computer_test.go | 25 +++++++++++++------ .../execution_verification_test.go | 3 ++- engine/execution/computation/manager.go | 5 ++-- .../computation/manager_benchmark_test.go | 3 ++- engine/execution/computation/manager_test.go | 4 ++- engine/execution/computation/programs_test.go | 6 +++-- engine/execution/testutil/fixtures.go | 13 ++++++++++ engine/verification/utils/unittest/fixture.go | 1 + fvm/context.go | 6 ++--- fvm/environment/env.go | 2 +- fvm/environment/facade_env.go | 12 ++++----- fvm/fvm_bench_test.go | 1 + fvm/fvm_blockcontext_test.go | 4 +-- 14 files changed, 65 insertions(+), 32 deletions(-) diff --git a/engine/execution/computation/computer/computer.go b/engine/execution/computation/computer/computer.go index a75e7ebee91..50e2efda0bb 100644 --- a/engine/execution/computation/computer/computer.go +++ b/engine/execution/computation/computer/computer.go @@ -24,6 +24,7 @@ import ( "github.com/onflow/flow-go/module/executiondatasync/provider" "github.com/onflow/flow-go/module/mempool/entity" "github.com/onflow/flow-go/module/trace" + "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/utils/logging" ) @@ -113,6 +114,7 @@ type blockComputer struct { spockHasher hash.Hasher receiptHasher hash.Hasher colResCons []result.ExecutedCollectionConsumer + protocolState protocol.State } func SystemChunkContext(vmCtx fvm.Context, logger zerolog.Logger) fvm.Context { @@ -140,6 +142,7 @@ func NewBlockComputer( signer module.Local, executionDataProvider *provider.Provider, colResCons []result.ExecutedCollectionConsumer, + state protocol.State, ) (BlockComputer, error) { systemChunkCtx := SystemChunkContext(vmCtx, logger) vmCtx = fvm.NewContextFromParent( @@ -159,6 +162,7 @@ func NewBlockComputer( spockHasher: utils.NewSPOCKHasher(), receiptHasher: utils.NewExecutionReceiptHasher(), colResCons: colResCons, + protocolState: state, }, nil } @@ -198,7 +202,9 @@ func (e *blockComputer) queueTransactionRequests( collectionCtx := fvm.NewContextFromParent( e.vmCtx, - fvm.WithBlockHeader(blockHeader)) + fvm.WithBlockHeader(blockHeader), + fvm.WithProtocolSnapshot(e.protocolState.AtBlockID(blockId)), + ) for idx, collection := range rawCollections { collectionLogger := collectionCtx.Logger.With(). @@ -231,7 +237,9 @@ func (e *blockComputer) queueTransactionRequests( systemCtx := fvm.NewContextFromParent( e.systemChunkCtx, - fvm.WithBlockHeader(blockHeader)) + fvm.WithBlockHeader(blockHeader), + fvm.WithProtocolSnapshot(e.protocolState.AtBlockID(blockId)), + ) systemCollectionLogger := systemCtx.Logger.With(). Str("block_id", blockIdStr). Uint64("height", blockHeader.Height). diff --git a/engine/execution/computation/computer/computer_test.go b/engine/execution/computation/computer/computer_test.go index cca9fca1a7b..5d3c5560f39 100644 --- a/engine/execution/computation/computer/computer_test.go +++ b/engine/execution/computation/computer/computer_test.go @@ -166,7 +166,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) // create a block with 1 collection with 2 transactions @@ -299,7 +300,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) // create an empty block @@ -395,7 +397,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { comm, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) // create an empty block @@ -453,7 +456,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) collectionCount := 2 @@ -669,6 +673,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, + testutil.ProtocolStateFixture(), ) require.NoError(t, err) @@ -778,7 +783,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) const collectionCount = 2 @@ -889,7 +895,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) key := flow.AccountStatusRegisterID( @@ -932,7 +939,8 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) collectionCount := 5 @@ -1249,7 +1257,8 @@ func Test_ExecutingSystemCollection(t *testing.T) { committer, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) // create empty block, it will have system collection attached while executing diff --git a/engine/execution/computation/execution_verification_test.go b/engine/execution/computation/execution_verification_test.go index fd4e4c8c0a0..a1bcba328c9 100644 --- a/engine/execution/computation/execution_verification_test.go +++ b/engine/execution/computation/execution_verification_test.go @@ -774,7 +774,8 @@ func executeBlockAndVerifyWithParameters(t *testing.T, ledgerCommiter, me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) executableBlock := unittest.ExecutableBlockFromTransactions(chain.ChainID(), txs) diff --git a/engine/execution/computation/manager.go b/engine/execution/computation/manager.go index a4331ca31bd..bd06b2ac13f 100644 --- a/engine/execution/computation/manager.go +++ b/engine/execution/computation/manager.go @@ -116,9 +116,7 @@ func New( // Attachments are enabled everywhere except for Mainnet AttachmentsEnabled: chainID != flow.Mainnet, }, - ), - ), - fvm.WithProtocolState(protoState), + )), } if params.ExtensiveTracing { options = append(options, fvm.WithExtensiveTracing()) @@ -136,6 +134,7 @@ func New( me, executionDataProvider, nil, // TODO(ramtin): update me with proper consumers + protoState, ) if err != nil { diff --git a/engine/execution/computation/manager_benchmark_test.go b/engine/execution/computation/manager_benchmark_test.go index c8e9c150a56..dd04b60533f 100644 --- a/engine/execution/computation/manager_benchmark_test.go +++ b/engine/execution/computation/manager_benchmark_test.go @@ -159,7 +159,8 @@ func BenchmarkComputeBlock(b *testing.B) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(b, err) derivedChainData, err := derived.NewDerivedChainData( diff --git a/engine/execution/computation/manager_test.go b/engine/execution/computation/manager_test.go index 0f9440d462f..69ae1acd861 100644 --- a/engine/execution/computation/manager_test.go +++ b/engine/execution/computation/manager_test.go @@ -142,7 +142,8 @@ func TestComputeBlockWithStorage(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -827,6 +828,7 @@ func Test_EventEncodingFailsOnlyTxAndCarriesOn(t *testing.T) { me, prov, nil, + testutil.ProtocolStateFixture(), ) require.NoError(t, err) diff --git a/engine/execution/computation/programs_test.go b/engine/execution/computation/programs_test.go index 2f3a273e176..2857ea40346 100644 --- a/engine/execution/computation/programs_test.go +++ b/engine/execution/computation/programs_test.go @@ -133,7 +133,8 @@ func TestPrograms_TestContractUpdates(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) @@ -243,7 +244,8 @@ func TestPrograms_TestBlockForks(t *testing.T) { committer.NewNoopViewCommitter(), me, prov, - nil) + nil, + testutil.ProtocolStateFixture()) require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 57c125786f2..141c2f30624 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/cadence" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto" @@ -23,6 +24,8 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/epochs" "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/state/protocol" + pmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -625,3 +628,13 @@ func ComputationResultFixture(t *testing.T) *execution.ComputationResult { }, } } + +// ProtocolStateFixture returns a protocol state that can be used +// by BlockComputer tests +func ProtocolStateFixture() protocol.State { + snapshot := pmock.Snapshot{} + state := pmock.State{} + state.On("AtBlockID", mock.Anything).Return(&snapshot) + snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) + return &state +} diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index dc572cc0622..d128619ada7 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -288,6 +288,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB committer, me, prov, + nil, nil) require.NoError(t, err) diff --git a/fvm/context.go b/fvm/context.go index 1ce1eb297eb..349f25af8d8 100644 --- a/fvm/context.go +++ b/fvm/context.go @@ -161,13 +161,13 @@ func WithEventCollectionSizeLimit(limit uint64) Option { } } -// WithProtocolState sets the protocol state for a virtual machine context. +// WithProtocolSnapshot sets the protocol state for a virtual machine context. // // The VM uses the protocol state to provide protocol information to the Cadence runtime, // including the source of the pseudorandom number generator. -func WithProtocolState(protocolState protocol.State) Option { +func WithProtocolSnapshot(snapshot protocol.Snapshot) Option { return func(ctx Context) Context { - ctx.State = protocolState + ctx.Snapshot = snapshot return ctx } } diff --git a/fvm/environment/env.go b/fvm/environment/env.go index 97e23b060c2..21808c40e65 100644 --- a/fvm/environment/env.go +++ b/fvm/environment/env.go @@ -99,7 +99,7 @@ type EnvironmentParams struct { BlockInfoParams TransactionInfoParams - protocol.State + protocol.Snapshot ContractUpdaterParams } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 9df083b90af..e8e38fa1205 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -220,13 +220,11 @@ func NewTransactionEnvironment( txnState, env) - if params.State != nil && params.BlockHeader != nil { - env.UnsafeRandomGenerator = NewUnsafeRandomGenerator( - tracer, - params.State.AtBlockID(params.BlockHeader.ID()), - params.TxId, - ) - } + env.UnsafeRandomGenerator = NewUnsafeRandomGenerator( + tracer, + params.Snapshot, + params.TxId, + ) env.addParseRestrictedChecks() diff --git a/fvm/fvm_bench_test.go b/fvm/fvm_bench_test.go index 05069a3b4e8..3694c6422a7 100644 --- a/fvm/fvm_bench_test.go +++ b/fvm/fvm_bench_test.go @@ -223,6 +223,7 @@ func NewBasicBlockExecutor(tb testing.TB, chain flow.Chain, logger zerolog.Logge ledgerCommitter, me, prov, + nil, nil) require.NoError(tb, err) diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index 7d3f9f5ca73..ba0818f67bb 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -1672,14 +1672,12 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { header := &flow.Header{Height: 42} snapshot := pmock.Snapshot{} - state := pmock.State{} - state.On("AtBlockID", mock.Anything).Return(&snapshot) snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) ctx := fvm.NewContext( fvm.WithChain(chain), fvm.WithBlockHeader(header), - fvm.WithProtocolState(&state), + fvm.WithProtocolSnapshot(&snapshot), fvm.WithCadenceLogging(true), ) From 8a52a2ed28ae6f5cb8b4d4346071b466b60aa018 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 20 Jun 2023 12:18:56 -0600 Subject: [PATCH 046/169] update verification engine to use protocol snapshot --- engine/execution/testutil/fixtures.go | 2 +- engine/verification/fetcher/engine.go | 5 ++++- engine/verification/utils/unittest/fixture.go | 2 +- engine/verification/utils/unittest/helper.go | 8 ++++---- model/verification/verifiableChunkData.go | 2 ++ module/chunks/chunkVerifier.go | 8 ++++++-- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 141c2f30624..5e91a2ec0b9 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -633,8 +633,8 @@ func ComputationResultFixture(t *testing.T) *execution.ComputationResult { // by BlockComputer tests func ProtocolStateFixture() protocol.State { snapshot := pmock.Snapshot{} + snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) state := pmock.State{} state.On("AtBlockID", mock.Anything).Return(&snapshot) - snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) return &state } diff --git a/engine/verification/fetcher/engine.go b/engine/verification/fetcher/engine.go index 23d02c02474..9373001918d 100644 --- a/engine/verification/fetcher/engine.go +++ b/engine/verification/fetcher/engine.go @@ -526,8 +526,9 @@ func (e *Engine) pushToVerifier(chunk *flow.Chunk, if err != nil { return fmt.Errorf("could not get block: %w", err) } + snapshot := e.state.AtBlockID(header.ID()) - vchunk, err := e.makeVerifiableChunkData(chunk, header, result, chunkDataPack) + vchunk, err := e.makeVerifiableChunkData(chunk, header, snapshot, result, chunkDataPack) if err != nil { return fmt.Errorf("could not verify chunk: %w", err) } @@ -545,6 +546,7 @@ func (e *Engine) pushToVerifier(chunk *flow.Chunk, // chunk data to verify it. func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk, header *flow.Header, + snapshot protocol.Snapshot, result *flow.ExecutionResult, chunkDataPack *flow.ChunkDataPack, ) (*verification.VerifiableChunkData, error) { @@ -566,6 +568,7 @@ func (e *Engine) makeVerifiableChunkData(chunk *flow.Chunk, IsSystemChunk: isSystemChunk, Chunk: chunk, Header: header, + Snapshot: snapshot, Result: result, ChunkDataPack: chunkDataPack, EndState: endState, diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index d128619ada7..c3a474e9e76 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -289,7 +289,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB me, prov, nil, - nil) + testutil.ProtocolStateFixture()) require.NoError(t, err) completeColls := make(map[flow.Identifier]*entity.CompleteCollection) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 7c6e6eec323..e2bad5768d8 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -591,10 +591,10 @@ func withConsumers(t *testing.T, } // verifies memory resources are cleaned up all over pipeline - assert.True(t, verNode.BlockConsumer.Size() == 0) - assert.True(t, verNode.ChunkConsumer.Size() == 0) - assert.True(t, verNode.ChunkStatuses.Size() == 0) - assert.True(t, verNode.ChunkRequests.Size() == 0) + assert.Equal(t, verNode.BlockConsumer.Size(), uint(0)) + assert.Equal(t, verNode.ChunkConsumer.Size(), uint(0)) + assert.Equal(t, verNode.ChunkStatuses.Size(), uint(0)) + assert.Equal(t, verNode.ChunkRequests.Size(), uint(0)) } // bootstrapSystem is a test helper that bootstraps a flow system with node of each main roles (except execution nodes that are two). diff --git a/model/verification/verifiableChunkData.go b/model/verification/verifiableChunkData.go index 298beece37f..293240a55c3 100644 --- a/model/verification/verifiableChunkData.go +++ b/model/verification/verifiableChunkData.go @@ -2,6 +2,7 @@ package verification import ( "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" ) // VerifiableChunkData represents a ready-to-verify chunk @@ -10,6 +11,7 @@ type VerifiableChunkData struct { IsSystemChunk bool // indicates whether this is a system chunk Chunk *flow.Chunk // the chunk to be verified Header *flow.Header // BlockHeader that contains this chunk + Snapshot protocol.Snapshot // protocol state snapshot Result *flow.ExecutionResult // execution result of this block ChunkDataPack *flow.ChunkDataPack // chunk data package needed to verify this chunk EndState flow.StateCommitment // state commitment at the end of this chunk diff --git a/module/chunks/chunkVerifier.go b/module/chunks/chunkVerifier.go index 11b3a2d6c2b..1d3561ac092 100644 --- a/module/chunks/chunkVerifier.go +++ b/module/chunks/chunkVerifier.go @@ -57,7 +57,9 @@ func (fcv *ChunkVerifier) Verify( if vc.IsSystemChunk { ctx = fvm.NewContextFromParent( fcv.systemChunkCtx, - fvm.WithBlockHeader(vc.Header)) + fvm.WithBlockHeader(vc.Header), + fvm.WithProtocolSnapshot(vc.Snapshot), + ) txBody, err := blueprints.SystemChunkTransaction(fcv.vmCtx.Chain) if err != nil { @@ -70,7 +72,9 @@ func (fcv *ChunkVerifier) Verify( } else { ctx = fvm.NewContextFromParent( fcv.vmCtx, - fvm.WithBlockHeader(vc.Header)) + fvm.WithBlockHeader(vc.Header), + fvm.WithProtocolSnapshot(vc.Snapshot), + ) transactions = make( []*fvm.TransactionProcedure, From 23b4cd00f62f3361d80c689be387705208ac7549 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Wed, 21 Jun 2023 15:15:37 +0300 Subject: [PATCH 047/169] Added observer rest integration test, linted --- engine/access/rest/blocks_test.go | 1 - engine/access/rest/handler.go | 5 + engine/access/rest/rest_server_api.go | 6 +- engine/access/rest/test_helpers.go | 6 +- insecure/go.sum | 2 - integration/testnet/network.go | 3 + integration/tests/access/observer_test.go | 208 ++++++++++++++++++++-- module/metrics/noop.go | 6 + module/metrics/observer.go | 5 + 9 files changed, 218 insertions(+), 24 deletions(-) diff --git a/engine/access/rest/blocks_test.go b/engine/access/rest/blocks_test.go index 2ea9a1f3f6b..7e337e07e2a 100644 --- a/engine/access/rest/blocks_test.go +++ b/engine/access/rest/blocks_test.go @@ -179,7 +179,6 @@ func TestObserverGetBlocks(t *testing.T) { require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) actualResp := responseRec.Body.String() require.JSONEq(t, tv.expectedResponse, actualResp, "Failed: %s: incorrect response body", tv.description) - } } diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index 025e9eb0d01..03d8695fbb7 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -123,6 +123,11 @@ func (h *Handler) errorHandler(w http.ResponseWriter, err error, errorLogger zer h.errorResponse(w, http.StatusBadRequest, msg, errorLogger) return } + if se.Code() == codes.Unavailable { + msg := fmt.Sprintf("Invalid Upstream request: %s", se.Message()) + h.errorResponse(w, http.StatusServiceUnavailable, msg, errorLogger) + return + } } // stop going further - catch all error diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go index d37edf7f731..27d85960026 100644 --- a/engine/access/rest/rest_server_api.go +++ b/engine/access/rest/rest_server_api.go @@ -25,7 +25,7 @@ import ( // It splits requests between a local and a remote rest service. type RestRouter struct { Logger zerolog.Logger - Metrics *metrics.ObserverCollector + Metrics metrics.ObserverMetrics Upstream RestServerApi Observer *RequestHandler } @@ -427,7 +427,7 @@ func (h *RequestHandler) GetAccount(r request.GetAccount, context context.Contex account, err := h.backend.GetAccountAtBlockHeight(context, r.Address, r.Height) if err != nil { - return response, err + return response, NewNotFoundError("not found account at block height", err) } err = response.Build(account, link, expandFields) @@ -906,7 +906,7 @@ func (f *RestForwarder) GetAccount(r request.GetAccount, context context.Context accountResponse, err := upstream.GetAccountAtBlockHeight(context, getAccountAtBlockHeightRequest) if err != nil { - return response, err + return response, NewNotFoundError("not found account at block height", err) } flowAccount, err := convert.MessageToAccount(accountResponse.Account) diff --git a/engine/access/rest/test_helpers.go b/engine/access/rest/test_helpers.go index a9723728a42..8ce8c2f50d2 100644 --- a/engine/access/rest/test_helpers.go +++ b/engine/access/rest/test_helpers.go @@ -30,9 +30,8 @@ const ( func executeRequest(req *http.Request, restHandler RestServerApi) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - metrics := metrics.NewNoopCollector() - router, err := newRouter(restHandler, logger, flow.Testnet.Chain(), metrics) + router, err := newRouter(restHandler, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) if err != nil { return nil, err } @@ -52,8 +51,7 @@ func newAccessRestHandler(backend *mock.API) RestServerApi { func newObserverRestHandler(backend *mock.API, restForwarder *restmock.RestServerApi) (RestServerApi, error) { var b bytes.Buffer logger := zerolog.New(&b) - observerCollector := metrics.NewObserverCollector() //TODO: - //metrics := metrics.NewNoopCollector() + observerCollector := metrics.NewNoopCollector() return &RestRouter{ Logger: logger, diff --git a/insecure/go.sum b/insecure/go.sum index a2e659e29d8..129d83cb596 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -92,7 +92,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -1177,7 +1176,6 @@ github.com/onflow/atree v0.5.0 h1:y3lh8hY2fUo8KVE2ALVcz0EiNTq0tXJ6YTXKYVDA+3E= github.com/onflow/atree v0.5.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.38.1 h1:8YpnE1ixAGB8hF3t+slkHGhjfIBJ95dqUS+sEHrM2kY= github.com/onflow/cadence v0.38.1/go.mod h1:SpfjNhPsJxGIHbOthE9JD/e8JFaFY73joYLPsov+PY4= -github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= diff --git a/integration/testnet/network.go b/integration/testnet/network.go index 1520725b335..e16f6bcf79e 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -714,6 +714,9 @@ func (net *FlowNetwork) addObserver(t *testing.T, conf ObserverConfig) { nodeContainer.exposePort(AdminPort, testingdock.RandomPort(t)) nodeContainer.AddFlag("admin-addr", nodeContainer.ContainerAddr(AdminPort)) + nodeContainer.exposePort(RESTPort, testingdock.RandomPort(t)) + nodeContainer.AddFlag("rest-addr", nodeContainer.ContainerAddr(RESTPort)) + nodeContainer.opts.HealthCheck = testingdock.HealthCheckCustom(nodeContainer.HealthcheckCallback()) suiteContainer := net.suite.Container(containerOpts) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 29b96da49e6..e36642eb662 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -2,21 +2,25 @@ package access import ( "context" + "fmt" + "net/http" + "strings" "testing" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func TestObserver(t *testing.T) { @@ -25,9 +29,10 @@ func TestObserver(t *testing.T) { type ObserverSuite struct { suite.Suite - net *testnet.FlowNetwork - teardown func() - local map[string]struct{} + net *testnet.FlowNetwork + teardown func() + localRpc map[string]struct{} + localRest map[string]struct{} cancel context.CancelFunc } @@ -44,7 +49,7 @@ func (s *ObserverSuite) TearDownTest() { } func (s *ObserverSuite) SetupTest() { - s.local = map[string]struct{}{ + s.localRpc = map[string]struct{}{ "Ping": {}, "GetLatestBlockHeader": {}, "GetBlockHeaderByID": {}, @@ -56,6 +61,14 @@ func (s *ObserverSuite) SetupTest() { "GetNetworkParameters": {}, } + s.localRest = map[string]struct{}{ + "getBlocksByIDs": {}, + "getBlocksByHeight": {}, + "getBlockPayloadByID": {}, + "getNetworkParameters": {}, + "getNodeVersionInfo": {}, + } + nodeConfigs := []testnet.NodeConfig{ // access node with unstaked nodes supported testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.InfoLevel), testnet.WithAdditionalFlag("--supports-observer=true")), @@ -90,11 +103,11 @@ func (s *ObserverSuite) SetupTest() { s.net.Start(ctx) } -// TestObserver runs the following tests: +// TestObserverRPC runs the following tests: // 1. CompareRPCs: verifies that the observer client returns the same errors as the access client for rpcs proxied to the upstream AN // 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rpcs handled by the upstream // 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries -func (s *ObserverSuite) TestObserver() { +func (s *ObserverSuite) TestObserverRPC() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -111,7 +124,7 @@ func (s *ObserverSuite) TestObserver() { // verify that both clients return the same errors for proxied rpcs for _, rpc := range s.getRPCs() { // skip rpcs handled locally by observer - if _, local := s.local[rpc.name]; local { + if _, local := s.localRpc[rpc.name]; local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -129,7 +142,7 @@ func (s *ObserverSuite) TestObserver() { t.Run("HandledByUpstream", func(t *testing.T) { // verify that we receive Unavailable errors from all rpcs handled upstream for _, rpc := range s.getRPCs() { - if _, local := s.local[rpc.name]; local { + if _, local := s.localRpc[rpc.name]; local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -142,7 +155,7 @@ func (s *ObserverSuite) TestObserver() { t.Run("HandledByObserver", func(t *testing.T) { // verify that we receive NotFound or no error from all rpcs handled locally for _, rpc := range s.getRPCs() { - if _, local := s.local[rpc.name]; !local { + if _, local := s.localRpc[rpc.name]; !local { continue } t.Run(rpc.name, func(t *testing.T) { @@ -154,7 +167,86 @@ func (s *ObserverSuite) TestObserver() { }) } }) +} +// TestObserverRest runs the following tests: +// 1. CompareRPCs: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN +// 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rests handled by the upstream +// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries +func (s *ObserverSuite) TestObserverRest() { + t := s.T() + + accessAddr := s.net.ContainerByName(testnet.PrimaryAN).Addr(testnet.RESTPort) + observerAddr := s.net.ContainerByName("observer_1").Addr(testnet.RESTPort) + + httpClient := http.DefaultClient + makeHttpCall := func(method string, url string) (*http.Response, error) { + switch method { + case http.MethodGet: + return httpClient.Get(url) + case http.MethodPost: + return httpClient.Post(url, "application/json", strings.NewReader("{}")) + } + panic("not supported") + } + makeObserverCall := func(method string, path string) (*http.Response, error) { + return makeHttpCall(method, "http://"+observerAddr+"/v1"+path) + } + makeAccessCall := func(method string, path string) (*http.Response, error) { + return makeHttpCall(method, "http://"+accessAddr+"/v1"+path) + } + + t.Run("CompareEndpoints", func(t *testing.T) { + // verify that both clients return the same errors for proxied rests + for _, endpoint := range s.getRestEndpoints() { + // skip rest handled locally by observer + if _, local := s.localRest[endpoint.name]; local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + accessResp, accessErr := makeAccessCall(endpoint.method, endpoint.path) + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + assert.NoError(t, accessErr) + assert.NoError(t, observerErr) + assert.Equal(t, accessResp.Status, observerResp.Status) + }) + } + }) + + // stop the upstream access container + err := s.net.StopContainerByName(context.Background(), testnet.PrimaryAN) + require.NoError(t, err) + + t.Run("HandledByUpstream", func(t *testing.T) { + // verify that we receive StatusInternalServerError, StatusServiceUnavailable, StatusBadRequest errors from all rests handled upstream + for _, endpoint := range s.getRestEndpoints() { + if _, local := s.localRest[endpoint.name]; local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + require.NoError(t, observerErr) + assert.Contains(t, [...]int{ + http.StatusInternalServerError, + http.StatusServiceUnavailable, + http.StatusBadRequest}, observerResp.StatusCode) + }) + } + }) + + t.Run("HandledByObserver", func(t *testing.T) { + // verify that we receive NotFound or no error from all rests handled locally + for _, endpoint := range s.getRestEndpoints() { + if _, local := s.localRest[endpoint.name]; !local { + continue + } + t.Run(endpoint.name, func(t *testing.T) { + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + require.NoError(t, observerErr) + assert.Contains(t, [...]int{http.StatusNotFound, http.StatusOK}, observerResp.StatusCode) + }) + } + }) } func (s *ObserverSuite) getAccessClient() (accessproto.AccessAPIClient, error) { @@ -287,3 +379,91 @@ func (s *ObserverSuite) getRPCs() []RPCTest { }}, } } + +type RestEndpointTest struct { + name string + method string + path string +} + +func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { + transactionId := unittest.IdentifierFixture().String() + account, _ := unittest.AccountFixture() + block := unittest.BlockFixture() + executionResult := unittest.ExecutionResultFixture() + collection := unittest.CollectionFixture(2) + blockEvents := unittest.BlockEventsFixture(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(2))), 2) + + return []RestEndpointTest{ + { + name: "getTransactionByID", + method: http.MethodGet, + path: "/transactions/" + transactionId, + }, + { + name: "createTransaction", + method: http.MethodPost, + path: "/transactions", + }, + { + name: "getTransactionResultByID", + method: http.MethodGet, + path: fmt.Sprintf("/transaction_results/%s?block_id=%s&collection_id=%s", transactionId, block.ID().String(), collection.ID().String()), + }, + { + name: "getBlocksByIDs", + method: http.MethodGet, + path: "/blocks/" + block.ID().String(), + }, + { + name: "getBlocksByHeight", + method: http.MethodGet, + path: "/blocks?height=0", + }, + { + name: "getBlockPayloadByID", + method: http.MethodGet, + path: "/blocks/" + block.ID().String() + "/payload", + }, + { + name: "getExecutionResultByID", + method: http.MethodGet, + path: "/execution_results/" + executionResult.ID().String(), + }, + { + name: "getExecutionResultByBlockID", + method: http.MethodGet, + path: "/execution_results?block_id=" + block.ID().String(), + }, + { + name: "getCollectionByID", + method: http.MethodGet, + path: "/collections/" + collection.ID().String(), + }, + { + name: "executeScript", + method: http.MethodPost, + path: "/scripts", + }, + { + name: "getAccount", + method: http.MethodGet, + path: "/accounts/" + account.Address.HexWithPrefix(), + }, + { + name: "getEvents", + method: http.MethodGet, + path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", blockEvents.Events[0].Type, 1, 3), + }, + { + name: "getNetworkParameters", + method: http.MethodGet, + path: "/network/parameters", + }, + { + name: "getNodeVersionInfo", + method: http.MethodGet, + path: "/node_version_info", + }, + } +} diff --git a/module/metrics/noop.go b/module/metrics/noop.go index eddfe1a1a26..4c6ae18c255 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -4,6 +4,8 @@ import ( "context" "time" + "google.golang.org/grpc/codes" + "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" @@ -297,3 +299,7 @@ func (nc *NoopCollector) AsyncProcessingStarted(string) func (nc *NoopCollector) AsyncProcessingFinished(string, time.Duration) {} func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} + +var _ ObserverMetrics = (*NoopCollector)(nil) + +func (nc *NoopCollector) RecordRPC(handler, rpc string, code codes.Code) {} diff --git a/module/metrics/observer.go b/module/metrics/observer.go index 4e885c9bf4c..95116b9f8f1 100644 --- a/module/metrics/observer.go +++ b/module/metrics/observer.go @@ -6,7 +6,12 @@ import ( "google.golang.org/grpc/codes" ) +type ObserverMetrics interface { + RecordRPC(handler, rpc string, code codes.Code) +} + type ObserverCollector struct { + ObserverMetrics rpcs *prometheus.CounterVec } From ea525fdb28daf407a5331922473747ded0199339 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Wed, 21 Jun 2023 17:53:15 +0300 Subject: [PATCH 048/169] Fixed MaxMsgSize argument --- cmd/access/node_builder/access_node_builder.go | 2 +- cmd/observer/node_builder/observer_builder.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 12e6e03a683..844d39289dd 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1007,7 +1007,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.collectionGRPCPort, builder.executionGRPCPort, builder.retryEnabled, - config.MaxHeightRange, + config.MaxMsgSize, config.ExecutionClientTimeout, config.CollectionClientTimeout, config.ConnectionPoolSize, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index ddef152feda..95637f7f4e5 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -866,7 +866,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { 0, 0, false, - config.MaxHeightRange, + config.MaxMsgSize, config.ExecutionClientTimeout, config.CollectionClientTimeout, config.ConnectionPoolSize, From 8c8cd4875468a54795a860ef49029ae42af2fb4e Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 22 Jun 2023 17:43:45 +0300 Subject: [PATCH 049/169] Created Config structure for backend --- .../node_builder/access_node_builder.go | 55 +++++++++---------- cmd/observer/node_builder/observer_builder.go | 39 ++++++------- .../export_report.json | 6 -- engine/access/rest_api_test.go | 9 +-- engine/access/rpc/backend/backend.go | 36 +++++++----- engine/access/rpc/engine.go | 33 +++++------ engine/access/rpc/rate_limit_test.go | 8 +-- engine/access/secure_grpcr_test.go | 8 +-- 8 files changed, 86 insertions(+), 108 deletions(-) delete mode 100644 cmd/util/cmd/execution-state-extract/export_report.json diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 0a933d2ea0a..38e36baaa2f 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -147,20 +147,22 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { collectionGRPCPort: 9000, executionGRPCPort: 9000, rpcConf: rpc.Config{ - UnsecureGRPCListenAddr: "0.0.0.0:9000", - SecureGRPCListenAddr: "0.0.0.0:9001", - HTTPListenAddr: "0.0.0.0:8000", - RESTListenAddr: "", - CollectionAddr: "", - HistoricalAccessAddrs: "", - CollectionClientTimeout: 3 * time.Second, - ExecutionClientTimeout: 3 * time.Second, - ConnectionPoolSize: backend.DefaultConnectionPoolSize, - MaxHeightRange: backend.DefaultMaxHeightRange, - PreferredExecutionNodeIDs: nil, - FixedExecutionNodeIDs: nil, - ArchiveAddressList: nil, - MaxMsgSize: grpcutils.DefaultMaxMsgSize, + UnsecureGRPCListenAddr: "0.0.0.0:9000", + SecureGRPCListenAddr: "0.0.0.0:9001", + HTTPListenAddr: "0.0.0.0:8000", + RESTListenAddr: "", + CollectionAddr: "", + HistoricalAccessAddrs: "", + BackendConfig: backend.Config{ + CollectionClientTimeout: 3 * time.Second, + ExecutionClientTimeout: 3 * time.Second, + ConnectionPoolSize: backend.DefaultConnectionPoolSize, + MaxHeightRange: backend.DefaultMaxHeightRange, + PreferredExecutionNodeIDs: nil, + FixedExecutionNodeIDs: nil, + ArchiveAddressList: nil, + }, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, stateStreamConf: state_stream.Config{ MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, @@ -662,15 +664,15 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") flags.StringVarP(&builder.rpcConf.CollectionAddr, "static-collection-ingress-addr", "", defaultConfig.rpcConf.CollectionAddr, "the address (of the collection node) to send transactions to") flags.StringVarP(&builder.ExecutionNodeAddress, "script-addr", "s", defaultConfig.ExecutionNodeAddress, "the address (of the execution node) forward the script to") - flags.StringSliceVar(&builder.rpcConf.ArchiveAddressList, "archive-address-list", defaultConfig.rpcConf.ArchiveAddressList, "the list of address of the archive node to forward the script queries to") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.ArchiveAddressList, "archive-address-list", defaultConfig.rpcConf.BackendConfig.ArchiveAddressList, "the list of address of the archive node to forward the script queries to") flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, "historical-access-addr", "", defaultConfig.rpcConf.HistoricalAccessAddrs, "comma separated rpc addresses for historical access nodes") - flags.DurationVar(&builder.rpcConf.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.CollectionClientTimeout, "grpc client timeout for a collection node") - flags.DurationVar(&builder.rpcConf.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.ExecutionClientTimeout, "grpc client timeout for an execution node") - flags.UintVar(&builder.rpcConf.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") + flags.DurationVar(&builder.rpcConf.BackendConfig.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.BackendConfig.CollectionClientTimeout, "grpc client timeout for a collection node") + flags.DurationVar(&builder.rpcConf.BackendConfig.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.BackendConfig.ExecutionClientTimeout, "grpc client timeout for an execution node") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", grpcutils.DefaultMaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") - flags.StringSliceVar(&builder.rpcConf.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") - flags.StringSliceVar(&builder.rpcConf.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.BackendConfig.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") + flags.StringSliceVar(&builder.rpcConf.BackendConfig.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.BackendConfig.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized") flags.BoolVar(&builder.logTxTimeToExecuted, "log-tx-time-to-executed", defaultConfig.logTxTimeToExecuted, "log transaction time to executed") flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed") @@ -910,7 +912,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.rpcConf.CollectionAddr, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), grpc.WithTransportCredentials(insecure.NewCredentials()), - backend.WithClientUnaryInterceptor(builder.rpcConf.CollectionClientTimeout)) + backend.WithClientUnaryInterceptor(builder.rpcConf.BackendConfig.CollectionClientTimeout)) if err != nil { return err } @@ -989,6 +991,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { config := builder.rpcConf + backend, err := backend.NewBackend(node.Logger, node.State, builder.CollectionRPC, @@ -1005,13 +1008,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.executionGRPCPort, builder.retryEnabled, config.MaxMsgSize, - config.ExecutionClientTimeout, - config.CollectionClientTimeout, - config.ConnectionPoolSize, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - config.ArchiveAddressList) + builder.rpcConf.BackendConfig) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index d0754647f29..9e7ef69fb9a 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -113,19 +113,22 @@ type ObserverServiceConfig struct { func DefaultObserverServiceConfig() *ObserverServiceConfig { return &ObserverServiceConfig{ rpcConf: rpc.Config{ - UnsecureGRPCListenAddr: "0.0.0.0:9000", - SecureGRPCListenAddr: "0.0.0.0:9001", - HTTPListenAddr: "0.0.0.0:8000", - RESTListenAddr: "", - CollectionAddr: "", - HistoricalAccessAddrs: "", - CollectionClientTimeout: 3 * time.Second, - ExecutionClientTimeout: 3 * time.Second, - MaxHeightRange: backend.DefaultMaxHeightRange, - PreferredExecutionNodeIDs: nil, - FixedExecutionNodeIDs: nil, - ArchiveAddressList: nil, - MaxMsgSize: grpcutils.DefaultMaxMsgSize, + UnsecureGRPCListenAddr: "0.0.0.0:9000", + SecureGRPCListenAddr: "0.0.0.0:9001", + HTTPListenAddr: "0.0.0.0:8000", + RESTListenAddr: "", + CollectionAddr: "", + HistoricalAccessAddrs: "", + BackendConfig: backend.Config{ + CollectionClientTimeout: 3 * time.Second, + ExecutionClientTimeout: 3 * time.Second, + ConnectionPoolSize: backend.DefaultConnectionPoolSize, + MaxHeightRange: backend.DefaultMaxHeightRange, + PreferredExecutionNodeIDs: nil, + FixedExecutionNodeIDs: nil, + ArchiveAddressList: nil, + }, + MaxMsgSize: grpcutils.DefaultMaxMsgSize, }, rpcMetricsEnabled: false, apiRatelimits: nil, @@ -450,7 +453,7 @@ func (builder *ObserverServiceBuilder) extraFlags() { flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", defaultConfig.rpcConf.MaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") - flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests") + flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") flags.StringVar(&builder.observerNetworkingKeyPath, "observer-networking-key-path", defaultConfig.observerNetworkingKeyPath, "path to the networking key for observer") @@ -867,13 +870,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { 0, false, config.MaxMsgSize, - config.ExecutionClientTimeout, - config.CollectionClientTimeout, - config.ConnectionPoolSize, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - config.ArchiveAddressList) + builder.rpcConf.BackendConfig) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json deleted file mode 100644 index 55627d0684d..00000000000 --- a/cmd/util/cmd/execution-state-extract/export_report.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "EpochCounter": 0, - "PreviousStateCommitment": "bf63e3eff31913c99688ddbac17d5d6bddeb9574b2bf947dca47bce2d164b8dd", - "CurrentStateCommitment": "bf63e3eff31913c99688ddbac17d5d6bddeb9574b2bf947dca47bce2d164b8dd", - "ReportSucceeded": true -} \ No newline at end of file diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index fafb33f7d6b..7b418fd33ee 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -136,13 +136,8 @@ func (suite *RestAPITestSuite) SetupTest() { 0, false, 0, - 0, - 0, - 0, - 0, - nil, - nil, - nil) + config.BackendConfig, + ) require.NoError(suite.T(), err) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 5ebdf139f47..d5a29d04adb 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -9,7 +9,6 @@ import ( "google.golang.org/grpc/status" lru "github.com/hashicorp/golang-lru" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/onflow/flow-go/access" @@ -21,6 +20,8 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) // maxExecutionNodesCnt is the max number of execution nodes that will be contacted to complete an execution api request @@ -78,6 +79,17 @@ type Backend struct { connFactory ConnectionFactory } +// Config defines the configurable options for creating Backend +type Config struct { + ExecutionClientTimeout time.Duration // execution API GRPC client timeout + CollectionClientTimeout time.Duration // collection API GRPC client timeout + ConnectionPoolSize uint // size of the cache for storing collection and execution connections + MaxHeightRange uint // max size of height range requests + PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs + FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node ID can be chosen from the PreferredExecutionNodeIDs + ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node +} + func New( state protocol.State, collectionRPC accessproto.AccessAPIClient, @@ -206,17 +218,11 @@ func NewBackend( executionGRPCPort uint, retryEnabled bool, maxMsgSize uint, - executionClientTimeout time.Duration, - collectionClientTimeout time.Duration, - connectionPoolSize uint, - maxHeightRange uint, - preferredExecutionNodeIDs []string, - fixedExecutionNodeIDs, - archiveAddressList []string, + config Config, ) (*Backend, error) { var cache *lru.Cache - cacheSize := connectionPoolSize + cacheSize := config.ConnectionPoolSize if cacheSize > 0 { // TODO: remove this fallback after fixing issues with evictions // It was observed that evictions cause connection errors for in flight requests. This works around @@ -242,8 +248,8 @@ func NewBackend( connectionFactory := &ConnectionFactoryImpl{ CollectionGRPCPort: collectionGRPCPort, ExecutionGRPCPort: executionGRPCPort, - CollectionNodeGRPCTimeout: collectionClientTimeout, - ExecutionNodeGRPCTimeout: executionClientTimeout, + CollectionNodeGRPCTimeout: config.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, ConnectionsCache: cache, CacheSize: cacheSize, MaxMsgSize: maxMsgSize, @@ -264,12 +270,12 @@ func NewBackend( accessMetrics, connectionFactory, retryEnabled, - maxHeightRange, - preferredExecutionNodeIDs, - fixedExecutionNodeIDs, + config.MaxHeightRange, + config.PreferredExecutionNodeIDs, + config.FixedExecutionNodeIDs, log, DefaultSnapshotHistoryLimit, - archiveAddressList, + config.ArchiveAddressList, ), nil } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 4fde8fe1acf..875a9389e6d 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "sync" - "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -32,21 +31,23 @@ import ( // A secure GRPC server here implies a server that presents a self-signed TLS certificate and a client that authenticates // the server via a pre-shared public key type Config struct { - UnsecureGRPCListenAddr string // the non-secure GRPC server address as ip:port - SecureGRPCListenAddr string // the secure GRPC server address as ip:port - TransportCredentials credentials.TransportCredentials // the secure GRPC credentials - HTTPListenAddr string // the HTTP web proxy address as ip:port - RESTListenAddr string // the REST server address as ip:port (if empty the REST server will not be started) - CollectionAddr string // the address of the upstream collection node - HistoricalAccessAddrs string // the list of all access nodes from previous spork - MaxMsgSize uint // GRPC max message size - ExecutionClientTimeout time.Duration // execution API GRPC client timeout - CollectionClientTimeout time.Duration // collection API GRPC client timeout - ConnectionPoolSize uint // size of the cache for storing collection and execution connections - MaxHeightRange uint // max size of height range requests - PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs - FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node node ID can be chosen from the PreferredExecutionNodeIDs - ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node + UnsecureGRPCListenAddr string // the non-secure GRPC server address as ip:port + SecureGRPCListenAddr string // the secure GRPC server address as ip:port + TransportCredentials credentials.TransportCredentials // the secure GRPC credentials + HTTPListenAddr string // the HTTP web proxy address as ip:port + RESTListenAddr string // the REST server address as ip:port (if empty the REST server will not be started) + CollectionAddr string // the address of the upstream collection node + HistoricalAccessAddrs string // the list of all access nodes from previous spork + + BackendConfig backend.Config // configurable options for creating Backend + MaxMsgSize uint // GRPC max message size + //ExecutionClientTimeout time.Duration // execution API GRPC client timeout + //CollectionClientTimeout time.Duration // collection API GRPC client timeout + //ConnectionPoolSize uint // size of the cache for storing collection and execution connections + //MaxHeightRange uint // max size of height range requests + //PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs + //FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node node ID can be chosen from the PreferredExecutionNodeIDs + //ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node } // Engine exposes the server with a simplified version of the Access API. diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index a9dadb34839..d0b31b19118 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -136,13 +136,7 @@ func (suite *RateLimitTestSuite) SetupTest() { 0, false, 0, - 0, - 0, - 0, - 0, - nil, - nil, - nil) + config.BackendConfig) require.NoError(suite.T(), err) rpcEngBuilder, err := NewBuilder( diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 73b6d32349a..86d018e0548 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -127,13 +127,7 @@ func (suite *SecureGRPCTestSuite) SetupTest() { 0, false, 0, - 0, - 0, - 0, - 0, - nil, - nil, - nil) + config.BackendConfig) require.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder( From dd3464d361495fe6e28394773d4e0ed9a99c8a38 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 22 Jun 2023 11:26:40 -0600 Subject: [PATCH 050/169] commit missing master changes --- engine/execution/computation/programs_test.go | 5 +---- engine/verification/utils/unittest/fixture.go | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/engine/execution/computation/programs_test.go b/engine/execution/computation/programs_test.go index 25b6310ce36..092a4a6e8ea 100644 --- a/engine/execution/computation/programs_test.go +++ b/engine/execution/computation/programs_test.go @@ -138,11 +138,8 @@ func TestPrograms_TestContractUpdates(t *testing.T) { me, prov, nil, -<<<<<<< HEAD - testutil.ProtocolStateFixture()) -======= + testutil.ProtocolStateFixture(), testMaxConcurrency) ->>>>>>> master require.NoError(t, err) derivedChainData, err := derived.NewDerivedChainData(10) diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index 90e86009b36..c6166a6af7f 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -295,11 +295,8 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB me, prov, nil, -<<<<<<< HEAD - testutil.ProtocolStateFixture()) -======= + testutil.ProtocolStateFixture(), testMaxConcurrency) ->>>>>>> master require.NoError(t, err) completeColls := make(map[flow.Identifier]*entity.CompleteCollection) From 4820fa6958728cecd999635e6b68e20b4b72d91d Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Thu, 22 Jun 2023 14:10:03 -0600 Subject: [PATCH 051/169] add check for math/rand usage in production code --- .github/workflows/ci.yml | 4 ++-- Makefile | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc0a7b5ebec..5772ef5dcb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,8 +66,8 @@ jobs: cache: true - name: Run tidy run: make tidy - - name: Emulator no relic check - run: make emulator-norelic-check + - name: code sanity check + run: make code-sanity-check shell-check: name: ShellCheck diff --git a/Makefile b/Makefile index 7f511197260..ae15fe5a59a 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,23 @@ emulator-norelic-check: # test the fvm package compiles with Relic library disabled (required for the emulator build) cd ./fvm && go test ./... -run=NoTestHasThisPrefix +.SILENT: go-math-rand-check +go-math-rand-check: + # check that the insecure math/rand Go package isn't used by production code. + # `exclude` should only specify non production code (test, bench..). + # If this check fails, try updating your code by using: + # - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness + # - "flow-go/crypto/random" for deterministic randomness + grep --include=\*.go --exclude={*test*,*helper*,*example*,*fixture*,*benchmark*,*profiler*} -rnw '"math/rand"'; \ + if [ $$? -ne 1 ]; \ + then \ + echo "[Error] Go production code should not use math/rand package"; \ + exit 1; \ + fi + +.PHONY: code-sanity-check +code-sanity-check: go-math-rand-check emulator-norelic-check + .PHONY: fuzz-fvm fuzz-fvm: # run fuzz tests in the fvm package From 8d5d523dc1e5e6e882020d8113c0906107936e7b Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 22 Jun 2023 20:00:46 -0400 Subject: [PATCH 052/169] integrate ALSP misbehavior reporter with slashing violations consumer --- network/alsp/misbehavior.go | 25 +++++ network/mocknetwork/adapter.go | 6 +- .../misbehavior_report_consumer.go | 35 +++++++ network/mocknetwork/violations_consumer.go | 92 +++++++++++++++--- network/network.go | 6 +- network/slashing/consumer.go | 93 ++++++++++++------- network/slashing/violations_consumer.go | 17 ++-- network/stub/network.go | 2 +- 8 files changed, 219 insertions(+), 57 deletions(-) create mode 100644 network/mocknetwork/misbehavior_report_consumer.go diff --git a/network/alsp/misbehavior.go b/network/alsp/misbehavior.go index 326b113cd8b..af4921cd06a 100644 --- a/network/alsp/misbehavior.go +++ b/network/alsp/misbehavior.go @@ -24,6 +24,25 @@ const ( // the message is not valid according to the engine's validation logic. The decision to consider a message invalid // is up to the engine. InvalidMessage network.Misbehavior = "misbehavior-invalid-message" + + // UnExpectedValidationError is a misbehavior that is reported when a validation error is encountered during message validation before the message + // is processed by an engine. + UnExpectedValidationError network.Misbehavior = "unexpected-validation-error" + + // UnknownMsgType is a misbehavior that is reported when a message of unknown type is received from a peer. + UnknownMsgType network.Misbehavior = "unknown-message-type" + + // SenderEjected is a misbehavior that is reported when a message is received from an ejected peer. + SenderEjected network.Misbehavior = "sender-ejected" + + // UnauthorizedUnicastOnChannel is a misbehavior that is reported when a message not authorized to be sent via unicast is received via unicast. + UnauthorizedUnicastOnChannel network.Misbehavior = "unauthorized-unicast-on-channel" + + // UnAuthorizedSender is a misbehavior that is reported when a message is sent by an unauthorized role. + UnAuthorizedSender network.Misbehavior = "unauthorized-sender" + + // UnauthorizedPublishOnChannel is a misbehavior that is reported when a message not authorized to be sent via pubsub is received via pubsub. + UnauthorizedPublishOnChannel network.Misbehavior = "unauthorized-pubsub-on-channel" ) func AllMisbehaviorTypes() []network.Misbehavior { @@ -33,5 +52,11 @@ func AllMisbehaviorTypes() []network.Misbehavior { RedundantMessage, UnsolicitedMessage, InvalidMessage, + UnExpectedValidationError, + UnknownMsgType, + SenderEjected, + UnauthorizedUnicastOnChannel, + UnauthorizedPublishOnChannel, + UnAuthorizedSender, } } diff --git a/network/mocknetwork/adapter.go b/network/mocknetwork/adapter.go index 364ec1027ce..2700f6eb0cc 100644 --- a/network/mocknetwork/adapter.go +++ b/network/mocknetwork/adapter.go @@ -58,9 +58,9 @@ func (_m *Adapter) PublishOnChannel(_a0 channels.Channel, _a1 interface{}, _a2 . return r0 } -// ReportMisbehaviorOnChannel provides a mock function with given fields: _a0, _a1 -func (_m *Adapter) ReportMisbehaviorOnChannel(_a0 channels.Channel, _a1 network.MisbehaviorReport) { - _m.Called(_a0, _a1) +// ReportMisbehaviorOnChannel provides a mock function with given fields: channel, report +func (_m *Adapter) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + _m.Called(channel, report) } // UnRegisterChannel provides a mock function with given fields: channel diff --git a/network/mocknetwork/misbehavior_report_consumer.go b/network/mocknetwork/misbehavior_report_consumer.go new file mode 100644 index 00000000000..8731a6ae8fe --- /dev/null +++ b/network/mocknetwork/misbehavior_report_consumer.go @@ -0,0 +1,35 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mocknetwork + +import ( + channels "github.com/onflow/flow-go/network/channels" + mock "github.com/stretchr/testify/mock" + + network "github.com/onflow/flow-go/network" +) + +// MisbehaviorReportConsumer is an autogenerated mock type for the MisbehaviorReportConsumer type +type MisbehaviorReportConsumer struct { + mock.Mock +} + +// ReportMisbehaviorOnChannel provides a mock function with given fields: channel, report +func (_m *MisbehaviorReportConsumer) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + _m.Called(channel, report) +} + +type mockConstructorTestingTNewMisbehaviorReportConsumer interface { + mock.TestingT + Cleanup(func()) +} + +// NewMisbehaviorReportConsumer creates a new instance of MisbehaviorReportConsumer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewMisbehaviorReportConsumer(t mockConstructorTestingTNewMisbehaviorReportConsumer) *MisbehaviorReportConsumer { + mock := &MisbehaviorReportConsumer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/network/mocknetwork/violations_consumer.go b/network/mocknetwork/violations_consumer.go index 9c6f252b095..c02f2ed94cc 100644 --- a/network/mocknetwork/violations_consumer.go +++ b/network/mocknetwork/violations_consumer.go @@ -13,33 +13,101 @@ type ViolationsConsumer struct { } // OnInvalidMsgError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnInvalidMsgError(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnInvalidMsgError(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } // OnSenderEjectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnSenderEjectedError(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnSenderEjectedError(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } // OnUnAuthorizedSenderError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// OnUnauthorizedPublishOnChannel provides a mock function with given fields: violation +func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } // OnUnauthorizedUnicastOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } // OnUnexpectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnexpectedError(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnUnexpectedError(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } // OnUnknownMsgTypeError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *slashing.Violation) { - _m.Called(violation) +func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *slashing.Violation) error { + ret := _m.Called(violation) + + var r0 error + if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + r0 = rf(violation) + } else { + r0 = ret.Error(0) + } + + return r0 } type mockConstructorTestingTNewViolationsConsumer interface { diff --git a/network/network.go b/network/network.go index 703c5e627c8..4f77892b666 100644 --- a/network/network.go +++ b/network/network.go @@ -47,6 +47,7 @@ type Network interface { // Adapter is meant to be utilized by the Conduit interface to send messages to the Network layer to be // delivered to the remote targets. type Adapter interface { + MisbehaviorReportConsumer // UnicastOnChannel sends the message in a reliable way to the given recipient. UnicastOnChannel(channels.Channel, interface{}, flow.Identifier) error @@ -60,7 +61,10 @@ type Adapter interface { // UnRegisterChannel unregisters the engine for the specified channel. The engine will no longer be able to send or // receive messages from that channel. UnRegisterChannel(channel channels.Channel) error +} +// MisbehaviorReportConsumer set of funcs used to handle MisbehaviorReport disseminated from misbehavior reporters. +type MisbehaviorReportConsumer interface { // ReportMisbehaviorOnChannel reports the misbehavior of a node on sending a message to the current node that appears // valid based on the networking layer but is considered invalid by the current node based on the Flow protocol. // The misbehavior report is sent to the current node's networking layer on the given channel to be processed. @@ -69,5 +73,5 @@ type Adapter interface { // - report: The misbehavior report to be sent. // Returns: // none - ReportMisbehaviorOnChannel(channels.Channel, MisbehaviorReport) + ReportMisbehaviorOnChannel(channel channels.Channel, report MisbehaviorReport) } diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index aaac28fccc5..3a7943c16ed 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -7,35 +7,34 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/utils/logging" ) const ( - unknown = "unknown" - unExpectedValidationError = "unexpected_validation_error" - unAuthorizedSenderViolation = "unauthorized_sender" - unknownMsgTypeViolation = "unknown_message_type" - invalidMsgViolation = "invalid_message" - senderEjectedViolation = "sender_ejected" - unauthorizedUnicastOnChannel = "unauthorized_unicast_on_channel" + unknown = "unknown" ) // Consumer is a struct that logs a message for any slashable offenses. // This struct will be updated in the future when slashing is implemented. type Consumer struct { - log zerolog.Logger - metrics module.NetworkSecurityMetrics + log zerolog.Logger + metrics module.NetworkSecurityMetrics + misbehaviorReportConsumer network.MisbehaviorReportConsumer } // NewSlashingViolationsConsumer returns a new Consumer. -func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics) *Consumer { +func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) *Consumer { return &Consumer{ - log: log.With().Str("module", "network_slashing_consumer").Logger(), - metrics: metrics, + log: log.With().Str("module", "network_slashing_consumer").Logger(), + metrics: metrics, + misbehaviorReportConsumer: consumer, } } -func (c *Consumer) logOffense(networkOffense string, violation *Violation) { +// logOffense logs the slashing violation with details. +func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *Violation) { // if violation fails before the message is decoded the violation.MsgType will be unknown if len(violation.MsgType) == 0 { violation.MsgType = unknown @@ -51,7 +50,7 @@ func (c *Consumer) logOffense(networkOffense string, violation *Violation) { e := c.log.Error(). Str("peer_id", violation.PeerID). - Str("networking_offense", networkOffense). + Str("misbehavior", misbehavior.String()). Str("message_type", violation.MsgType). Str("channel", violation.Channel.String()). Str("protocol", violation.Protocol.String()). @@ -62,37 +61,65 @@ func (c *Consumer) logOffense(networkOffense string, violation *Violation) { e.Msg(fmt.Sprintf("potential slashable offense: %s", violation.Err)) // capture unauthorized message count metric - c.metrics.OnUnauthorizedMessage(role, violation.MsgType, violation.Channel.String(), networkOffense) + c.metrics.OnUnauthorizedMessage(role, violation.MsgType, violation.Channel.String(), misbehavior.String()) } -// OnUnAuthorizedSenderError logs an error for unauthorized sender error. -func (c *Consumer) OnUnAuthorizedSenderError(violation *Violation) { - c.logOffense(unAuthorizedSenderViolation, violation) +// reportMisbehavior reports the slashing violation to the alsp misbehavior report manager. When violation identity +// is nil this indicates the misbehavior occurred either on a public network and the identity of the sender is unknown +// we can skip reporting the misbehavior. +func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *Violation) error { + if violation.Identity == nil { + c.log.Debug().Str("peerID", violation.PeerID).Msg("violation identity unknown skipping misbehavior reporting") + return nil + } + report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) + if err != nil { + return fmt.Errorf("failed to create misbehavior report: %w", err) + } + c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) + return nil +} + +// OnUnAuthorizedSenderError logs an error for unauthorized sender error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnAuthorizedSenderError(violation *Violation) error { + c.logOffense(alsp.UnAuthorizedSender, violation) + return c.reportMisbehavior(alsp.UnAuthorizedSender, violation) } -// OnUnknownMsgTypeError logs an error for unknown message type error. -func (c *Consumer) OnUnknownMsgTypeError(violation *Violation) { - c.logOffense(unknownMsgTypeViolation, violation) +// OnUnknownMsgTypeError logs an error for unknown message type error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnknownMsgTypeError(violation *Violation) error { + c.logOffense(alsp.UnknownMsgType, violation) + return c.reportMisbehavior(alsp.UnknownMsgType, violation) } // OnInvalidMsgError logs an error for messages that contained payloads that could not -// be unmarshalled into the message type denoted by message code byte. -func (c *Consumer) OnInvalidMsgError(violation *Violation) { - c.logOffense(invalidMsgViolation, violation) +// be unmarshalled into the message type denoted by message code byte and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnInvalidMsgError(violation *Violation) error { + c.logOffense(alsp.InvalidMessage, violation) + return c.reportMisbehavior(alsp.InvalidMessage, violation) +} + +// OnSenderEjectedError logs an error for sender ejected error and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnSenderEjectedError(violation *Violation) error { + c.logOffense(alsp.SenderEjected, violation) + return c.reportMisbehavior(alsp.SenderEjected, violation) } -// OnSenderEjectedError logs an error for sender ejected error. -func (c *Consumer) OnSenderEjectedError(violation *Violation) { - c.logOffense(senderEjectedViolation, violation) +// OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *Violation) error { + c.logOffense(alsp.UnauthorizedUnicastOnChannel, violation) + return c.reportMisbehavior(alsp.UnauthorizedUnicastOnChannel, violation) } -// OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast. -func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *Violation) { - c.logOffense(unauthorizedUnicastOnChannel, violation) +// OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. +func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *Violation) error { + c.logOffense(alsp.UnauthorizedPublishOnChannel, violation) + return c.reportMisbehavior(alsp.UnauthorizedPublishOnChannel, violation) } // OnUnexpectedError logs an error for unexpected errors. This indicates message validation -// has failed for an unknown reason and could potentially be n slashable offense. -func (c *Consumer) OnUnexpectedError(violation *Violation) { - c.logOffense(unExpectedValidationError, violation) +// has failed for an unknown reason and could potentially be n slashable offense and reports a misbehavior to alsp misbehavior report manager. +func (c *Consumer) OnUnexpectedError(violation *Violation) error { + c.logOffense(alsp.UnExpectedValidationError, violation) + return c.reportMisbehavior(alsp.UnExpectedValidationError, violation) } diff --git a/network/slashing/violations_consumer.go b/network/slashing/violations_consumer.go index cf1f8ea7d85..981fa549089 100644 --- a/network/slashing/violations_consumer.go +++ b/network/slashing/violations_consumer.go @@ -8,23 +8,26 @@ import ( type ViolationsConsumer interface { // OnUnAuthorizedSenderError logs an error for unauthorized sender error - OnUnAuthorizedSenderError(violation *Violation) + OnUnAuthorizedSenderError(violation *Violation) error // OnUnknownMsgTypeError logs an error for unknown message type error - OnUnknownMsgTypeError(violation *Violation) + OnUnknownMsgTypeError(violation *Violation) error // OnInvalidMsgError logs an error for messages that contained payloads that could not // be unmarshalled into the message type denoted by message code byte. - OnInvalidMsgError(violation *Violation) + OnInvalidMsgError(violation *Violation) error // OnSenderEjectedError logs an error for sender ejected error - OnSenderEjectedError(violation *Violation) + OnSenderEjectedError(violation *Violation) error - // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast - OnUnauthorizedUnicastOnChannel(violation *Violation) + // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast. + OnUnauthorizedUnicastOnChannel(violation *Violation) error + + // OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. + OnUnauthorizedPublishOnChannel(violation *Violation) error // OnUnexpectedError logs an error for unknown errors - OnUnexpectedError(violation *Violation) + OnUnexpectedError(violation *Violation) error } type Violation struct { diff --git a/network/stub/network.go b/network/stub/network.go index fc93cf9b588..9e91b386922 100644 --- a/network/stub/network.go +++ b/network/stub/network.go @@ -306,6 +306,6 @@ func (n *Network) StopConDev() { close(n.qCD) } -func (n *Network) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { +func (n *Network) ReportMisbehaviorOnChannel(_ channels.Channel, _ network.MisbehaviorReport) { // no-op for stub network. } From 2aa9db84047e2ff89884c15dcbd5680e9165ac34 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 22 Jun 2023 20:01:43 -0400 Subject: [PATCH 053/169] update slashing violations consumer usages, handle error returned from funcs --- network/p2p/middleware/middleware.go | 28 +++++++--- .../validator/authorized_sender_validator.go | 53 +++++++++++++------ 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index ac5f349264a..f260d434ca8 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -525,7 +525,8 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { msgCode, err := codec.MessageCodeFromPayload(msg.Payload) if err != nil { violation.Err = err - m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return } @@ -533,13 +534,15 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { _, what, err := codec.InterfaceFromMessageCode(msgCode) if err != nil { violation.Err = err - m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return } violation.MsgType = what violation.Err = ErrUnicastMsgWithoutSub - m.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) + svcErr := m.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return } @@ -651,9 +654,10 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag // we can remove this check maxSize, err := unicastMaxMsgSizeByCode(msg.Payload) if err != nil { - m.slashingViolationsConsumer.OnUnknownMsgTypeError(&slashing.Violation{ + svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(&slashing.Violation{ Identity: nil, PeerID: remotePeer.String(), MsgType: "", Channel: channel, Protocol: message.ProtocolTypeUnicast, Err: err, }) + m.checkSlashingViolationsConsumerErr(svcErr) return } if msg.Size() > maxSize { @@ -708,14 +712,16 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe violation := &slashing.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte violation := &slashing.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - m.slashingViolationsConsumer.OnInvalidMsgError(violation) + svcErr := m.slashingViolationsConsumer.OnInvalidMsgError(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return case err != nil: // this condition should never happen and indicates there's a bug @@ -725,7 +731,8 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe violation := &slashing.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - m.slashingViolationsConsumer.OnUnexpectedError(violation) + svcErr := m.slashingViolationsConsumer.OnUnexpectedError(violation) + m.checkSlashingViolationsConsumerErr(svcErr) return } @@ -744,7 +751,6 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe // processMessage processes a message and eventually passes it to the overlay func (m *Middleware) processMessage(scope *network.IncomingMessageScope) { - logger := m.log.With(). Str("channel", scope.Channel().String()). Str("type", scope.Protocol().String()). @@ -821,6 +827,12 @@ func (m *Middleware) IsConnected(nodeID flow.Identifier) (bool, error) { return m.libP2PNode.IsConnected(peerID) } +func (m *Middleware) checkSlashingViolationsConsumerErr(err error) { + if err != nil { + m.log.Error().Err(err).Msg("failed to disseminate slashing violation on slashing violations consumer") + } +} + // unicastMaxMsgSize returns the max permissible size for a unicast message func unicastMaxMsgSize(messageType string) int { switch messageType { diff --git a/network/validator/authorized_sender_validator.go b/network/validator/authorized_sender_validator.go index 0af21b45e39..4e83b0762c7 100644 --- a/network/validator/authorized_sender_validator.go +++ b/network/validator/authorized_sender_validator.go @@ -61,15 +61,17 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan // something terrible went wrong. identity, ok := av.getIdentity(from) if !ok { - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} - av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) + violation := &slashing.Violation{PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} + err := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) + av.checkSlashingViolationsConsumerErr(err) return "", ErrIdentityUnverified } msgCode, err := codec.MessageCodeFromPayload(payload) if err != nil { - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + av.checkSlashingViolationsConsumerErr(svcErr) return "", err } @@ -77,29 +79,44 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan switch { case err == nil: return msgType, nil - case message.IsUnknownMsgTypeErr(err): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + case message.IsUnknownMsgTypeErr(err) || codec.IsErrUnknownMsgCode(err): + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) + av.checkSlashingViolationsConsumerErr(svcErr) return msgType, err case errors.Is(err, message.ErrUnauthorizedMessageOnChannel) || errors.Is(err, message.ErrUnauthorizedRole): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) + av.checkSlashingViolationsConsumerErr(svcErr) + return msgType, err case errors.Is(err, ErrSenderEjected): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnSenderEjectedError(violation) + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnSenderEjectedError(violation) + av.checkSlashingViolationsConsumerErr(svcErr) + return msgType, err case errors.Is(err, message.ErrUnauthorizedUnicastOnChannel): - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) + av.checkSlashingViolationsConsumerErr(svcErr) + + return msgType, err + case errors.Is(err, message.ErrUnauthorizedPublishOnChannel): + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnauthorizedPublishOnChannel(violation) + av.checkSlashingViolationsConsumerErr(svcErr) + return msgType, err default: // this condition should never happen and indicates there's a bug // don't crash as a result of external inputs since that creates a DoS vector // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) - violation := &slashing.Violation{Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - av.slashingViolationsConsumer.OnUnexpectedError(violation) + violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + svcErr := av.slashingViolationsConsumer.OnUnexpectedError(violation) + av.checkSlashingViolationsConsumerErr(svcErr) + return msgType, err } } @@ -145,3 +162,9 @@ func (av *AuthorizedSenderValidator) isAuthorizedSender(identity *flow.Identity, return what, nil } + +func (av *AuthorizedSenderValidator) checkSlashingViolationsConsumerErr(err error) { + if err != nil { + av.log.Error().Err(err).Msg("failed to disseminate slashing violation on slashing violations consumer") + } +} From a10498870872e908bb69ee219f60a1ea20e80414 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 22 Jun 2023 20:11:41 -0400 Subject: [PATCH 054/169] update tests --- network/alsp/manager/manager_test.go | 8 +- network/p2p/test/topic_validator_test.go | 57 ++++-- .../authorized_sender_validator_test.go | 169 ++++++++++++------ utils/unittest/unittest.go | 17 +- 4 files changed, 176 insertions(+), 75 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 3fd57430e21..1d9db225af5 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -58,7 +58,7 @@ func TestNetworkPassesReportedMisbehavior(t *testing.T) { 1, unittest.Logger(), unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) + unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(misbehaviorReportManger))) sms := testutils.GenerateSubscriptionManagers(t, mws) networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0]) @@ -121,7 +121,7 @@ func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { 1, unittest.Logger(), unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) + unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) sms := testutils.GenerateSubscriptionManagers(t, mws) networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) net, err := p2p.NewNetwork(networkCfg) @@ -220,7 +220,7 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) 3, unittest.Logger(), unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) + unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) sms := testutils.GenerateSubscriptionManagers(t, mws) networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) victimNetwork, err := p2p.NewNetwork(networkCfg) @@ -298,7 +298,7 @@ func TestMisbehaviorReportMetrics(t *testing.T) { 1, unittest.Logger(), unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector())) + unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) sms := testutils.GenerateSubscriptionManagers(t, mws) networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index b6f0dfe7ba5..e25e8673220 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -15,9 +15,11 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/translator" @@ -26,6 +28,7 @@ import ( "github.com/onflow/flow-go/network/validator" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" "github.com/onflow/flow-go/utils/unittest" + "github.com/stretchr/testify/mock" ) // TestTopicValidator_Unstaked tests that the libP2P node topic validator rejects unauthenticated messages on non-public channels (unstaked) @@ -51,12 +54,12 @@ func TestTopicValidator_Unstaked(t *testing.T) { //NOTE: identity2 is not in the ids list simulating an un-staked node ids := flow.IdentityList{&identity1} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) // peer filter used by the topic validator to check if node is staked isStaked := func(pid peer.ID) error { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return fmt.Errorf("could not translate the peer_id %s to a Flow identifier: %w", pid.String(), err) } @@ -272,8 +275,7 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) idProvider := mockmodule.NewIdentityProvider(t) - // create a hooked logger - logger, hook := unittest.HookedLogger() + logger := unittest.Logger() sporkId := unittest.IdentifierFixture() @@ -292,12 +294,22 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + violation := &slashing.Violation{ + Identity: &identity3, + PeerID: an1.Host().ID().String(), + OriginID: identity3.NodeID, + MsgType: "*messages.BlockProposal", + Channel: channel, + Protocol: message.ProtocolTypePubSub, + Err: message.ErrUnauthorizedRole, + } + violationsConsumer := mocknetwork.NewViolationsConsumer(t) + violationsConsumer.On("OnUnAuthorizedSenderError", violation).Once().Return(nil) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -373,9 +385,6 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { p2pfixtures.SubMustNeverReceiveAnyMessage(t, timedCtx, sub2) unittest.RequireReturnsBefore(t, wg.Wait, 5*time.Second, "could not receive message on time") - - // ensure the correct error is contained in the logged error - require.Contains(t, hook.Logs(), message.ErrUnauthorizedRole.Error()) } // TestAuthorizedSenderValidator_Authorized tests that the authorized sender validator rejects messages being sent on the wrong channel @@ -401,12 +410,16 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity2.NodeID, alsp.UnAuthorizedSender) + require.NoError(t, err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -474,12 +487,16 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity2.NodeID, alsp.SenderEjected) + require.NoError(t, err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } @@ -568,13 +585,15 @@ func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { topic := channels.TopicFromChannel(channel, sporkId) ids := flow.IdentityList{&identity1, &identity2, &identity3} - translator, err := translator.NewFixedTableIdentityTranslator(ids) + translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) logger := unittest.Logger() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector()) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) + defer misbehaviorReportConsumer.AssertNotCalled(t, "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { - fid, err := translator.GetFlowID(pid) + fid, err := translatorFixture.GetFlowID(pid) if err != nil { return &flow.Identity{}, false } diff --git a/network/validator/authorized_sender_validator_test.go b/network/validator/authorized_sender_validator_test.go index 966ae5ba127..50e76e16855 100644 --- a/network/validator/authorized_sender_validator_test.go +++ b/network/validator/authorized_sender_validator_test.go @@ -6,16 +6,20 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/onflow/flow-go/model/flow" + libp2pmessage "github.com/onflow/flow-go/model/libp2p/message" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" @@ -54,7 +58,6 @@ func (s *TestAuthorizedSenderValidatorSuite) SetupTest() { s.initializeInvalidMessageOnChannelTestCases() s.initializeUnicastOnChannelTestCases() s.log = unittest.Logger() - s.slashingViolationsConsumer = slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector()) s.codec = unittest.NetworkCodec() } @@ -64,37 +67,64 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen for _, c := range s.authorizedSenderTestCases { str := fmt.Sprintf("role (%s) should be authorized to send message type (%s) on channel (%s)", c.Identity.Role, c.MessageStr, c.Channel) s.Run(str, func() { - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) - + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) + validateUnicast := authorizedSenderValidator.Validate + validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - + switch { // ensure according to the message auth config, if a message is authorized to be sent via unicast it - // is accepted or rejected. - msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) - if c.Protocols.Contains(message.ProtocolTypeUnicast) { + // is accepted. + case c.Protocols.Contains(message.ProtocolTypeUnicast): + msgType, err := validateUnicast(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) + if c.Protocols.Contains(message.ProtocolTypeUnicast) { + require.NoError(s.T(), err) + require.Equal(s.T(), c.MessageStr, msgType) + } + // ensure according to the message auth config, if a message is authorized to be sent via pubsub it + // is accepted. + case c.Protocols.Contains(message.ProtocolTypePubSub): + payload, err := s.codec.Encode(c.Message) require.NoError(s.T(), err) - require.Equal(s.T(), c.MessageStr, msgType) - } else { - require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), c.MessageStr, msgType) - } - - payload, err := s.codec.Encode(c.Message) - require.NoError(s.T(), err) - m := &message.Message{ - ChannelID: c.Channel.String(), - Payload: payload, - } - validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) - pubsubResult := validatePubsub(pid, m) - if !c.Protocols.Contains(message.ProtocolTypePubSub) { - require.Equal(s.T(), p2p.ValidationReject, pubsubResult) - } else { + m := &message.Message{ + ChannelID: c.Channel.String(), + Payload: payload, + } + pubsubResult := validatePubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) + default: + s.T().Fatal("authconfig does not contain any protocols") } }) } + + s.Run("test messages should be allowed to be sent via both protocols unicast/pubsub on test channel", func() { + identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(flow.RoleCollection)) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + getIdentityFunc := s.getIdentity(identity) + pid, err := unittest.PeerIDFromFlowID(identity) + require.NoError(s.T(), err) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) + + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeEcho.Uint8()}, channels.TestNetworkChannel, message.ProtocolTypeUnicast) + require.NoError(s.T(), err) + require.Equal(s.T(), "*message.TestMessage", msgType) + + payload, err := s.codec.Encode(&libp2pmessage.TestMessage{}) + require.NoError(s.T(), err) + m := &message.Message{ + ChannelID: channels.TestNetworkChannel.String(), + Payload: payload, + } + validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.TestNetworkChannel) + pubsubResult := validatePubsub(pid, m) + require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) + }) } // TestValidatorCallback_UnAuthorizedSender checks that AuthorizedSenderValidator.Validate return's p2p.ValidationReject @@ -105,8 +135,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedS s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnAuthorizedSender) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) payload, err := s.codec.Encode(c.Message) require.NoError(s.T(), err) @@ -129,8 +163,10 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedUni s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.NoError(s.T(), err) @@ -147,8 +183,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedU s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnauthorizedUnicastOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) @@ -165,8 +205,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedM s.Run(str, func() { pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnAuthorizedSender) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedMessageOnChannel) @@ -195,10 +239,22 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.UnauthorizedUnicastOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCluster(clusterID), expectedMisbehaviorReport).Once() + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCluster(clusterID), expectedMisbehaviorReport).Once() + + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) + + // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast + msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCluster(clusterID), message.ProtocolTypeUnicast) + require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) + require.Equal(s.T(), "*messages.SyncRequest", msgType) // ensure ClusterBlockProposal not allowed to be sent on channel via unicast - msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeClusterBlockProposal.Uint8()}, channels.ConsensusCluster(clusterID), message.ProtocolTypeUnicast) + msgType, err = authorizedSenderValidator.Validate(pid, []byte{codec.CodeClusterBlockProposal.Uint8()}, channels.ConsensusCluster(clusterID), message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) require.Equal(s.T(), "*messages.ClusterBlockProposal", msgType) @@ -213,11 +269,6 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix pubsubResult := validateCollConsensusPubsub(pid, m) require.Equal(s.T(), p2p.ValidationAccept, pubsubResult) - // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast - msgType, err = authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCluster(clusterID), message.ProtocolTypeUnicast) - require.ErrorIs(s.T(), err, message.ErrUnauthorizedUnicastOnChannel) - require.Equal(s.T(), "*messages.SyncRequest", msgType) - // ensure SyncRequest is allowed to be sent via pubsub by authorized sender payload, err = s.codec.Encode(&messages.SyncRequest{}) require.NoError(s.T(), err) @@ -239,7 +290,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.SenderEjected) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCommittee, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrSenderEjected) @@ -263,7 +319,12 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(identity.NodeID, alsp.UnknownMsgType) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCommittee, expectedMisbehaviorReport).Twice() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCommittee) // unknown message types are rejected @@ -291,7 +352,11 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, getIdentityFunc) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + // we cannot penalize a peer if identity is not known, in this case we don't expect any misbehavior reports to be reported + defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) require.ErrorIs(s.T(), err, ErrIdentityUnverified) @@ -314,17 +379,21 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnauthorizedP for _, c := range s.authorizedUnicastOnChannel { str := fmt.Sprintf("message type (%s) is not authorized to be sent via libp2p publish", c.MessageStr) s.Run(str, func() { + // skip test message check + if c.MessageStr == "*message.TestMessage" { + return + } pid, err := unittest.PeerIDFromFlowID(c.Identity) require.NoError(s.T(), err) - - authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, s.slashingViolationsConsumer, c.GetIdentity) + expectedMisbehaviorReport, err := alsp.NewMisbehaviorReport(c.Identity.NodeID, alsp.UnauthorizedPublishOnChannel) + require.NoError(s.T(), err) + misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) + misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypePubSub) - if c.MessageStr == "*message.TestMessage" { - require.NoError(s.T(), err) - } else { - require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) - require.Equal(s.T(), c.MessageStr, msgType) - } + require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) + require.Equal(s.T(), c.MessageStr, msgType) }) } } diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index 459a4db0e16..0a24b447966 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -21,6 +21,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go/network" + "github.com/onflow/flow-go/network/channels" cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" @@ -438,6 +439,18 @@ func GenerateRandomStringWithLen(commentLen uint) string { } // NetworkSlashingViolationsConsumer returns a slashing violations consumer for network middleware -func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics) slashing.ViolationsConsumer { - return slashing.NewSlashingViolationsConsumer(logger, metrics) +func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) slashing.ViolationsConsumer { + return slashing.NewSlashingViolationsConsumer(logger, metrics, consumer) +} + +type MisbehaviorReportConsumerFixture struct { + network.MisbehaviorReportManager +} + +func (c *MisbehaviorReportConsumerFixture) ReportMisbehaviorOnChannel(channel channels.Channel, report network.MisbehaviorReport) { + c.HandleMisbehaviorReport(channel, report) +} + +func NewMisbehaviorReportConsumerFixture(manager network.MisbehaviorReportManager) *MisbehaviorReportConsumerFixture { + return &MisbehaviorReportConsumerFixture{manager} } From 2368ca2486ae299bc3ff238b7d93bccdecf5ef69 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 23 Jun 2023 13:39:39 +0300 Subject: [PATCH 055/169] Removed old unnecessary comment, added small upgrade for ObserverCollector structure --- cmd/access/node_builder/access_node_builder.go | 2 +- cmd/observer/node_builder/observer_builder.go | 2 +- engine/access/rpc/engine.go | 7 ------- module/metrics/observer.go | 3 ++- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 38e36baaa2f..6222c1d4b53 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1008,7 +1008,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.executionGRPCPort, builder.retryEnabled, config.MaxMsgSize, - builder.rpcConf.BackendConfig) + config.BackendConfig) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) } diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 9e7ef69fb9a..ac4d1ed6823 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -870,7 +870,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { 0, false, config.MaxMsgSize, - builder.rpcConf.BackendConfig) + config.BackendConfig) if err != nil { return nil, fmt.Errorf("could not initialize backend: %w", err) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 875a9389e6d..e30e7d7a405 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -41,13 +41,6 @@ type Config struct { BackendConfig backend.Config // configurable options for creating Backend MaxMsgSize uint // GRPC max message size - //ExecutionClientTimeout time.Duration // execution API GRPC client timeout - //CollectionClientTimeout time.Duration // collection API GRPC client timeout - //ConnectionPoolSize uint // size of the cache for storing collection and execution connections - //MaxHeightRange uint // max size of height range requests - //PreferredExecutionNodeIDs []string // preferred list of upstream execution node IDs - //FixedExecutionNodeIDs []string // fixed list of execution node IDs to choose from if no node node ID can be chosen from the PreferredExecutionNodeIDs - //ArchiveAddressList []string // the archive node address list to send script executions. when configured, script executions will be all sent to the archive node } // Engine exposes the server with a simplified version of the Access API. diff --git a/module/metrics/observer.go b/module/metrics/observer.go index 95116b9f8f1..375aa66a2ac 100644 --- a/module/metrics/observer.go +++ b/module/metrics/observer.go @@ -11,10 +11,11 @@ type ObserverMetrics interface { } type ObserverCollector struct { - ObserverMetrics rpcs *prometheus.CounterVec } +var _ ObserverMetrics = (*ObserverCollector)(nil) + func NewObserverCollector() *ObserverCollector { return &ObserverCollector{ rpcs: promauto.NewCounterVec(prometheus.CounterOpts{ From 743b408ba14f3943d58244df0adc7d809eafd75e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 23 Jun 2023 08:56:53 -0400 Subject: [PATCH 056/169] add misbehavior report consumer factory --- .../node_builder/access_node_builder.go | 20 +++++++++-------- cmd/node_builder.go | 2 ++ cmd/observer/node_builder/observer_builder.go | 4 +++- cmd/scaffold.go | 22 +++++++++++-------- follower/follower_builder.go | 20 +++++++++-------- network/p2p/test/topic_validator_test.go | 6 ++--- network/slashing/consumer.go | 16 +++++++------- .../authorized_sender_validator_test.go | 22 +++++++++---------- utils/unittest/unittest.go | 8 ++++++- 9 files changed, 69 insertions(+), 51 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index e50dadc9948..128795c885c 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1262,15 +1262,17 @@ func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, ) network.Middleware { logger := builder.Logger.With().Bool("staked", false).Logger() mw := middleware.NewMiddleware(&middleware.Config{ - Logger: logger, - Libp2pNode: libp2pNode, - FlowId: nodeID, - BitSwapMetrics: builder.Metrics.Bitswap, - RootBlockID: builder.SporkID, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - IdTranslator: builder.IDTranslator, - Codec: builder.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(logger, networkMetrics), + Logger: logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), + SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(logger, networkMetrics, func() network.MisbehaviorReportConsumer { + return builder.MisbehaviorReportConsumer + }), }, middleware.WithMessageValidators(validators...), // use default identifier provider ) diff --git a/cmd/node_builder.go b/cmd/node_builder.go index e637ab3715c..314fecd6296 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -293,6 +293,8 @@ type NodeConfig struct { // UnicastRateLimiterDistributor notifies consumers when a peer's unicast message is rate limited. UnicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor + // MisbehaviorReportConsumer consumers used to disseminate misbehavior reports to the ALSP misbehavior report manager. + MisbehaviorReportConsumer network.MisbehaviorReportConsumer } // StateExcerptAtBoot stores information about the root snapshot and latest finalized block for use in bootstrapping. diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index be2a8b35260..2b114086d93 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -910,7 +910,9 @@ func (builder *ObserverServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, validators ...network.MessageValidator, ) network.Middleware { - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network) + slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network, func() network.MisbehaviorReportConsumer { + return builder.MisbehaviorReportConsumer + }) mw := middleware.NewMiddleware(&middleware.Config{ Logger: builder.Logger, Libp2pNode: libp2pNode, diff --git a/cmd/scaffold.go b/cmd/scaffold.go index cc9771734a3..3094aa2d563 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -485,16 +485,19 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory( if len(peerManagerFilters) > 0 { mwOpts = append(mwOpts, middleware.WithPeerManagerFilters(peerManagerFilters)) } + mw := middleware.NewMiddleware(&middleware.Config{ - Logger: fnb.Logger, - Libp2pNode: fnb.LibP2PNode, - FlowId: fnb.Me.NodeID(), - BitSwapMetrics: fnb.Metrics.Bitswap, - RootBlockID: fnb.SporkID, - UnicastMessageTimeout: fnb.BaseConfig.UnicastMessageTimeout, - IdTranslator: fnb.IDTranslator, - Codec: fnb.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(fnb.Logger, fnb.Metrics.Network), + Logger: fnb.Logger, + Libp2pNode: fnb.LibP2PNode, + FlowId: fnb.Me.NodeID(), + BitSwapMetrics: fnb.Metrics.Bitswap, + RootBlockID: fnb.SporkID, + UnicastMessageTimeout: fnb.BaseConfig.UnicastMessageTimeout, + IdTranslator: fnb.IDTranslator, + Codec: fnb.CodecFactory(), + SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(fnb.Logger, fnb.Metrics.Network, func() network.MisbehaviorReportConsumer { + return fnb.MisbehaviorReportConsumer + }), }, mwOpts...) @@ -539,6 +542,7 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory( } fnb.Network = net + fnb.MisbehaviorReportConsumer = net // register middleware's ReadyDoneAware interface so other components can depend on it for startup if fnb.middlewareDependable != nil { diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 5d2d2d15582..5e0f699a91b 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -748,15 +748,17 @@ func (builder *FollowerServiceBuilder) initMiddleware(nodeID flow.Identifier, validators ...network.MessageValidator, ) network.Middleware { mw := middleware.NewMiddleware(&middleware.Config{ - Logger: builder.Logger, - Libp2pNode: libp2pNode, - FlowId: nodeID, - BitSwapMetrics: builder.Metrics.Bitswap, - RootBlockID: builder.SporkID, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - IdTranslator: builder.IDTranslator, - Codec: builder.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network), + Logger: builder.Logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), + SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network, func() network.MisbehaviorReportConsumer { + return builder.MisbehaviorReportConsumer + }), }, middleware.WithMessageValidators(validators...), ) diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index e25e8673220..9348544174f 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -417,7 +417,7 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { require.NoError(t, err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { @@ -494,7 +494,7 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { require.NoError(t, err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { @@ -591,7 +591,7 @@ func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { logger := unittest.Logger() misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) defer misbehaviorReportConsumer.AssertNotCalled(t, "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index 3a7943c16ed..b2c8c74ca85 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -19,17 +19,17 @@ const ( // Consumer is a struct that logs a message for any slashable offenses. // This struct will be updated in the future when slashing is implemented. type Consumer struct { - log zerolog.Logger - metrics module.NetworkSecurityMetrics - misbehaviorReportConsumer network.MisbehaviorReportConsumer + log zerolog.Logger + metrics module.NetworkSecurityMetrics + reportConsumerFactory func() network.MisbehaviorReportConsumer } // NewSlashingViolationsConsumer returns a new Consumer. -func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) *Consumer { +func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, reportConsumerFactory func() network.MisbehaviorReportConsumer) *Consumer { return &Consumer{ - log: log.With().Str("module", "network_slashing_consumer").Logger(), - metrics: metrics, - misbehaviorReportConsumer: consumer, + log: log.With().Str("module", "network_slashing_consumer").Logger(), + metrics: metrics, + reportConsumerFactory: reportConsumerFactory, } } @@ -76,7 +76,7 @@ func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation if err != nil { return fmt.Errorf("failed to create misbehavior report: %w", err) } - c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) + c.reportConsumerFactory().ReportMisbehaviorOnChannel(violation.Channel, report) return nil } diff --git a/network/validator/authorized_sender_validator_test.go b/network/validator/authorized_sender_validator_test.go index 50e76e16855..7ba2fb80e2d 100644 --- a/network/validator/authorized_sender_validator_test.go +++ b/network/validator/authorized_sender_validator_test.go @@ -69,7 +69,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen s.Run(str, func() { misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) validateUnicast := authorizedSenderValidator.Validate validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) @@ -105,7 +105,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(flow.RoleCollection)) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) getIdentityFunc := s.getIdentity(identity) pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) @@ -139,7 +139,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedS require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) payload, err := s.codec.Encode(c.Message) @@ -165,7 +165,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedUni require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -187,7 +187,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedU require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -209,7 +209,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedM require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -245,7 +245,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCluster(clusterID), expectedMisbehaviorReport).Once() misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCluster(clusterID), expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast @@ -294,7 +294,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCommittee, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) @@ -323,7 +323,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCommittee, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCommittee) @@ -355,7 +355,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) // we cannot penalize a peer if identity is not known, in this case we don't expect any misbehavior reports to be reported defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) @@ -389,7 +389,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnauthorizedP require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypePubSub) require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index 0a24b447966..ea4c3c6c28e 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -440,7 +440,7 @@ func GenerateRandomStringWithLen(commentLen uint) string { // NetworkSlashingViolationsConsumer returns a slashing violations consumer for network middleware func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) slashing.ViolationsConsumer { - return slashing.NewSlashingViolationsConsumer(logger, metrics, consumer) + return slashing.NewSlashingViolationsConsumer(logger, metrics, MisbehaviorReportConsumerFactory(consumer)) } type MisbehaviorReportConsumerFixture struct { @@ -454,3 +454,9 @@ func (c *MisbehaviorReportConsumerFixture) ReportMisbehaviorOnChannel(channel ch func NewMisbehaviorReportConsumerFixture(manager network.MisbehaviorReportManager) *MisbehaviorReportConsumerFixture { return &MisbehaviorReportConsumerFixture{manager} } + +func MisbehaviorReportConsumerFactory(consumer network.MisbehaviorReportConsumer) func() network.MisbehaviorReportConsumer { + return func() network.MisbehaviorReportConsumer { + return consumer + } +} From 8854978fddf7b0fdacb8c1ec474201318eeffc6f Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 23 Jun 2023 17:04:25 +0300 Subject: [PATCH 057/169] Refactored rest api --- cmd/observer/node_builder/observer_builder.go | 7 +- .../export_report.json | 6 + engine/access/rest/api/api.go | 41 + engine/access/rest/apiproxy/forwarder.go | 604 ++++++++++ engine/access/rest/apiproxy/router.go | 126 ++ engine/access/rest/handler.go | 9 +- engine/access/rest/{ => models}/error.go | 2 +- engine/access/rest/rest_server_api.go | 1017 ----------------- engine/access/rest/router.go | 32 +- engine/access/rest/{ => routes}/accounts.go | 7 +- engine/access/rest/{ => routes}/blocks.go | 108 +- .../access/rest/{ => routes}/collections.go | 7 +- engine/access/rest/{ => routes}/events.go | 11 +- .../rest/{ => routes}/execution_result.go | 11 +- engine/access/rest/{ => routes}/network.go | 5 +- .../rest/{ => routes}/node_version_info.go | 5 +- engine/access/rest/{ => routes}/scripts.go | 7 +- .../access/rest/{ => routes}/transactions.go | 15 +- engine/access/rest/server.go | 5 +- engine/access/rest/server_request_handler.go | 346 ++++++ .../access/rest/{ => tests}/accounts_test.go | 2 +- engine/access/rest/{ => tests}/blocks_test.go | 2 +- .../rest/{ => tests}/collections_test.go | 2 +- engine/access/rest/{ => tests}/events_test.go | 20 +- .../rest/{ => tests}/execution_result_test.go | 2 +- .../access/rest/{ => tests}/network_test.go | 2 +- .../{ => tests}/node_version_info_test.go | 2 +- .../access/rest/{ => tests}/scripts_test.go | 2 +- .../access/rest/{ => tests}/test_helpers.go | 23 +- .../rest/{ => tests}/transactions_test.go | 2 +- engine/access/rest_api_test.go | 9 +- engine/access/rpc/engine.go | 3 +- engine/access/rpc/engine_builder.go | 7 +- 33 files changed, 1260 insertions(+), 1189 deletions(-) create mode 100644 cmd/util/cmd/execution-state-extract/export_report.json create mode 100644 engine/access/rest/api/api.go create mode 100644 engine/access/rest/apiproxy/forwarder.go create mode 100644 engine/access/rest/apiproxy/router.go rename engine/access/rest/{ => models}/error.go (98%) delete mode 100644 engine/access/rest/rest_server_api.go rename engine/access/rest/{ => routes}/accounts.go (59%) rename engine/access/rest/{ => routes}/blocks.go (61%) rename engine/access/rest/{ => routes}/collections.go (59%) rename engine/access/rest/{ => routes}/events.go (51%) rename engine/access/rest/{ => routes}/execution_result.go (56%) rename engine/access/rest/{ => routes}/network.go (56%) rename engine/access/rest/{ => routes}/node_version_info.go (54%) rename engine/access/rest/{ => routes}/scripts.go (57%) rename engine/access/rest/{ => routes}/transactions.go (55%) create mode 100644 engine/access/rest/server_request_handler.go rename engine/access/rest/{ => tests}/accounts_test.go (99%) rename engine/access/rest/{ => tests}/blocks_test.go (99%) rename engine/access/rest/{ => tests}/collections_test.go (99%) rename engine/access/rest/{ => tests}/events_test.go (98%) rename engine/access/rest/{ => tests}/execution_result_test.go (99%) rename engine/access/rest/{ => tests}/network_test.go (98%) rename engine/access/rest/{ => tests}/node_version_info_test.go (99%) rename engine/access/rest/{ => tests}/scripts_test.go (99%) rename engine/access/rest/{ => tests}/test_helpers.go (68%) rename engine/access/rest/{ => tests}/transactions_test.go (99%) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index ac4d1ed6823..1920aec0106 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -30,6 +30,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/apiproxy" "github.com/onflow/flow-go/engine/access/rest" + restapiproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/follower" @@ -912,7 +913,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { )), } - restForwarder, err := rest.NewRestForwarder(builder.Logger, + restForwarder, err := restapiproxy.NewRestForwarder(builder.Logger, builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) @@ -920,11 +921,11 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return nil, err } - restHandler := &rest.RestRouter{ + restHandler := &restapiproxy.RestRouter{ Logger: builder.Logger, Metrics: observerCollector, Upstream: restForwarder, - Observer: rest.NewRequestHandler(builder.Logger, accessBackend), + Observer: rest.NewServerRequestHandler(builder.Logger, accessBackend), } // build the rpc engine diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json new file mode 100644 index 00000000000..3c4a27478db --- /dev/null +++ b/cmd/util/cmd/execution-state-extract/export_report.json @@ -0,0 +1,6 @@ +{ + "EpochCounter": 0, + "PreviousStateCommitment": "8536ee2769a5b35be123a54e45a23d2eaf3fa9f3df3bde6a713c87c286b9ec40", + "CurrentStateCommitment": "8536ee2769a5b35be123a54e45a23d2eaf3fa9f3df3bde6a713c87c286b9ec40", + "ReportSucceeded": true +} \ No newline at end of file diff --git a/engine/access/rest/api/api.go b/engine/access/rest/api/api.go new file mode 100644 index 00000000000..aea539e0571 --- /dev/null +++ b/engine/access/rest/api/api.go @@ -0,0 +1,41 @@ +package api + +import ( + "context" + + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/model/flow" +) + +// RestServerApi is the server API for REST service. +type RestServerApi interface { + // GetTransactionByID gets a transaction by requested ID. + GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) + // CreateTransaction creates a new transaction from provided payload. + CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) + // GetTransactionResultByID retrieves transaction result by the transaction ID. + GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) + // GetBlocksByIDs gets blocks by provided ID or list of IDs. + GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) + // GetBlocksByHeight gets blocks by provided height. + GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) + // GetBlockPayloadByID gets block payload by ID + GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) + // GetExecutionResultByID gets execution result by the ID. + GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) + // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. + GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) + // GetCollectionByID retrieves a collection by ID and builds a response + GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) + // ExecuteScript handler sends the script from the request to be executed. + ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) + // GetAccount handler retrieves account by address and returns the response. + GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) + // GetEvents for the provided block range or list of block IDs filtered by type. + GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) + // GetNetworkParameters returns network-wide parameters of the blockchain + GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) + // GetNodeVersionInfo returns node version information + GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) +} diff --git a/engine/access/rest/apiproxy/forwarder.go b/engine/access/rest/apiproxy/forwarder.go new file mode 100644 index 00000000000..6a62e302973 --- /dev/null +++ b/engine/access/rest/apiproxy/forwarder.go @@ -0,0 +1,604 @@ +package apiproxy + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/access/rest/routes" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/forwarder" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" +) + +// RestForwarder handles the request forwarding to upstream +type RestForwarder struct { + log zerolog.Logger + *forwarder.Forwarder +} + +var _ api.RestServerApi = (*RestForwarder)(nil) + +// NewRestForwarder returns new RestForwarder. +func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*RestForwarder, error) { + f, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) + + restForwarder := &RestForwarder{ + log: log, + Forwarder: f, + } + return restForwarder, err +} + +// GetTransactionByID gets a transaction by requested ID. +func (f *RestForwarder) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { + var response models.Transaction + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + } + transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) + if err != nil { + return response, err + } + + var transactionResultResponse *accessproto.TransactionResultResponse + // only lookup result if transaction result is to be expanded + if r.ExpandsResult { + getTransactionResultRequest := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + BlockId: r.BlockID[:], + CollectionId: r.CollectionID[:], + } + transactionResultResponse, err = upstream.GetTransactionResult(context, getTransactionResultRequest) + if err != nil { + return response, err + } + } + flowTransaction, err := convert.MessageToTransaction(transactionResponse.Transaction, chain) + if err != nil { + return response, err + } + + flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) + + response.Build(&flowTransaction, flowTransactionResult, link) + return response, nil +} + +// CreateTransaction creates a new transaction from provided payload. +func (f *RestForwarder) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + var response models.Transaction + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + entitiesTransaction := convert.TransactionToMessage(r.Transaction) + sendTransactionRequest := &accessproto.SendTransactionRequest{ + Transaction: entitiesTransaction, + } + + _, err = upstream.SendTransaction(context, sendTransactionRequest) + if err != nil { + return response, err + } + + response.Build(&r.Transaction, nil, link) + return response, nil +} + +// GetTransactionResultByID retrieves transaction result by the transaction ID. +func (f *RestForwarder) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + var response models.TransactionResult + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getTransactionResult := &accessproto.GetTransactionRequest{ + Id: r.ID[:], + BlockId: r.BlockID[:], + CollectionId: r.CollectionID[:], + } + transactionResultResponse, err := upstream.GetTransactionResult(context, getTransactionResult) + if err != nil { + return response, err + } + + flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) + response.Build(flowTransactionResult, r.ID, link) + return response, nil +} + +// GetBlocksByIDs gets blocks by provided ID or list of IDs. +func (f *RestForwarder) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + blocks := make([]*models.Block, len(r.IDs)) + + upstream, err := f.FaultTolerantClient() + if err != nil { + return blocks, err + } + + for i, id := range r.IDs { + block, err := getBlockFromGrpc(routes.ForID(&id), context, expandFields, upstream, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil +} + +// GetBlocksByHeight gets blocks by provided height. +func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + req, err := r.GetBlockRequest() + if err != nil { + return nil, models.NewBadRequestError(err) + } + + upstream, err := f.FaultTolerantClient() + if err != nil { + return nil, err + } + + if req.FinalHeight || req.SealedHeight { + block, err := getBlockFromGrpc(routes.ForFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + + return []*models.Block{block}, nil + } + + // if the query is /blocks/height=1000,1008,1049... + if req.HasHeights() { + blocks := make([]*models.Block, len(req.Heights)) + for i, height := range req.Heights { + block, err := getBlockFromGrpc(routes.ForHeight(height), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil + } + + // support providing end height as "sealed" or "final" + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ + IsSealed: req.EndHeight == request.SealedHeight, + } + blockResponse, err := upstream.GetLatestBlock(r.Context(), getLatestBlockRequest) + if err != nil { + return nil, err + } + + req.EndHeight = blockResponse.Block.BlockHeader.Height // overwrite special value height with fetched + + if req.StartHeight > req.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + } + } + + blocks := make([]*models.Block, 0) + // start and end height inclusive + for i := req.StartHeight; i <= req.EndHeight; i++ { + block, err := getBlockFromGrpc(routes.ForHeight(i), r.Context(), r.ExpandFields, upstream, link) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil +} + +// GetBlockPayloadByID gets block payload by ID +func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { + var payload models.BlockPayload + + upstream, err := f.FaultTolerantClient() + if err != nil { + return payload, err + } + + blkProvider := routes.NewBlockFromGrpcProvider(upstream, routes.ForID(&r.ID)) + block, _, statusErr := blkProvider.GetBlock(context) + if statusErr != nil { + return payload, statusErr + } + + flowPayload, err := convert.PayloadFromMessage(block) + if err != nil { + return payload, err + } + + err = payload.Build(flowPayload) + if err != nil { + return payload, err + } + + return payload, nil +} + +// GetExecutionResultByID gets execution result by the ID. +func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + var response models.ExecutionResult + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ + Id: r.ID[:], + } + + executionResultByIDResponse, err := upstream.GetExecutionResultByID(context, executionResultByIDRequest) + if err != nil { + return response, err + } + + if executionResultByIDResponse == nil { + err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) + return response, models.NewNotFoundError(err.Error(), err) + } + + flowExecResult, err := convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) + if err != nil { + return response, err + } + err = response.Build(flowExecResult, link) + if err != nil { + return response, err + } + + return response, nil +} + +// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. +func (f *RestForwarder) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + // for each block ID we retrieve execution result + results := make([]models.ExecutionResult, len(r.BlockIDs)) + + upstream, err := f.FaultTolerantClient() + if err != nil { + return results, err + } + + for i, id := range r.BlockIDs { + getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: id[:], + } + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockID) + if err != nil { + return nil, err + } + + var response models.ExecutionResult + flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) + if err != nil { + return nil, err + } + err = response.Build(flowExecResult, link) + if err != nil { + return nil, err + } + results[i] = response + } + + return results, nil +} + +// GetCollectionByID retrieves a collection by ID and builds a response +func (f *RestForwarder) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { + var response models.Collection + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ + Id: r.ID[:], + } + + collectionResponse, err := upstream.GetCollectionByID(context, getCollectionByIDRequest) + if err != nil { + return response, err + } + + // if we expand transactions in the query retrieve each transaction data + transactions := make([]*entities.Transaction, 0) + if r.ExpandsTransactions { + for _, tid := range collectionResponse.Collection.TransactionIds { + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: tid, + } + transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) + if err != nil { + return response, err + } + + transactions = append(transactions, transactionResponse.Transaction) + } + } + + err = response.BuildFromGrpc(collectionResponse.Collection, transactions, link, expandFields, chain) + if err != nil { + return response, err + } + + return response, nil +} + +// ExecuteScript handler sends the script from the request to be executed. +func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { + upstream, err := f.FaultTolerantClient() + if err != nil { + return nil, err + } + + if r.BlockID != flow.ZeroID { + executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ + BlockId: r.BlockID[:], + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(context, executeScriptAtBlockIDRequest) + if err != nil { + return nil, err + } + return executeScriptAtBlockIDResponse.Value, nil + } + + // default to sealed height + if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { + executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(context, executeScriptAtLatestBlockRequest) + if err != nil { + return nil, err + } + return executeScriptAtLatestBlockResponse.Value, nil + } + + if r.BlockHeight == request.FinalHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: false, + } + getLatestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return nil, err + } + r.BlockHeight = getLatestBlockHeaderResponse.Block.Height + } + + executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ + BlockHeight: r.BlockHeight, + Script: r.Script.Source, + Arguments: r.Script.Args, + } + executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(context, executeScriptAtBlockHeightRequest) + if err != nil { + return nil, err + } + return executeScriptAtBlockHeightResponse.Value, nil +} + +// GetAccount handler retrieves account by address and returns the response. +func (f *RestForwarder) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + var response models.Account + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it + if r.Height == request.FinalHeight || r.Height == request.SealedHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: r.Height == request.SealedHeight, + } + blockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return response, err + } + r.Height = blockHeaderResponse.Block.Height + } + getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ + Address: r.Address.Bytes(), + BlockHeight: r.Height, + } + + accountResponse, err := upstream.GetAccountAtBlockHeight(context, getAccountAtBlockHeightRequest) + if err != nil { + return response, models.NewNotFoundError("not found account at block height", err) + } + + flowAccount, err := convert.MessageToAccount(accountResponse.Account) + if err != nil { + return response, err + } + + err = response.Build(flowAccount, link, expandFields) + return response, err +} + +// GetEvents for the provided block range or list of block IDs filtered by type. +func (f *RestForwarder) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { + // if the request has block IDs provided then return events for block IDs + var blocksEvents models.BlocksEvents + + upstream, err := f.FaultTolerantClient() + if err != nil { + return blocksEvents, err + } + + if len(r.BlockIDs) > 0 { + var blockIds [][]byte + for _, id := range r.BlockIDs { + blockIds = append(blockIds, id[:]) + } + getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ + Type: r.Type, + BlockIds: blockIds, + } + eventsResponse, err := upstream.GetEventsForBlockIDs(context, getEventsForBlockIDsRequest) + if err != nil { + return nil, err + } + + blocksEvents.BuildFromGrpc(eventsResponse.Results) + + return blocksEvents, nil + } + + // if end height is provided with special values then load the height + if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: r.EndHeight == request.SealedHeight, + } + latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) + if err != nil { + return nil, err + } + + r.EndHeight = latestBlockHeaderResponse.Block.Height + // special check after we resolve special height value + if r.StartHeight > r.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + } + } + + // if request provided block height range then return events for that range + getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ + Type: r.Type, + StartHeight: r.StartHeight, + EndHeight: r.EndHeight, + } + eventsResponse, err := upstream.GetEventsForHeightRange(context, getEventsForHeightRangeRequest) + if err != nil { + return nil, err + } + + blocksEvents.BuildFromGrpc(eventsResponse.Results) + return blocksEvents, nil +} + +// GetNetworkParameters returns network-wide parameters of the blockchain +func (f *RestForwarder) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { + var response models.NetworkParameters + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getNetworkParametersRequest := &accessproto.GetNetworkParametersRequest{} + getNetworkParametersResponse, err := upstream.GetNetworkParameters(r.Context(), getNetworkParametersRequest) + if err != nil { + return response, err + } + response.BuildFromGrpc(getNetworkParametersResponse) + return response, nil +} + +// GetNodeVersionInfo returns node version information +func (f *RestForwarder) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { + var response models.NodeVersionInfo + + upstream, err := f.FaultTolerantClient() + if err != nil { + return response, err + } + + getNodeVersionInfoRequest := &accessproto.GetNodeVersionInfoRequest{} + getNodeVersionInfoResponse, err := upstream.GetNodeVersionInfo(r.Context(), getNodeVersionInfoRequest) + if err != nil { + return response, err + } + + response.BuildFromGrpc(getNodeVersionInfoResponse.Info) + return response, nil +} + +func getBlockFromGrpc(option routes.BlockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { + // lookup block + blkProvider := routes.NewBlockFromGrpcProvider(upstream, option) + blk, blockStatus, err := blkProvider.GetBlock(context) + if err != nil { + return nil, err + } + + // lookup execution result + // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) + var block models.Block + getExecutionResultForBlockIDRequest := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: blk.Id, + } + + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockIDRequest) + if err != nil { + return nil, err + } + + flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) + if err != nil { + return nil, err + } + + flowBlock, err := convert.MessageToBlock(blk) + if err != nil { + return nil, err + } + + flowBlockStatus, err := convert.MessagesToBlockStatus(blockStatus) + if err != nil { + return nil, err + } + + if err != nil { + // handle case where execution result is not yet available + if se, ok := status.FromError(err); ok { + if se.Code() == codes.NotFound { + err := block.Build(flowBlock, nil, link, flowBlockStatus, expandFields) + if err != nil { + return nil, err + } + return &block, nil + } + } + return nil, err + } + + err = block.Build(flowBlock, flowExecResult, link, flowBlockStatus, expandFields) + if err != nil { + return nil, err + } + return &block, nil +} diff --git a/engine/access/rest/apiproxy/router.go b/engine/access/rest/apiproxy/router.go new file mode 100644 index 00000000000..beb74b6ca41 --- /dev/null +++ b/engine/access/rest/apiproxy/router.go @@ -0,0 +1,126 @@ +package apiproxy + +import ( + "context" + + "google.golang.org/grpc/status" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" +) + +// RestRouter is a structure that represents the routing proxy algorithm for observer node. +// It splits requests between a local requests and request forwarding to upstream service. +type RestRouter struct { + Logger zerolog.Logger + Metrics metrics.ObserverMetrics + Upstream api.RestServerApi + Observer api.RestServerApi +} + +func (r *RestRouter) log(handler, rpc string, err error) { + code := status.Code(err) + r.Metrics.RecordRPC(handler, rpc, code) + + logger := r.Logger.With(). + Str("handler", handler). + Str("rest_method", rpc). + Str("rest_code", code.String()). + Logger() + + if err != nil { + logger.Error().Err(err).Msg("request failed") + return + } + + logger.Info().Msg("request succeeded") +} + +func (r *RestRouter) GetTransactionByID(req request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { + res, err := r.Upstream.GetTransactionByID(req, context, link, chain) + r.log("upstream", "GetNodeVersionInfo", err) + return res, err +} + +func (r *RestRouter) CreateTransaction(req request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + res, err := r.Upstream.CreateTransaction(req, context, link) + r.log("upstream", "CreateTransaction", err) + return res, err +} + +func (r *RestRouter) GetTransactionResultByID(req request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + res, err := r.Upstream.GetTransactionResultByID(req, context, link) + r.log("upstream", "GetTransactionResultByID", err) + return res, err +} + +func (r *RestRouter) GetBlocksByIDs(req request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + res, err := r.Observer.GetBlocksByIDs(req, context, expandFields, link) + r.log("observer", "GetBlocksByIDs", err) + return res, err +} + +func (r *RestRouter) GetBlocksByHeight(req *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + res, err := r.Observer.GetBlocksByHeight(req, link) + r.log("observer", "GetBlocksByHeight", err) + return res, err +} + +func (r *RestRouter) GetBlockPayloadByID(req request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) { + res, err := r.Observer.GetBlockPayloadByID(req, context, link) + r.log("observer", "GetBlockPayloadByID", err) + return res, err +} + +func (r *RestRouter) GetExecutionResultByID(req request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + res, err := r.Upstream.GetExecutionResultByID(req, context, link) + r.log("upstream", "GetExecutionResultByID", err) + return res, err +} + +func (r *RestRouter) GetExecutionResultsByBlockIDs(req request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + res, err := r.Upstream.GetExecutionResultsByBlockIDs(req, context, link) + r.log("upstream", "GetExecutionResultsByBlockIDs", err) + return res, err +} + +func (r *RestRouter) GetCollectionByID(req request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { + res, err := r.Upstream.GetCollectionByID(req, context, expandFields, link, chain) + r.log("upstream", "GetCollectionByID", err) + return res, err +} + +func (r *RestRouter) ExecuteScript(req request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { + res, err := r.Upstream.ExecuteScript(req, context, link) + r.log("upstream", "ExecuteScript", err) + return res, err +} + +func (r *RestRouter) GetAccount(req request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + res, err := r.Upstream.GetAccount(req, context, expandFields, link) + r.log("upstream", "GetAccount", err) + return res, err +} + +func (r *RestRouter) GetEvents(req request.GetEvents, context context.Context) (models.BlocksEvents, error) { + res, err := r.Upstream.GetEvents(req, context) + r.log("upstream", "GetEvents", err) + return res, err +} + +func (r *RestRouter) GetNetworkParameters(req *request.Request) (models.NetworkParameters, error) { + res, err := r.Observer.GetNetworkParameters(req) + r.log("observer", "GetNetworkParameters", err) + return res, err +} + +func (r *RestRouter) GetNodeVersionInfo(req *request.Request) (models.NodeVersionInfo, error) { + res, err := r.Observer.GetNodeVersionInfo(req) + r.log("observer", "GetNodeVersionInfo", err) + return res, err +} diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index 03d8695fbb7..c56b5ccf114 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -11,6 +11,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/util" @@ -24,7 +25,7 @@ const MaxRequestSize = 2 << 20 // 2MB // it fetches necessary resources and returns an error or response model. type ApiHandlerFunc func( r *request.Request, - srv RestServerApi, + srv api.RestServerApi, generator models.LinkGenerator, ) (interface{}, error) @@ -33,7 +34,7 @@ type ApiHandlerFunc func( // wraps functionality for handling error and responses outside of endpoint handling. type Handler struct { logger zerolog.Logger - restServerAPI RestServerApi + restServerAPI api.RestServerApi linkGenerator models.LinkGenerator apiHandlerFunc ApiHandlerFunc chain flow.Chain @@ -41,7 +42,7 @@ type Handler struct { func NewHandler( logger zerolog.Logger, - restServerAPI RestServerApi, + restServerAPI api.RestServerApi, handlerFunc ApiHandlerFunc, generator models.LinkGenerator, chain flow.Chain, @@ -92,7 +93,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *Handler) errorHandler(w http.ResponseWriter, err error, errorLogger zerolog.Logger) { // rest status type error should be returned with status and user message provided - var statusErr StatusError + var statusErr models.StatusError if errors.As(err, &statusErr) { h.errorResponse(w, statusErr.Status(), statusErr.UserMessage(), errorLogger) return diff --git a/engine/access/rest/error.go b/engine/access/rest/models/error.go similarity index 98% rename from engine/access/rest/error.go rename to engine/access/rest/models/error.go index 7403510ba55..2247b38743b 100644 --- a/engine/access/rest/error.go +++ b/engine/access/rest/models/error.go @@ -1,4 +1,4 @@ -package rest +package models import "net/http" diff --git a/engine/access/rest/rest_server_api.go b/engine/access/rest/rest_server_api.go deleted file mode 100644 index 27d85960026..00000000000 --- a/engine/access/rest/rest_server_api.go +++ /dev/null @@ -1,1017 +0,0 @@ -package rest - -import ( - "context" - "fmt" - "time" - - "google.golang.org/grpc/status" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/common/rpc/convert" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/forwarder" - "github.com/onflow/flow-go/module/metrics" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/entities" -) - -// RestRouter is a structure that represents the routing proxy algorithm. -// It splits requests between a local and a remote rest service. -type RestRouter struct { - Logger zerolog.Logger - Metrics metrics.ObserverMetrics - Upstream RestServerApi - Observer *RequestHandler -} - -func (r *RestRouter) log(handler, rpc string, err error) { - code := status.Code(err) - r.Metrics.RecordRPC(handler, rpc, code) - - logger := r.Logger.With(). - Str("handler", handler). - Str("rest_method", rpc). - Str("rest_code", code.String()). - Logger() - - if err != nil { - logger.Error().Err(err).Msg("request failed") - return - } - - logger.Info().Msg("request succeeded") -} - -func (r *RestRouter) GetTransactionByID(req request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { - res, err := r.Upstream.GetTransactionByID(req, context, link, chain) - r.log("upstream", "GetNodeVersionInfo", err) - return res, err -} - -func (r *RestRouter) CreateTransaction(req request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - res, err := r.Upstream.CreateTransaction(req, context, link) - r.log("upstream", "CreateTransaction", err) - return res, err -} - -func (r *RestRouter) GetTransactionResultByID(req request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - res, err := r.Upstream.GetTransactionResultByID(req, context, link) - r.log("upstream", "GetTransactionResultByID", err) - return res, err -} - -func (r *RestRouter) GetBlocksByIDs(req request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - res, err := r.Observer.GetBlocksByIDs(req, context, expandFields, link) - r.log("observer", "GetBlocksByIDs", err) - return res, err -} - -func (r *RestRouter) GetBlocksByHeight(req *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - res, err := r.Observer.GetBlocksByHeight(req, link) - r.log("observer", "GetBlocksByHeight", err) - return res, err -} - -func (r *RestRouter) GetBlockPayloadByID(req request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) { - res, err := r.Observer.GetBlockPayloadByID(req, context, link) - r.log("observer", "GetBlockPayloadByID", err) - return res, err -} - -func (r *RestRouter) GetExecutionResultByID(req request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - res, err := r.Upstream.GetExecutionResultByID(req, context, link) - r.log("upstream", "GetExecutionResultByID", err) - return res, err -} - -func (r *RestRouter) GetExecutionResultsByBlockIDs(req request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - res, err := r.Upstream.GetExecutionResultsByBlockIDs(req, context, link) - r.log("upstream", "GetExecutionResultsByBlockIDs", err) - return res, err -} - -func (r *RestRouter) GetCollectionByID(req request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { - res, err := r.Upstream.GetCollectionByID(req, context, expandFields, link, chain) - r.log("upstream", "GetCollectionByID", err) - return res, err -} - -func (r *RestRouter) ExecuteScript(req request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { - res, err := r.Upstream.ExecuteScript(req, context, link) - r.log("upstream", "ExecuteScript", err) - return res, err -} - -func (r *RestRouter) GetAccount(req request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - res, err := r.Upstream.GetAccount(req, context, expandFields, link) - r.log("upstream", "GetAccount", err) - return res, err -} - -func (r *RestRouter) GetEvents(req request.GetEvents, context context.Context) (models.BlocksEvents, error) { - res, err := r.Upstream.GetEvents(req, context) - r.log("upstream", "GetEvents", err) - return res, err -} - -func (r *RestRouter) GetNetworkParameters(req *request.Request) (models.NetworkParameters, error) { - res, err := r.Observer.GetNetworkParameters(req) - r.log("observer", "GetNetworkParameters", err) - return res, err -} - -func (r *RestRouter) GetNodeVersionInfo(req *request.Request) (models.NodeVersionInfo, error) { - res, err := r.Observer.GetNodeVersionInfo(req) - r.log("observer", "GetNodeVersionInfo", err) - return res, err -} - -// RestServerApi is the server API for REST service. -type RestServerApi interface { - // GetTransactionByID gets a transaction by requested ID. - GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) - // CreateTransaction creates a new transaction from provided payload. - CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) - // GetTransactionResultByID retrieves transaction result by the transaction ID. - GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) - // GetBlocksByIDs gets blocks by provided ID or list of IDs. - GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) - // GetBlocksByHeight gets blocks by provided height. - GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) - // GetBlockPayloadByID gets block payload by ID - GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) - // GetExecutionResultByID gets execution result by the ID. - GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) - // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. - GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) - // GetCollectionByID retrieves a collection by ID and builds a response - GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) - // ExecuteScript handler sends the script from the request to be executed. - ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) - // GetAccount handler retrieves account by address and returns the response. - GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) - // GetEvents for the provided block range or list of block IDs filtered by type. - GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) - // GetNetworkParameters returns network-wide parameters of the blockchain - GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) - // GetNodeVersionInfo returns node version information - GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) -} - -// RequestHandler is a structure that represents local requests -type RequestHandler struct { - RestServerApi - log zerolog.Logger - backend access.API -} - -// NewRequestHandler returns new RequestHandler. -func NewRequestHandler(log zerolog.Logger, backend access.API) *RequestHandler { - return &RequestHandler{ - log: log, - backend: backend, - } -} - -// GetTransactionByID gets a transaction by requested ID. -func (h *RequestHandler) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, _ flow.Chain) (models.Transaction, error) { - var response models.Transaction - - tx, err := h.backend.GetTransaction(context, r.ID) - if err != nil { - return response, err - } - - var txr *access.TransactionResult - // only lookup result if transaction result is to be expanded - if r.ExpandsResult { - txr, err = h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) - if err != nil { - return response, err - } - } - - response.Build(tx, txr, link) - return response, nil -} - -// CreateTransaction creates a new transaction from provided payload. -func (h *RequestHandler) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - var response models.Transaction - - err := h.backend.SendTransaction(context, &r.Transaction) - if err != nil { - return response, err - } - - response.Build(&r.Transaction, nil, link) - return response, nil -} - -// GetTransactionResultByID retrieves transaction result by the transaction ID. -func (h *RequestHandler) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - var response models.TransactionResult - - txr, err := h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) - if err != nil { - return response, err - } - - response.Build(txr, r.ID, link) - return response, nil -} - -// GetBlocksByIDs gets blocks by provided ID or list of IDs. -func (h *RequestHandler) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - blocks := make([]*models.Block, len(r.IDs)) - - for i, id := range r.IDs { - block, err := getBlock(forID(&id), context, expandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil -} - -// GetBlocksByHeight gets blocks by provided height. -func (h *RequestHandler) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - req, err := r.GetBlockRequest() - if err != nil { - return nil, NewBadRequestError(err) - } - - if req.FinalHeight || req.SealedHeight { - block, err := getBlock(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - - return []*models.Block{block}, nil - } - - // if the query is /blocks/height=1000,1008,1049... - if req.HasHeights() { - blocks := make([]*models.Block, len(req.Heights)) - for i, height := range req.Heights { - block, err := getBlock(forHeight(height), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil - } - - // support providing end height as "sealed" or "final" - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - latest, _, err := h.backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - req.EndHeight = latest.Header.Height // overwrite special value height with fetched - - if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) - } - } - - blocks := make([]*models.Block, 0) - // start and end height inclusive - for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := getBlock(forHeight(i), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks = append(blocks, block) - } - - return blocks, nil -} - -// GetBlockPayloadByID gets block payload by ID -func (h *RequestHandler) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { - var payload models.BlockPayload - - blkProvider := NewBlockRequestProvider(h.backend, forID(&r.ID)) - blk, _, statusErr := blkProvider.getBlock(context) - if statusErr != nil { - return payload, statusErr - } - - err := payload.Build(blk.Payload) - if err != nil { - return payload, err - } - - return payload, nil -} - -// GetExecutionResultByID gets execution result by the ID. -func (h *RequestHandler) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - var response models.ExecutionResult - - res, err := h.backend.GetExecutionResultByID(context, r.ID) - if err != nil { - return response, err - } - - if res == nil { - err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) - return response, NewNotFoundError(err.Error(), err) - } - - err = response.Build(res, link) - if err != nil { - return response, err - } - - return response, nil -} - -// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func (h *RequestHandler) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - // for each block ID we retrieve execution result - results := make([]models.ExecutionResult, len(r.BlockIDs)) - for i, id := range r.BlockIDs { - res, err := h.backend.GetExecutionResultForBlockID(context, id) - if err != nil { - return nil, err - } - - var response models.ExecutionResult - err = response.Build(res, link) - if err != nil { - return nil, err - } - results[i] = response - } - - return results, nil -} - -// GetCollectionByID retrieves a collection by ID and builds a response -func (h *RequestHandler) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, _ flow.Chain) (models.Collection, error) { - var response models.Collection - - collection, err := h.backend.GetCollectionByID(context, r.ID) - if err != nil { - return response, err - } - - // if we expand transactions in the query retrieve each transaction data - transactions := make([]*flow.TransactionBody, 0) - if r.ExpandsTransactions { - for _, tid := range collection.Transactions { - tx, err := h.backend.GetTransaction(context, tid) - if err != nil { - return response, err - } - - transactions = append(transactions, tx) - } - } - - err = response.Build(collection, transactions, link, expandFields) - if err != nil { - return response, err - } - - return response, nil -} - -// ExecuteScript handler sends the script from the request to be executed. -func (h *RequestHandler) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { - if r.BlockID != flow.ZeroID { - return h.backend.ExecuteScriptAtBlockID(context, r.BlockID, r.Script.Source, r.Script.Args) - } - - // default to sealed height - if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { - return h.backend.ExecuteScriptAtLatestBlock(context, r.Script.Source, r.Script.Args) - } - - if r.BlockHeight == request.FinalHeight { - finalBlock, _, err := h.backend.GetLatestBlockHeader(context, false) - if err != nil { - return nil, err - } - r.BlockHeight = finalBlock.Height - } - - return h.backend.ExecuteScriptAtBlockHeight(context, r.BlockHeight, r.Script.Source, r.Script.Args) -} - -// GetAccount handler retrieves account by address and returns the response. -func (h *RequestHandler) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - var response models.Account - - // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it - if r.Height == request.FinalHeight || r.Height == request.SealedHeight { - header, _, err := h.backend.GetLatestBlockHeader(context, r.Height == request.SealedHeight) - if err != nil { - return response, err - } - r.Height = header.Height - } - - account, err := h.backend.GetAccountAtBlockHeight(context, r.Address, r.Height) - if err != nil { - return response, NewNotFoundError("not found account at block height", err) - } - - err = response.Build(account, link, expandFields) - return response, err -} - -// GetEvents for the provided block range or list of block IDs filtered by type. -func (h *RequestHandler) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { - // if the request has block IDs provided then return events for block IDs - var blocksEvents models.BlocksEvents - if len(r.BlockIDs) > 0 { - events, err := h.backend.GetEventsForBlockIDs(context, r.Type, r.BlockIDs) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil - } - - // if end height is provided with special values then load the height - if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { - latest, _, err := h.backend.GetLatestBlockHeader(context, r.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - r.EndHeight = latest.Height - // special check after we resolve special height value - if r.StartHeight > r.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) - } - } - - // if request provided block height range then return events for that range - events, err := h.backend.GetEventsForHeightRange(context, r.Type, r.StartHeight, r.EndHeight) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil -} - -// GetNetworkParameters returns network-wide parameters of the blockchain -func (h *RequestHandler) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { - params := h.backend.GetNetworkParameters(r.Context()) - - var response models.NetworkParameters - response.Build(¶ms) - return response, nil -} - -// GetNodeVersionInfo returns node version information -func (h *RequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { - var response models.NodeVersionInfo - - params, err := h.backend.GetNodeVersionInfo(r.Context()) - if err != nil { - return response, err - } - - response.Build(params) - return response, nil -} - -// RestForwarder handles the request forwarding to upstream -type RestForwarder struct { - log zerolog.Logger - *forwarder.Forwarder -} - -// NewRestForwarder returns new RestForwarder. -func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*RestForwarder, error) { - f, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) - - restForwarder := &RestForwarder{ - log: log, - Forwarder: f, - } - return restForwarder, err -} - -// GetTransactionByID gets a transaction by requested ID. -func (f *RestForwarder) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { - var response models.Transaction - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getTransactionRequest := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - } - transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) - if err != nil { - return response, err - } - - var transactionResultResponse *accessproto.TransactionResultResponse - // only lookup result if transaction result is to be expanded - if r.ExpandsResult { - getTransactionResultRequest := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - BlockId: r.BlockID[:], - CollectionId: r.CollectionID[:], - } - transactionResultResponse, err = upstream.GetTransactionResult(context, getTransactionResultRequest) - if err != nil { - return response, err - } - } - flowTransaction, err := convert.MessageToTransaction(transactionResponse.Transaction, chain) - if err != nil { - return response, err - } - - flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) - - response.Build(&flowTransaction, flowTransactionResult, link) - return response, nil -} - -// CreateTransaction creates a new transaction from provided payload. -func (f *RestForwarder) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - var response models.Transaction - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - entitiesTransaction := convert.TransactionToMessage(r.Transaction) - sendTransactionRequest := &accessproto.SendTransactionRequest{ - Transaction: entitiesTransaction, - } - - _, err = upstream.SendTransaction(context, sendTransactionRequest) - if err != nil { - return response, err - } - - response.Build(&r.Transaction, nil, link) - return response, nil -} - -// GetTransactionResultByID retrieves transaction result by the transaction ID. -func (f *RestForwarder) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - var response models.TransactionResult - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getTransactionResult := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - BlockId: r.BlockID[:], - CollectionId: r.CollectionID[:], - } - transactionResultResponse, err := upstream.GetTransactionResult(context, getTransactionResult) - if err != nil { - return response, err - } - - flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) - response.Build(flowTransactionResult, r.ID, link) - return response, nil -} - -// GetBlocksByIDs gets blocks by provided ID or list of IDs. -func (f *RestForwarder) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - blocks := make([]*models.Block, len(r.IDs)) - - upstream, err := f.FaultTolerantClient() - if err != nil { - return blocks, err - } - - for i, id := range r.IDs { - block, err := getBlockFromGrpc(forID(&id), context, expandFields, upstream, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil -} - -// GetBlocksByHeight gets blocks by provided height. -func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - req, err := r.GetBlockRequest() - if err != nil { - return nil, NewBadRequestError(err) - } - - upstream, err := f.FaultTolerantClient() - if err != nil { - return nil, err - } - - if req.FinalHeight || req.SealedHeight { - block, err := getBlockFromGrpc(forFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - - return []*models.Block{block}, nil - } - - // if the query is /blocks/height=1000,1008,1049... - if req.HasHeights() { - blocks := make([]*models.Block, len(req.Heights)) - for i, height := range req.Heights { - block, err := getBlockFromGrpc(forHeight(height), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil - } - - // support providing end height as "sealed" or "final" - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ - IsSealed: req.EndHeight == request.SealedHeight, - } - blockResponse, err := upstream.GetLatestBlock(r.Context(), getLatestBlockRequest) - if err != nil { - return nil, err - } - - req.EndHeight = blockResponse.Block.BlockHeader.Height // overwrite special value height with fetched - - if req.StartHeight > req.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) - } - } - - blocks := make([]*models.Block, 0) - // start and end height inclusive - for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := getBlockFromGrpc(forHeight(i), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - blocks = append(blocks, block) - } - - return blocks, nil -} - -// GetBlockPayloadByID gets block payload by ID -func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { - var payload models.BlockPayload - - upstream, err := f.FaultTolerantClient() - if err != nil { - return payload, err - } - - blkProvider := NewBlockFromGrpcProvider(upstream, forID(&r.ID)) - block, _, statusErr := blkProvider.getBlock(context) - if statusErr != nil { - return payload, statusErr - } - - flowPayload, err := convert.PayloadFromMessage(block) - if err != nil { - return payload, err - } - - err = payload.Build(flowPayload) - if err != nil { - return payload, err - } - - return payload, nil -} - -// GetExecutionResultByID gets execution result by the ID. -func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - var response models.ExecutionResult - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ - Id: r.ID[:], - } - - executionResultByIDResponse, err := upstream.GetExecutionResultByID(context, executionResultByIDRequest) - if err != nil { - return response, err - } - - if executionResultByIDResponse == nil { - err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) - return response, NewNotFoundError(err.Error(), err) - } - - flowExecResult, err := convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) - if err != nil { - return response, err - } - err = response.Build(flowExecResult, link) - if err != nil { - return response, err - } - - return response, nil -} - -// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func (f *RestForwarder) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - // for each block ID we retrieve execution result - results := make([]models.ExecutionResult, len(r.BlockIDs)) - - upstream, err := f.FaultTolerantClient() - if err != nil { - return results, err - } - - for i, id := range r.BlockIDs { - getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ - BlockId: id[:], - } - executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockID) - if err != nil { - return nil, err - } - - var response models.ExecutionResult - flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) - if err != nil { - return nil, err - } - err = response.Build(flowExecResult, link) - if err != nil { - return nil, err - } - results[i] = response - } - - return results, nil -} - -// GetCollectionByID retrieves a collection by ID and builds a response -func (f *RestForwarder) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { - var response models.Collection - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ - Id: r.ID[:], - } - - collectionResponse, err := upstream.GetCollectionByID(context, getCollectionByIDRequest) - if err != nil { - return response, err - } - - // if we expand transactions in the query retrieve each transaction data - transactions := make([]*entities.Transaction, 0) - if r.ExpandsTransactions { - for _, tid := range collectionResponse.Collection.TransactionIds { - getTransactionRequest := &accessproto.GetTransactionRequest{ - Id: tid, - } - transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) - if err != nil { - return response, err - } - - transactions = append(transactions, transactionResponse.Transaction) - } - } - - err = response.BuildFromGrpc(collectionResponse.Collection, transactions, link, expandFields, chain) - if err != nil { - return response, err - } - - return response, nil -} - -// ExecuteScript handler sends the script from the request to be executed. -func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { - upstream, err := f.FaultTolerantClient() - if err != nil { - return nil, err - } - - if r.BlockID != flow.ZeroID { - executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ - BlockId: r.BlockID[:], - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(context, executeScriptAtBlockIDRequest) - if err != nil { - return nil, err - } - return executeScriptAtBlockIDResponse.Value, nil - } - - // default to sealed height - if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { - executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(context, executeScriptAtLatestBlockRequest) - if err != nil { - return nil, err - } - return executeScriptAtLatestBlockResponse.Value, nil - } - - if r.BlockHeight == request.FinalHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: false, - } - getLatestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return nil, err - } - r.BlockHeight = getLatestBlockHeaderResponse.Block.Height - } - - executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ - BlockHeight: r.BlockHeight, - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(context, executeScriptAtBlockHeightRequest) - if err != nil { - return nil, err - } - return executeScriptAtBlockHeightResponse.Value, nil -} - -// GetAccount handler retrieves account by address and returns the response. -func (f *RestForwarder) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - var response models.Account - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it - if r.Height == request.FinalHeight || r.Height == request.SealedHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: r.Height == request.SealedHeight, - } - blockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return response, err - } - r.Height = blockHeaderResponse.Block.Height - } - getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ - Address: r.Address.Bytes(), - BlockHeight: r.Height, - } - - accountResponse, err := upstream.GetAccountAtBlockHeight(context, getAccountAtBlockHeightRequest) - if err != nil { - return response, NewNotFoundError("not found account at block height", err) - } - - flowAccount, err := convert.MessageToAccount(accountResponse.Account) - if err != nil { - return response, err - } - - err = response.Build(flowAccount, link, expandFields) - return response, err -} - -// GetEvents for the provided block range or list of block IDs filtered by type. -func (f *RestForwarder) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { - // if the request has block IDs provided then return events for block IDs - var blocksEvents models.BlocksEvents - - upstream, err := f.FaultTolerantClient() - if err != nil { - return blocksEvents, err - } - - if len(r.BlockIDs) > 0 { - var blockIds [][]byte - for _, id := range r.BlockIDs { - blockIds = append(blockIds, id[:]) - } - getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ - Type: r.Type, - BlockIds: blockIds, - } - eventsResponse, err := upstream.GetEventsForBlockIDs(context, getEventsForBlockIDsRequest) - if err != nil { - return nil, err - } - - blocksEvents.BuildFromGrpc(eventsResponse.Results) - - return blocksEvents, nil - } - - // if end height is provided with special values then load the height - if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: r.EndHeight == request.SealedHeight, - } - latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return nil, err - } - - r.EndHeight = latestBlockHeaderResponse.Block.Height - // special check after we resolve special height value - if r.StartHeight > r.EndHeight { - return nil, NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) - } - } - - // if request provided block height range then return events for that range - getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ - Type: r.Type, - StartHeight: r.StartHeight, - EndHeight: r.EndHeight, - } - eventsResponse, err := upstream.GetEventsForHeightRange(context, getEventsForHeightRangeRequest) - if err != nil { - return nil, err - } - - blocksEvents.BuildFromGrpc(eventsResponse.Results) - return blocksEvents, nil -} - -// GetNetworkParameters returns network-wide parameters of the blockchain -func (f *RestForwarder) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { - var response models.NetworkParameters - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getNetworkParametersRequest := &accessproto.GetNetworkParametersRequest{} - getNetworkParametersResponse, err := upstream.GetNetworkParameters(r.Context(), getNetworkParametersRequest) - if err != nil { - return response, err - } - response.BuildFromGrpc(getNetworkParametersResponse) - return response, nil -} - -// GetNodeVersionInfo returns node version information -func (f *RestForwarder) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { - var response models.NodeVersionInfo - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getNodeVersionInfoRequest := &accessproto.GetNodeVersionInfoRequest{} - getNodeVersionInfoResponse, err := upstream.GetNodeVersionInfo(r.Context(), getNodeVersionInfoRequest) - if err != nil { - return response, err - } - - response.BuildFromGrpc(getNodeVersionInfoResponse.Info) - return response, nil -} diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index 9bb04239b66..2253d56033c 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -6,13 +6,15 @@ import ( "github.com/gorilla/mux" "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/middleware" "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) -func newRouter(serverAPI RestServerApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { +func NewRouter(serverAPI api.RestServerApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { router := mux.NewRouter().StrictSlash(true) v1SubRouter := router.PathPrefix("/v1").Subrouter() @@ -46,70 +48,70 @@ var Routes = []route{{ Method: http.MethodGet, Pattern: "/transactions/{id}", Name: "getTransactionByID", - Handler: GetTransactionByID, + Handler: routes.GetTransactionByID, }, { Method: http.MethodPost, Pattern: "/transactions", Name: "createTransaction", - Handler: CreateTransaction, + Handler: routes.CreateTransaction, }, { Method: http.MethodGet, Pattern: "/transaction_results/{id}", Name: "getTransactionResultByID", - Handler: GetTransactionResultByID, + Handler: routes.GetTransactionResultByID, }, { Method: http.MethodGet, Pattern: "/blocks/{id}", Name: "getBlocksByIDs", - Handler: GetBlocksByIDs, + Handler: routes.GetBlocksByIDs, }, { Method: http.MethodGet, Pattern: "/blocks", Name: "getBlocksByHeight", - Handler: GetBlocksByHeight, + Handler: routes.GetBlocksByHeight, }, { Method: http.MethodGet, Pattern: "/blocks/{id}/payload", Name: "getBlockPayloadByID", - Handler: GetBlockPayloadByID, + Handler: routes.GetBlockPayloadByID, }, { Method: http.MethodGet, Pattern: "/execution_results/{id}", Name: "getExecutionResultByID", - Handler: GetExecutionResultByID, + Handler: routes.GetExecutionResultByID, }, { Method: http.MethodGet, Pattern: "/execution_results", Name: "getExecutionResultByBlockID", - Handler: GetExecutionResultsByBlockIDs, + Handler: routes.GetExecutionResultsByBlockIDs, }, { Method: http.MethodGet, Pattern: "/collections/{id}", Name: "getCollectionByID", - Handler: GetCollectionByID, + Handler: routes.GetCollectionByID, }, { Method: http.MethodPost, Pattern: "/scripts", Name: "executeScript", - Handler: ExecuteScript, + Handler: routes.ExecuteScript, }, { Method: http.MethodGet, Pattern: "/accounts/{address}", Name: "getAccount", - Handler: GetAccount, + Handler: routes.GetAccount, }, { Method: http.MethodGet, Pattern: "/events", Name: "getEvents", - Handler: GetEvents, + Handler: routes.GetEvents, }, { Method: http.MethodGet, Pattern: "/network/parameters", Name: "getNetworkParameters", - Handler: GetNetworkParameters, + Handler: routes.GetNetworkParameters, }, { Method: http.MethodGet, Pattern: "/node_version_info", Name: "getNodeVersionInfo", - Handler: GetNodeVersionInfo, + Handler: routes.GetNodeVersionInfo, }} diff --git a/engine/access/rest/accounts.go b/engine/access/rest/routes/accounts.go similarity index 59% rename from engine/access/rest/accounts.go rename to engine/access/rest/routes/accounts.go index a7e194de9e3..76ff9d7fcb5 100644 --- a/engine/access/rest/accounts.go +++ b/engine/access/rest/routes/accounts.go @@ -1,15 +1,16 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetAccount handler retrieves account by address and returns the response -func GetAccount(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetAccount(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetAccountRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetAccount(req, r.Context(), r.ExpandFields, link) diff --git a/engine/access/rest/blocks.go b/engine/access/rest/routes/blocks.go similarity index 61% rename from engine/access/rest/blocks.go rename to engine/access/rest/routes/blocks.go index 9adcf0fec14..b6676d7076d 100644 --- a/engine/access/rest/blocks.go +++ b/engine/access/rest/routes/blocks.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "context" @@ -9,9 +9,9 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -19,34 +19,34 @@ import ( ) // GetBlocksByIDs gets blocks by provided ID or list of IDs. -func GetBlocksByIDs(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByIDs(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockByIDsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetBlocksByIDs(req, r.Context(), r.ExpandFields, link) } // GetBlocksByHeight gets blocks by height. -func GetBlocksByHeight(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByHeight(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { return srv.GetBlocksByHeight(r, link) } // GetBlockPayloadByID gets block payload by ID -func GetBlockPayloadByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetBlockPayloadByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockPayloadRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetBlockPayloadByID(req, r.Context(), link) } -func getBlock(option blockRequestOption, context context.Context, expandFields map[string]bool, backend access.API, link models.LinkGenerator) (*models.Block, error) { +func GetBlock(option BlockRequestOption, context context.Context, expandFields map[string]bool, backend access.API, link models.LinkGenerator) (*models.Block, error) { // lookup block blkProvider := NewBlockRequestProvider(backend, option) - blk, blockStatus, err := blkProvider.getBlock(context) + blk, blockStatus, err := blkProvider.GetBlock(context) if err != nil { return nil, err } @@ -76,62 +76,6 @@ func getBlock(option blockRequestOption, context context.Context, expandFields m return &block, nil } -func getBlockFromGrpc(option blockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { - // lookup block - blkProvider := NewBlockFromGrpcProvider(upstream, option) - blk, blockStatus, err := blkProvider.getBlock(context) - if err != nil { - return nil, err - } - - // lookup execution result - // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) - var block models.Block - getExecutionResultForBlockIDRequest := &accessproto.GetExecutionResultForBlockIDRequest{ - BlockId: blk.Id, - } - - executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockIDRequest) - if err != nil { - return nil, err - } - - flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) - if err != nil { - return nil, err - } - - flowBlock, err := convert.MessageToBlock(blk) - if err != nil { - return nil, err - } - - flowBlockStatus, err := convert.MessagesToBlockStatus(blockStatus) - if err != nil { - return nil, err - } - - if err != nil { - // handle case where execution result is not yet available - if se, ok := status.FromError(err); ok { - if se.Code() == codes.NotFound { - err := block.Build(flowBlock, nil, link, flowBlockStatus, expandFields) - if err != nil { - return nil, err - } - return &block, nil - } - } - return nil, err - } - - err = block.Build(flowBlock, flowExecResult, link, flowBlockStatus, expandFields) - if err != nil { - return nil, err - } - return &block, nil -} - type blockRequest struct { id *flow.Identifier height uint64 @@ -139,20 +83,20 @@ type blockRequest struct { sealed bool } -type blockRequestOption func(blkRequest *blockRequest) +type BlockRequestOption func(blkRequest *blockRequest) -func forID(id *flow.Identifier) blockRequestOption { +func ForID(id *flow.Identifier) BlockRequestOption { return func(blockRequest *blockRequest) { blockRequest.id = id } } -func forHeight(height uint64) blockRequestOption { +func ForHeight(height uint64) BlockRequestOption { return func(blockRequest *blockRequest) { blockRequest.height = height } } -func forFinalized(queryParam uint64) blockRequestOption { +func ForFinalized(queryParam uint64) BlockRequestOption { return func(blockRequest *blockRequest) { switch queryParam { case request.SealedHeight: @@ -171,7 +115,7 @@ type blockRequestProvider struct { backend access.API } -func NewBlockRequestProvider(backend access.API, options ...blockRequestOption) *blockRequestProvider { +func NewBlockRequestProvider(backend access.API, options ...BlockRequestOption) *blockRequestProvider { blockRequestProvider := &blockRequestProvider{ backend: backend, } @@ -182,11 +126,11 @@ func NewBlockRequestProvider(backend access.API, options ...blockRequestOption) return blockRequestProvider } -func (blkProvider *blockRequestProvider) getBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { +func (blkProvider *blockRequestProvider) GetBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { if blkProvider.id != nil { blk, _, err := blkProvider.backend.GetBlockByID(ctx, *blkProvider.id) if err != nil { // unfortunately backend returns internal error status if not found - return nil, flow.BlockStatusUnknown, NewNotFoundError( + return nil, flow.BlockStatusUnknown, models.NewNotFoundError( fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err, ) } @@ -197,29 +141,29 @@ func (blkProvider *blockRequestProvider) getBlock(ctx context.Context) (*flow.Bl blk, status, err := blkProvider.backend.GetLatestBlock(ctx, blkProvider.sealed) if err != nil { // cannot be a 'not found' error since final and sealed block should always be found - return nil, flow.BlockStatusUnknown, NewRestError(http.StatusInternalServerError, "block lookup failed", err) + return nil, flow.BlockStatusUnknown, models.NewRestError(http.StatusInternalServerError, "block lookup failed", err) } return blk, status, nil } blk, status, err := blkProvider.backend.GetBlockByHeight(ctx, blkProvider.height) if err != nil { // unfortunately backend returns internal error status if not found - return nil, flow.BlockStatusUnknown, NewNotFoundError( + return nil, flow.BlockStatusUnknown, models.NewNotFoundError( fmt.Sprintf("error looking up block at height %d", blkProvider.height), err, ) } return blk, status, nil } -// blockFromGrpcProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to +// BlockFromGrpcProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to // look up a block or a block header either by ID or by height -type blockFromGrpcProvider struct { +type BlockFromGrpcProvider struct { blockRequest upstream accessproto.AccessAPIClient } -func NewBlockFromGrpcProvider(upstream accessproto.AccessAPIClient, options ...blockRequestOption) *blockFromGrpcProvider { - blockFromGrpcProvider := &blockFromGrpcProvider{ +func NewBlockFromGrpcProvider(upstream accessproto.AccessAPIClient, options ...BlockRequestOption) *BlockFromGrpcProvider { + blockFromGrpcProvider := &BlockFromGrpcProvider{ upstream: upstream, } @@ -229,14 +173,14 @@ func NewBlockFromGrpcProvider(upstream accessproto.AccessAPIClient, options ...b return blockFromGrpcProvider } -func (blkProvider *blockFromGrpcProvider) getBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { +func (blkProvider *BlockFromGrpcProvider) GetBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { if blkProvider.id != nil { getBlockByIdRequest := &accessproto.GetBlockByIDRequest{ Id: []byte(blkProvider.id.String()), } blockResponse, err := blkProvider.upstream.GetBlockByID(ctx, getBlockByIdRequest) if err != nil { // unfortunately grpc returns internal error status if not found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewNotFoundError( + return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewNotFoundError( fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err, ) } @@ -250,7 +194,7 @@ func (blkProvider *blockFromGrpcProvider) getBlock(ctx context.Context) (*entiti blockResponse, err := blkProvider.upstream.GetLatestBlock(ctx, getLatestBlockRequest) if err != nil { // cannot be a 'not found' error since final and sealed block should always be found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewRestError(http.StatusInternalServerError, "block lookup failed", err) + return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewRestError(http.StatusInternalServerError, "block lookup failed", err) } return blockResponse.Block, blockResponse.BlockStatus, nil } @@ -261,7 +205,7 @@ func (blkProvider *blockFromGrpcProvider) getBlock(ctx context.Context) (*entiti } blockResponse, err := blkProvider.upstream.GetBlockByHeight(ctx, getBlockByHeight) if err != nil { // unfortunately grpc returns internal error status if not found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, NewNotFoundError( + return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewNotFoundError( fmt.Sprintf("error looking up block at height %d", blkProvider.height), err, ) } diff --git a/engine/access/rest/collections.go b/engine/access/rest/routes/collections.go similarity index 59% rename from engine/access/rest/collections.go rename to engine/access/rest/routes/collections.go index be1b751b348..97a38005961 100644 --- a/engine/access/rest/collections.go +++ b/engine/access/rest/routes/collections.go @@ -1,15 +1,16 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetCollectionByID retrieves a collection by ID and builds a response -func GetCollectionByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetCollectionByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetCollectionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetCollectionByID(req, r.Context(), r.ExpandFields, link, r.Chain) diff --git a/engine/access/rest/events.go b/engine/access/rest/routes/events.go similarity index 51% rename from engine/access/rest/events.go rename to engine/access/rest/routes/events.go index e1e3eb6c7ec..fd728bc1188 100644 --- a/engine/access/rest/events.go +++ b/engine/access/rest/routes/events.go @@ -1,18 +1,19 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) -const blockQueryParam = "block_ids" -const eventTypeQuery = "type" +const BlockQueryParam = "block_ids" +const EventTypeQuery = "type" // GetEvents for the provided block range or list of block IDs filtered by type. -func GetEvents(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { +func GetEvents(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetEventsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetEvents(req, r.Context()) diff --git a/engine/access/rest/execution_result.go b/engine/access/rest/routes/execution_result.go similarity index 56% rename from engine/access/rest/execution_result.go rename to engine/access/rest/routes/execution_result.go index 1b9af6c586d..4c1ba8aab7e 100644 --- a/engine/access/rest/execution_result.go +++ b/engine/access/rest/routes/execution_result.go @@ -1,25 +1,26 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func GetExecutionResultsByBlockIDs(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultsByBlockIDs(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultByBlockIDsRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetExecutionResultsByBlockIDs(req, r.Context(), link) } // GetExecutionResultByID gets execution result by the ID. -func GetExecutionResultByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetExecutionResultByID(req, r.Context(), link) diff --git a/engine/access/rest/network.go b/engine/access/rest/routes/network.go similarity index 56% rename from engine/access/rest/network.go rename to engine/access/rest/routes/network.go index 1fe904e29dd..a409fd4b893 100644 --- a/engine/access/rest/network.go +++ b/engine/access/rest/routes/network.go @@ -1,11 +1,12 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNetworkParameters returns network-wide parameters of the blockchain -func GetNetworkParameters(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { +func GetNetworkParameters(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { return srv.GetNetworkParameters(r) } diff --git a/engine/access/rest/node_version_info.go b/engine/access/rest/routes/node_version_info.go similarity index 54% rename from engine/access/rest/node_version_info.go rename to engine/access/rest/routes/node_version_info.go index a1a04977835..14fbd8bbf26 100644 --- a/engine/access/rest/node_version_info.go +++ b/engine/access/rest/routes/node_version_info.go @@ -1,11 +1,12 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNodeVersionInfo returns node version information -func GetNodeVersionInfo(r *request.Request, srv RestServerApi, _ models.LinkGenerator) (interface{}, error) { +func GetNodeVersionInfo(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { return srv.GetNodeVersionInfo(r) } diff --git a/engine/access/rest/scripts.go b/engine/access/rest/routes/scripts.go similarity index 57% rename from engine/access/rest/scripts.go rename to engine/access/rest/routes/scripts.go index 827bd25b13a..1debbb07934 100644 --- a/engine/access/rest/scripts.go +++ b/engine/access/rest/routes/scripts.go @@ -1,15 +1,16 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // ExecuteScript handler sends the script from the request to be executed. -func ExecuteScript(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func ExecuteScript(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetScriptRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.ExecuteScript(req, r.Context(), link) diff --git a/engine/access/rest/transactions.go b/engine/access/rest/routes/transactions.go similarity index 55% rename from engine/access/rest/transactions.go rename to engine/access/rest/routes/transactions.go index 1acdccfa1e8..9f9f49d8cf1 100644 --- a/engine/access/rest/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -1,35 +1,36 @@ -package rest +package routes import ( + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetTransactionByID gets a transaction by requested ID. -func GetTransactionByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetTransactionByID(req, r.Context(), link, r.Chain) } // GetTransactionResultByID retrieves transaction result by the transaction ID. -func GetTransactionResultByID(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionResultByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionResultRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.GetTransactionResultByID(req, r.Context(), link) } // CreateTransaction creates a new transaction from provided payload. -func CreateTransaction(r *request.Request, srv RestServerApi, link models.LinkGenerator) (interface{}, error) { +func CreateTransaction(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { req, err := r.CreateTransactionRequest() if err != nil { - return nil, NewBadRequestError(err) + return nil, models.NewBadRequestError(err) } return srv.CreateTransaction(req, r.Context(), link) diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index f196fb1a4a9..9dc7112e6b6 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -7,13 +7,14 @@ import ( "github.com/rs/cors" "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) // NewServer returns an HTTP server initialized with the REST API handler -func NewServer(serverAPI RestServerApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { - router, err := newRouter(serverAPI, logger, chain, restCollector) +func NewServer(serverAPI api.RestServerApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { + router, err := NewRouter(serverAPI, logger, chain, restCollector) if err != nil { return nil, err } diff --git a/engine/access/rest/server_request_handler.go b/engine/access/rest/server_request_handler.go new file mode 100644 index 00000000000..12d45550705 --- /dev/null +++ b/engine/access/rest/server_request_handler.go @@ -0,0 +1,346 @@ +package rest + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/access/rest/routes" + "github.com/onflow/flow-go/model/flow" +) + +// ServerRequestHandler is a structure that represents local requests +type ServerRequestHandler struct { + log zerolog.Logger + backend access.API +} + +var _ api.RestServerApi = (*ServerRequestHandler)(nil) + +// NewServerRequestHandler returns new ServerRequestHandler. +func NewServerRequestHandler(log zerolog.Logger, backend access.API) *ServerRequestHandler { + return &ServerRequestHandler{ + log: log, + backend: backend, + } +} + +// GetTransactionByID gets a transaction by requested ID. +func (h *ServerRequestHandler) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, _ flow.Chain) (models.Transaction, error) { + var response models.Transaction + + tx, err := h.backend.GetTransaction(context, r.ID) + if err != nil { + return response, err + } + + var txr *access.TransactionResult + // only lookup result if transaction result is to be expanded + if r.ExpandsResult { + txr, err = h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) + if err != nil { + return response, err + } + } + + response.Build(tx, txr, link) + return response, nil +} + +// CreateTransaction creates a new transaction from provided payload. +func (h *ServerRequestHandler) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { + var response models.Transaction + + err := h.backend.SendTransaction(context, &r.Transaction) + if err != nil { + return response, err + } + + response.Build(&r.Transaction, nil, link) + return response, nil +} + +// GetTransactionResultByID retrieves transaction result by the transaction ID. +func (h *ServerRequestHandler) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { + var response models.TransactionResult + + txr, err := h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) + if err != nil { + return response, err + } + + response.Build(txr, r.ID, link) + return response, nil +} + +// GetBlocksByIDs gets blocks by provided ID or list of IDs. +func (h *ServerRequestHandler) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { + blocks := make([]*models.Block, len(r.IDs)) + + for i, id := range r.IDs { + block, err := routes.GetBlock(routes.ForID(&id), context, expandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil +} + +// GetBlocksByHeight gets blocks by provided height. +func (h *ServerRequestHandler) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { + req, err := r.GetBlockRequest() + if err != nil { + return nil, models.NewBadRequestError(err) + } + + if req.FinalHeight || req.SealedHeight { + block, err := routes.GetBlock(routes.ForFinalized(req.Heights[0]), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + + return []*models.Block{block}, nil + } + + // if the query is /blocks/height=1000,1008,1049... + if req.HasHeights() { + blocks := make([]*models.Block, len(req.Heights)) + for i, height := range req.Heights { + block, err := routes.GetBlock(routes.ForHeight(height), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil + } + + // support providing end height as "sealed" or "final" + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + latest, _, err := h.backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + req.EndHeight = latest.Header.Height // overwrite special value height with fetched + + if req.StartHeight > req.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + } + } + + blocks := make([]*models.Block, 0) + // start and end height inclusive + for i := req.StartHeight; i <= req.EndHeight; i++ { + block, err := routes.GetBlock(routes.ForHeight(i), r.Context(), r.ExpandFields, h.backend, link) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil +} + +// GetBlockPayloadByID gets block payload by ID +func (h *ServerRequestHandler) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { + var payload models.BlockPayload + + blkProvider := routes.NewBlockRequestProvider(h.backend, routes.ForID(&r.ID)) + blk, _, statusErr := blkProvider.GetBlock(context) + if statusErr != nil { + return payload, statusErr + } + + err := payload.Build(blk.Payload) + if err != nil { + return payload, err + } + + return payload, nil +} + +// GetExecutionResultByID gets execution result by the ID. +func (h *ServerRequestHandler) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { + var response models.ExecutionResult + + res, err := h.backend.GetExecutionResultByID(context, r.ID) + if err != nil { + return response, err + } + + if res == nil { + err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) + return response, models.NewNotFoundError(err.Error(), err) + } + + err = response.Build(res, link) + if err != nil { + return response, err + } + + return response, nil +} + +// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. +func (h *ServerRequestHandler) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { + // for each block ID we retrieve execution result + results := make([]models.ExecutionResult, len(r.BlockIDs)) + for i, id := range r.BlockIDs { + res, err := h.backend.GetExecutionResultForBlockID(context, id) + if err != nil { + return nil, err + } + + var response models.ExecutionResult + err = response.Build(res, link) + if err != nil { + return nil, err + } + results[i] = response + } + + return results, nil +} + +// GetCollectionByID retrieves a collection by ID and builds a response +func (h *ServerRequestHandler) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, _ flow.Chain) (models.Collection, error) { + var response models.Collection + + collection, err := h.backend.GetCollectionByID(context, r.ID) + if err != nil { + return response, err + } + + // if we expand transactions in the query retrieve each transaction data + transactions := make([]*flow.TransactionBody, 0) + if r.ExpandsTransactions { + for _, tid := range collection.Transactions { + tx, err := h.backend.GetTransaction(context, tid) + if err != nil { + return response, err + } + + transactions = append(transactions, tx) + } + } + + err = response.Build(collection, transactions, link, expandFields) + if err != nil { + return response, err + } + + return response, nil +} + +// ExecuteScript handler sends the script from the request to be executed. +func (h *ServerRequestHandler) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { + if r.BlockID != flow.ZeroID { + return h.backend.ExecuteScriptAtBlockID(context, r.BlockID, r.Script.Source, r.Script.Args) + } + + // default to sealed height + if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { + return h.backend.ExecuteScriptAtLatestBlock(context, r.Script.Source, r.Script.Args) + } + + if r.BlockHeight == request.FinalHeight { + finalBlock, _, err := h.backend.GetLatestBlockHeader(context, false) + if err != nil { + return nil, err + } + r.BlockHeight = finalBlock.Height + } + + return h.backend.ExecuteScriptAtBlockHeight(context, r.BlockHeight, r.Script.Source, r.Script.Args) +} + +// GetAccount handler retrieves account by address and returns the response. +func (h *ServerRequestHandler) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { + var response models.Account + + // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it + if r.Height == request.FinalHeight || r.Height == request.SealedHeight { + header, _, err := h.backend.GetLatestBlockHeader(context, r.Height == request.SealedHeight) + if err != nil { + return response, err + } + r.Height = header.Height + } + + account, err := h.backend.GetAccountAtBlockHeight(context, r.Address, r.Height) + if err != nil { + return response, models.NewNotFoundError("not found account at block height", err) + } + + err = response.Build(account, link, expandFields) + return response, err +} + +// GetEvents for the provided block range or list of block IDs filtered by type. +func (h *ServerRequestHandler) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { + // if the request has block IDs provided then return events for block IDs + var blocksEvents models.BlocksEvents + if len(r.BlockIDs) > 0 { + events, err := h.backend.GetEventsForBlockIDs(context, r.Type, r.BlockIDs) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil + } + + // if end height is provided with special values then load the height + if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { + latest, _, err := h.backend.GetLatestBlockHeader(context, r.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + r.EndHeight = latest.Height + // special check after we resolve special height value + if r.StartHeight > r.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + } + } + + // if request provided block height range then return events for that range + events, err := h.backend.GetEventsForHeightRange(context, r.Type, r.StartHeight, r.EndHeight) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil +} + +// GetNetworkParameters returns network-wide parameters of the blockchain +func (h *ServerRequestHandler) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { + params := h.backend.GetNetworkParameters(r.Context()) + + var response models.NetworkParameters + response.Build(¶ms) + return response, nil +} + +// GetNodeVersionInfo returns node version information +func (h *ServerRequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { + var response models.NodeVersionInfo + + params, err := h.backend.GetNodeVersionInfo(r.Context()) + if err != nil { + return response, err + } + + response.Build(params) + return response, nil +} diff --git a/engine/access/rest/accounts_test.go b/engine/access/rest/tests/accounts_test.go similarity index 99% rename from engine/access/rest/accounts_test.go rename to engine/access/rest/tests/accounts_test.go index d248bd312c8..222928bc846 100644 --- a/engine/access/rest/accounts_test.go +++ b/engine/access/rest/tests/accounts_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "fmt" diff --git a/engine/access/rest/blocks_test.go b/engine/access/rest/tests/blocks_test.go similarity index 99% rename from engine/access/rest/blocks_test.go rename to engine/access/rest/tests/blocks_test.go index 7e337e07e2a..f2cab63c3c0 100644 --- a/engine/access/rest/blocks_test.go +++ b/engine/access/rest/tests/blocks_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "fmt" diff --git a/engine/access/rest/collections_test.go b/engine/access/rest/tests/collections_test.go similarity index 99% rename from engine/access/rest/collections_test.go rename to engine/access/rest/tests/collections_test.go index a033fb8a453..77a97957ae8 100644 --- a/engine/access/rest/collections_test.go +++ b/engine/access/rest/tests/collections_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "encoding/json" diff --git a/engine/access/rest/events_test.go b/engine/access/rest/tests/events_test.go similarity index 98% rename from engine/access/rest/events_test.go rename to engine/access/rest/tests/events_test.go index c5b9be0c00b..806b7860b96 100644 --- a/engine/access/rest/events_test.go +++ b/engine/access/rest/tests/events_test.go @@ -1,24 +1,26 @@ -package rest +package tests import ( "encoding/json" "fmt" + "net/http" "net/url" "strings" "testing" "time" - "github.com/onflow/flow-go/engine/access/rest/util" - - "github.com/onflow/flow-go/access/mock" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/utils/unittest" - mocks "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/engine/access/rest/routes" + "github.com/onflow/flow-go/engine/access/rest/util" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) func TestGetEvents(t *testing.T) { @@ -137,7 +139,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q := u.Query() if len(blockIDs) > 0 { - q.Add(blockQueryParam, strings.Join(blockIDs, ",")) + q.Add(routes.BlockQueryParam, strings.Join(blockIDs, ",")) } if start != "" && end != "" { @@ -145,7 +147,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q.Add(endHeightQueryParam, end) } - q.Add(eventTypeQuery, eventType) + q.Add(routes.EventTypeQuery, eventType) u.RawQuery = q.Encode() diff --git a/engine/access/rest/execution_result_test.go b/engine/access/rest/tests/execution_result_test.go similarity index 99% rename from engine/access/rest/execution_result_test.go rename to engine/access/rest/tests/execution_result_test.go index 241f14f6b59..f978f9bd299 100644 --- a/engine/access/rest/execution_result_test.go +++ b/engine/access/rest/tests/execution_result_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "fmt" diff --git a/engine/access/rest/network_test.go b/engine/access/rest/tests/network_test.go similarity index 98% rename from engine/access/rest/network_test.go rename to engine/access/rest/tests/network_test.go index f8013c86685..4af5b501c06 100644 --- a/engine/access/rest/network_test.go +++ b/engine/access/rest/tests/network_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "fmt" diff --git a/engine/access/rest/node_version_info_test.go b/engine/access/rest/tests/node_version_info_test.go similarity index 99% rename from engine/access/rest/node_version_info_test.go rename to engine/access/rest/tests/node_version_info_test.go index b2543950ea0..a4935a29438 100644 --- a/engine/access/rest/node_version_info_test.go +++ b/engine/access/rest/tests/node_version_info_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "fmt" diff --git a/engine/access/rest/scripts_test.go b/engine/access/rest/tests/scripts_test.go similarity index 99% rename from engine/access/rest/scripts_test.go rename to engine/access/rest/tests/scripts_test.go index 8cb19415ae3..0c93106b38a 100644 --- a/engine/access/rest/scripts_test.go +++ b/engine/access/rest/tests/scripts_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "bytes" diff --git a/engine/access/rest/test_helpers.go b/engine/access/rest/tests/test_helpers.go similarity index 68% rename from engine/access/rest/test_helpers.go rename to engine/access/rest/tests/test_helpers.go index 8ce8c2f50d2..4ce4fbc2f50 100644 --- a/engine/access/rest/test_helpers.go +++ b/engine/access/rest/tests/test_helpers.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "bytes" @@ -12,6 +12,9 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/access/mock" + "github.com/onflow/flow-go/engine/access/rest" + "github.com/onflow/flow-go/engine/access/rest/api" + restproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" @@ -27,11 +30,11 @@ const ( heightQueryParam = "height" ) -func executeRequest(req *http.Request, restHandler RestServerApi) (*httptest.ResponseRecorder, error) { +func executeRequest(req *http.Request, restHandler api.RestServerApi) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - router, err := newRouter(restHandler, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) + router, err := rest.NewRouter(restHandler, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) if err != nil { return nil, err } @@ -41,31 +44,31 @@ func executeRequest(req *http.Request, restHandler RestServerApi) (*httptest.Res return rr, nil } -func newAccessRestHandler(backend *mock.API) RestServerApi { +func newAccessRestHandler(backend *mock.API) api.RestServerApi { var b bytes.Buffer logger := zerolog.New(&b) - return NewRequestHandler(logger, backend) + return rest.NewServerRequestHandler(logger, backend) } -func newObserverRestHandler(backend *mock.API, restForwarder *restmock.RestServerApi) (RestServerApi, error) { +func newObserverRestHandler(backend *mock.API, restForwarder *restmock.RestServerApi) (api.RestServerApi, error) { var b bytes.Buffer logger := zerolog.New(&b) observerCollector := metrics.NewNoopCollector() - return &RestRouter{ + return &restproxy.RestRouter{ Logger: logger, Metrics: observerCollector, Upstream: restForwarder, - Observer: NewRequestHandler(logger, backend), + Observer: rest.NewServerRequestHandler(logger, backend), }, nil } -func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, restHandler RestServerApi) { +func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, restHandler api.RestServerApi) { assertResponse(t, req, http.StatusOK, expectedRespBody, restHandler) } -func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, restHandler RestServerApi) { +func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, restHandler api.RestServerApi) { rr, err := executeRequest(req, restHandler) assert.NoError(t, err) actualResponseBody := rr.Body.String() diff --git a/engine/access/rest/transactions_test.go b/engine/access/rest/tests/transactions_test.go similarity index 99% rename from engine/access/rest/transactions_test.go rename to engine/access/rest/tests/transactions_test.go index 36a0cfe9885..f4570d3a191 100644 --- a/engine/access/rest/transactions_test.go +++ b/engine/access/rest/tests/transactions_test.go @@ -1,4 +1,4 @@ -package rest +package tests import ( "bytes" diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 7b418fd33ee..7e099f00bf3 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -21,6 +21,7 @@ import ( accessmock "github.com/onflow/flow-go/engine/access/mock" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/engine/access/rest/tests" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" @@ -396,13 +397,13 @@ func assertError(t *testing.T, resp *http.Response, err error, expectedCode int, func optionsForBlockByID() *restclient.BlocksApiBlocksIdGetOpts { return &restclient.BlocksApiBlocksIdGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id"}), } } func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), StartHeight: optional.NewInterface(startHeight), EndHeight: optional.NewInterface(endHeight), @@ -411,7 +412,7 @@ func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient. func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(heights), } @@ -419,7 +420,7 @@ func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOp func optionsForFinalizedBlock(finalOrSealed string) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{rest.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(finalOrSealed), } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index e30e7d7a405..42b4aec9021 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/access/rest" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" @@ -68,7 +69,7 @@ type Engine struct { secureGrpcAddress net.Addr restAPIAddress net.Addr - restHandler rest.RestServerApi + restHandler api.RestServerApi } type Option func(*RPCEngineBuilder) diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index fbd925cee68..79db9853c29 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -9,6 +9,7 @@ import ( legacyaccess "github.com/onflow/flow-go/access/legacy" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/engine/access/rest" + restapi "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/module" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -39,7 +40,7 @@ func (builder *RPCEngineBuilder) RpcHandler() accessproto.AccessAPIServer { return builder.rpcHandler } -func (builder *RPCEngineBuilder) RestHandler() rest.RestServerApi { +func (builder *RPCEngineBuilder) RestHandler() restapi.RestServerApi { return builder.restHandler } @@ -69,7 +70,7 @@ func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPISer } // WithRestHandler specifies that the given `RestServerApi` should be used for REST. -func (builder *RPCEngineBuilder) WithRestHandler(handler rest.RestServerApi) *RPCEngineBuilder { +func (builder *RPCEngineBuilder) WithRestHandler(handler restapi.RestServerApi) *RPCEngineBuilder { builder.restHandler = handler return builder } @@ -116,7 +117,7 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { restHandler := builder.Engine.restHandler if restHandler == nil { - restHandler = rest.NewRequestHandler(builder.log, builder.backend) + restHandler = rest.NewServerRequestHandler(builder.log, builder.backend) } builder.Engine.restHandler = restHandler From 2db0debcba19a95d0eb1a00487e7bba4d054bb31 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 23 Jun 2023 18:05:22 +0300 Subject: [PATCH 058/169] Added more comments --- engine/access/rest/apiproxy/forwarder.go | 2 +- engine/access/rest/apiproxy/router.go | 2 +- engine/access/rest/tests/accounts_test.go | 17 ++++++++++++++++- engine/access/rest/tests/blocks_test.go | 21 ++++++++++++++++++--- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/engine/access/rest/apiproxy/forwarder.go b/engine/access/rest/apiproxy/forwarder.go index 6a62e302973..d5b65102196 100644 --- a/engine/access/rest/apiproxy/forwarder.go +++ b/engine/access/rest/apiproxy/forwarder.go @@ -23,7 +23,7 @@ import ( "github.com/onflow/flow/protobuf/go/flow/entities" ) -// RestForwarder handles the request forwarding to upstream +// RestForwarder - structure which handles requests to an upstream access node using gRPC API. type RestForwarder struct { log zerolog.Logger *forwarder.Forwarder diff --git a/engine/access/rest/apiproxy/router.go b/engine/access/rest/apiproxy/router.go index beb74b6ca41..b3a3e8db086 100644 --- a/engine/access/rest/apiproxy/router.go +++ b/engine/access/rest/apiproxy/router.go @@ -15,7 +15,7 @@ import ( ) // RestRouter is a structure that represents the routing proxy algorithm for observer node. -// It splits requests between a local requests and request forwarding to upstream service. +// It splits requests between a local requests and forward requests which can't be handled locally to an upstream access node. type RestRouter struct { Logger zerolog.Logger Metrics metrics.ObserverMetrics diff --git a/engine/access/rest/tests/accounts_test.go b/engine/access/rest/tests/accounts_test.go index 222928bc846..c9a247f8c15 100644 --- a/engine/access/rest/tests/accounts_test.go +++ b/engine/access/rest/tests/accounts_test.go @@ -36,6 +36,14 @@ func accountURL(t *testing.T, address string, height string) string { return u.String() } +// TestAccessGetAccount tests local getAccount request. +// +// Runs the following tests: +// 1. Get account by address at latest sealed block. +// 2. Get account by address at latest finalized block. +// 3. Get account by address at height. +// 4. Get account by address at height condensed. +// 5. Get invalid account. func TestAccessGetAccount(t *testing.T) { backend := &mock.API{} restHandler := newAccessRestHandler(backend) @@ -131,7 +139,14 @@ func TestAccessGetAccount(t *testing.T) { }) } -// TestObserverGetAccount tests the get account from observer node +// TestObserverGetAccount tests the get account request forwarding to an upstream. +// +// Runs the following tests: +// 1. Get account by address at latest sealed block. +// 2. Get account by address at latest finalized block. +// 3. Get account by address at height. +// 4. Get account by address at height condensed. +// 5. Get invalid account. func TestObserverGetAccount(t *testing.T) { backend := &mock.API{} restForwarder := &restmock.RestServerApi{} diff --git a/engine/access/rest/tests/blocks_test.go b/engine/access/rest/tests/blocks_test.go index f2cab63c3c0..0b8fd2c66c5 100644 --- a/engine/access/rest/tests/blocks_test.go +++ b/engine/access/rest/tests/blocks_test.go @@ -141,7 +141,7 @@ func prepareTestVectors(t *testing.T, return testVectors } -// TestGetBlocks tests the get blocks by ID and get blocks by heights API from access node +// TestGetBlocks tests local get blocks by ID and get blocks by heights API func TestAccessGetBlocks(t *testing.T) { backend := &mock.API{} @@ -159,12 +159,27 @@ func TestAccessGetBlocks(t *testing.T) { } } -// TestObserverGetBlocks tests the get blocks by ID and get blocks by heights API from observer node +// TestObserverGetBlocks tests requests forwarding for get blocks by ID and get blocks by heights. +// +// Check the following cases: +// 1. Get single expanded block by ID. +// 2. Get multiple expanded blocks by IDs +// 3. Get single condensed block by ID. +// 4. Get multiple condensed blocks by IDs. +// 5. Get single expanded block by height. +// 6. Get multiple expanded blocks by heights. +// 7. Get multiple expanded blocks by start and end height. +// 8. Get block by ID not found. +// 9. Get block by height not found. +// 10. Get block by end height less than start height. +// 11. Get block by both heights and start and end height. +// 12. Get block with missing height param. +// 13. Get block with missing height values. +// 14. Get block by more than maximum permissible number of IDs. func TestObserverGetBlocks(t *testing.T) { backend := &mock.API{} restForwarder := &restmock.RestServerApi{} - // Bring up upstream server blkCnt := 10 blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) From 264f86ff1b6332738f19130b19f24cd6eb82b47d Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Sat, 24 Jun 2023 00:03:31 +0300 Subject: [PATCH 059/169] Updated rest README --- engine/access/rest/README.md | 19 +++++++++++++------ engine/access/rest/server_request_handler.go | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/engine/access/rest/README.md b/engine/access/rest/README.md index fd7b970493d..a972fe8c385 100644 --- a/engine/access/rest/README.md +++ b/engine/access/rest/README.md @@ -6,10 +6,15 @@ available on our [docs site](https://docs.onflow.org/http-api/). ## Packages -- `rest`: The HTTP handlers for all the request, server generator and the select filter. +- `rest`: The HTTP handlers for the server generator and the select filter, implementation of handling local requests. - `middleware`: The common [middlewares](https://github.com/gorilla/mux#middleware) that all request pass through. - `models`: The generated models using openapi generators and implementation of model builders. - `request`: Implementation of API requests that provide validation for input data and build request models. +- `routes`: The common HTTP handlers for all the requests. +- `api`: The server API interface for REST service. +- `apiproxy`: Implementation of proxy router which splits requests for observer node between local and request +forwarding to upstream, implementation of handling request forwarding to an upstream access node using gRPC API. +- `tests`: Test for each request. ## Request lifecycle @@ -37,17 +42,19 @@ make generate-openapi ### Adding New API Endpoints -A new endpoint can be added by first implementing a new request handler, a request handle is a function in the rest +A new endpoint can be added by first implementing a new request handler, a request handle is a function in the routes package that complies with function interfaced defined as: ```go type ApiHandlerFunc func ( r *request.Request, -backend access.API, +srv api.RestServerApi, generator models.LinkGenerator, ) (interface{}, error) ``` -That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Adding a -new API endpoint also requires for a new request builder to be implemented and added in request package. Make sure to -not forget about adding tests for each of the API handler. +That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Then needs +to be added new request to `RestServerApi` interface and implemented it for local API service to the `RequestHandler` and for +request forwarding to the `RestForwarder`. After that new function needs to be added to the `RestRouter` for representing +the routing proxy algorithm. Adding a new API endpoint also requires for a new request builder to be implemented and added +in request package. Make sure to not forget about adding tests for each of the API handler. diff --git a/engine/access/rest/server_request_handler.go b/engine/access/rest/server_request_handler.go index 12d45550705..257c9f82deb 100644 --- a/engine/access/rest/server_request_handler.go +++ b/engine/access/rest/server_request_handler.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -// ServerRequestHandler is a structure that represents local requests +// ServerRequestHandler is a structure that represents handling local requests. type ServerRequestHandler struct { log zerolog.Logger backend access.API From e3447dc99bda9bffaed4819de36bde28c2b1d24c Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 23 Jun 2023 17:59:05 -0600 Subject: [PATCH 060/169] add exclude-dir so that GNU grep matches BSD grep for go-math-rand-check --- Makefile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 9dea7b94165..87ed5f5f944 100644 --- a/Makefile +++ b/Makefile @@ -94,11 +94,10 @@ go-math-rand-check: # If this check fails, try updating your code by using: # - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness # - "flow-go/crypto/random" for deterministic randomness - grep --include=\*.go --exclude={*test*,*helper*,*example*,*fixture*,*benchmark*,*profiler*} -rnw '"math/rand"'; \ - if [ $$? -ne 1 ]; \ - then \ - echo "[Error] Go production code should not use math/rand package"; \ - exit 1; \ + grep --include=\*.go --exclude=*{test,helper,example,fixture,benchmark,profiler}* \ + --exclude-dir=*{test,helper,example,fixture,benchmark,profiler}* -rnw '"math/rand"'; \ + if [ $$? -ne 1 ]; then \ + echo "[Error] Go production code should not use math/rand package"; exit 1; \ fi .PHONY: code-sanity-check From 8d6767eaf7b0a88f1f97ce0f8217f7bed7a65a28 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Sun, 25 Jun 2023 01:40:53 -0600 Subject: [PATCH 061/169] use protocol state in CompleteExecutionReceiptChainFixture --- .../assigner/blockconsumer/consumer_test.go | 2 +- engine/verification/utils/unittest/fixture.go | 41 +++++++++++++------ engine/verification/utils/unittest/helper.go | 2 +- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/engine/verification/assigner/blockconsumer/consumer_test.go b/engine/verification/assigner/blockconsumer/consumer_test.go index f57bc98ae26..e049ba7c662 100644 --- a/engine/verification/assigner/blockconsumer/consumer_test.go +++ b/engine/verification/assigner/blockconsumer/consumer_test.go @@ -149,7 +149,7 @@ func withConsumer( root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, vertestutils.WithClusterCommittee(clusterCommittee)) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, s.State, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) // makes sure that we generated a block chain of requested length. require.Len(t, blocks, blockCount) diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index c6166a6af7f..5b1a922098b 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/engine/execution/computation/committer" "github.com/onflow/flow-go/engine/execution/computation/computer" - "github.com/onflow/flow-go/engine/execution/state" + exstate "github.com/onflow/flow-go/engine/execution/state" "github.com/onflow/flow-go/engine/execution/state/bootstrap" "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm" @@ -25,6 +25,7 @@ import ( "github.com/onflow/flow-go/module/epochs" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/state/cluster" + "github.com/onflow/flow-go/state/protocol" envMock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/model/flow" @@ -189,8 +190,13 @@ func WithClusterCommittee(clusterCommittee flow.IdentityList) CompleteExecutionR // ExecutionResultFixture is a test helper that returns an execution result for the reference block header as well as the execution receipt data // for that result. -func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refBlkHeader *flow.Header, clusterCommittee flow.IdentityList) (*flow.ExecutionResult, - *ExecutionReceiptData) { +func ExecutionResultFixture(t *testing.T, + chunkCount int, + chain flow.Chain, + refBlkHeader *flow.Header, + clusterCommittee flow.IdentityList, + state protocol.ParticipantState, +) (*flow.ExecutionResult, *ExecutionReceiptData) { // setups up the first collection of block consists of three transactions tx1 := testutil.DeployCounterContractTransaction(chain.ServiceAddress(), chain) @@ -262,7 +268,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB ) // create state.View - snapshot := state.NewLedgerStorageSnapshot( + snapshot := exstate.NewLedgerStorageSnapshot( led, startStateCommitment) committer := committer.NewLedgerViewCommitter(led, trace.NewNoopTracer()) @@ -295,7 +301,7 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB me, prov, nil, - testutil.ProtocolStateFixture(), + state, testMaxConcurrency) require.NoError(t, err) @@ -368,7 +374,12 @@ func ExecutionResultFixture(t *testing.T, chunkCount int, chain flow.Chain, refB // For sake of simplicity and test, container blocks (i.e., C) do not contain any guarantee. // // It returns a slice of complete execution receipt fixtures that contains a container block as well as all data to verify its contained receipts. -func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count int, opts ...CompleteExecutionReceiptBuilderOpt) []*CompleteExecutionReceipt { +func CompleteExecutionReceiptChainFixture(t *testing.T, + root *flow.Header, + count int, + state protocol.ParticipantState, + opts ...CompleteExecutionReceiptBuilderOpt, +) []*CompleteExecutionReceipt { completeERs := make([]*CompleteExecutionReceipt, 0, count) parent := root @@ -393,7 +404,7 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count for i := 0; i < count; i++ { // Generates two blocks as parent <- R <- C where R is a reference block containing guarantees, // and C is a container block containing execution receipt for R. - receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder) + receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder, state) containerBlock := ContainerBlockFixture(head, receipts) completeERs = append(completeERs, &CompleteExecutionReceipt{ ContainerBlock: containerBlock, @@ -412,7 +423,10 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count // result (i.e., for the next result). // // Each result may appear in more than one receipt depending on the builder parameters. -func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder) ( +func ExecutionReceiptsFromParentBlockFixture(t *testing.T, + parent *flow.Header, + builder *CompleteExecutionReceiptBuilder, + state protocol.ParticipantState) ( []*flow.ExecutionReceipt, []*ExecutionReceiptData, *flow.Header) { @@ -420,7 +434,7 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, allReceipts := make([]*flow.ExecutionReceipt, 0, builder.resultsCount*builder.executorCount) for i := 0; i < builder.resultsCount; i++ { - result, data := ExecutionResultFromParentBlockFixture(t, parent, builder) + result, data := ExecutionResultFromParentBlockFixture(t, parent, builder, state) // makes several copies of the same result for cp := 0; cp < builder.executorCount; cp++ { @@ -438,10 +452,13 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, } // ExecutionResultFromParentBlockFixture is a test helper that creates a child (reference) block from the parent, as well as an execution for it. -func ExecutionResultFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder) (*flow.ExecutionResult, - *ExecutionReceiptData) { +func ExecutionResultFromParentBlockFixture(t *testing.T, + parent *flow.Header, + builder *CompleteExecutionReceiptBuilder, + state protocol.ParticipantState, +) (*flow.ExecutionResult, *ExecutionReceiptData) { refBlkHeader := unittest.BlockHeaderWithParentFixture(parent) - return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee) + return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee, state) } // ContainerBlockFixture builds and returns a block that contains input execution receipts. diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index e2bad5768d8..6636e24fbc3 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -494,7 +494,7 @@ func withConsumers(t *testing.T, builder.clusterCommittee = participants.Filter(filter.HasRole(flow.RoleCollection)) }) - completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, ops...) + completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, s.State, ops...) blocks := ExtendStateWithFinalizedBlocks(t, completeERs, s.State) // chunk assignment From 12f69d2674077e2cc0d3c8c53cae9b9b070c2b66 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 26 Jun 2023 15:52:41 +0300 Subject: [PATCH 062/169] Moved forwarder package to engine/common/grpc/forwarder, added constructor for backend cache, updated tests --- .../node_builder/access_node_builder.go | 37 +++++++++--- cmd/observer/node_builder/observer_builder.go | 38 +++++++++--- .../export_report.json | 6 -- engine/access/apiproxy/access_api_proxy.go | 2 +- .../access/apiproxy/access_api_proxy_test.go | 2 +- engine/access/rest/apiproxy/forwarder.go | 2 +- engine/access/rest_api_test.go | 15 +++-- engine/access/rpc/backend/backend.go | 58 ++----------------- engine/access/rpc/rate_limit_test.go | 13 +++-- engine/access/secure_grpcr_test.go | 14 ++--- .../common/grpc}/forwarder/forwarder.go | 0 11 files changed, 88 insertions(+), 99 deletions(-) delete mode 100644 cmd/util/cmd/execution-state-extract/export_report.json rename {module => engine/common/grpc}/forwarder/forwarder.go (100%) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 6222c1d4b53..fc69272d16a 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -991,8 +991,29 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { config := builder.rpcConf + backendConfig := config.BackendConfig + accessMetrics := builder.AccessMetrics - backend, err := backend.NewBackend(node.Logger, + cache, cacheSize, err := backend.NewCache(node.Logger, + accessMetrics, + backendConfig.ConnectionPoolSize) + if err != nil { + return nil, fmt.Errorf("could not initialize backend cache: %w", err) + } + + connFactory := &backend.ConnectionFactoryImpl{ + CollectionGRPCPort: builder.collectionGRPCPort, + ExecutionGRPCPort: builder.executionGRPCPort, + CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, + ConnectionsCache: cache, + CacheSize: cacheSize, + MaxMsgSize: config.MaxMsgSize, + AccessMetrics: accessMetrics, + Log: node.Logger, + } + + backend := backend.New( node.State, builder.CollectionRPC, builder.HistoricalAccessRPCs, @@ -1004,14 +1025,14 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { node.Storage.Results, node.RootChainID, builder.AccessMetrics, - builder.collectionGRPCPort, - builder.executionGRPCPort, + connFactory, builder.retryEnabled, - config.MaxMsgSize, - config.BackendConfig) - if err != nil { - return nil, fmt.Errorf("could not initialize backend: %w", err) - } + backendConfig.MaxHeightRange, + backendConfig.PreferredExecutionNodeIDs, + backendConfig.FixedExecutionNodeIDs, + node.Logger, + backend.DefaultSnapshotHistoryLimit, + backendConfig.ArchiveAddressList) engineBuilder, err := rpc.NewBuilder( node.Logger, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 1920aec0106..7a93d9e2afa 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -855,7 +855,28 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { accessMetrics := metrics.NewNoopCollector() config := builder.rpcConf - accessBackend, err := backend.NewBackend(node.Logger, + backendConfig := config.BackendConfig + + cache, cacheSize, err := backend.NewCache(node.Logger, + accessMetrics, + config.BackendConfig.ConnectionPoolSize) + if err != nil { + return nil, fmt.Errorf("could not initialize backend cache: %w", err) + } + + connFactory := &backend.ConnectionFactoryImpl{ + CollectionGRPCPort: 0, + ExecutionGRPCPort: 0, + CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, + ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, + ConnectionsCache: cache, + CacheSize: cacheSize, + MaxMsgSize: config.MaxMsgSize, + AccessMetrics: accessMetrics, + Log: node.Logger, + } + + accessBackend := backend.New( node.State, nil, nil, @@ -867,15 +888,14 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { node.Storage.Results, node.RootChainID, accessMetrics, - 0, - 0, + connFactory, false, - config.MaxMsgSize, - config.BackendConfig) - - if err != nil { - return nil, fmt.Errorf("could not initialize backend: %w", err) - } + backendConfig.MaxHeightRange, + backendConfig.PreferredExecutionNodeIDs, + backendConfig.FixedExecutionNodeIDs, + node.Logger, + backend.DefaultSnapshotHistoryLimit, + backendConfig.ArchiveAddressList) engineBuilder, err := rpc.NewBuilder( node.Logger, diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json deleted file mode 100644 index 3c4a27478db..00000000000 --- a/cmd/util/cmd/execution-state-extract/export_report.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "EpochCounter": 0, - "PreviousStateCommitment": "8536ee2769a5b35be123a54e45a23d2eaf3fa9f3df3bde6a713c87c286b9ec40", - "CurrentStateCommitment": "8536ee2769a5b35be123a54e45a23d2eaf3fa9f3df3bde6a713c87c286b9ec40", - "ReportSucceeded": true -} \ No newline at end of file diff --git a/engine/access/apiproxy/access_api_proxy.go b/engine/access/apiproxy/access_api_proxy.go index 2b345420229..f5898686fc6 100644 --- a/engine/access/apiproxy/access_api_proxy.go +++ b/engine/access/apiproxy/access_api_proxy.go @@ -9,9 +9,9 @@ import ( "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/engine/protocol" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/forwarder" "github.com/onflow/flow-go/module/metrics" ) diff --git a/engine/access/apiproxy/access_api_proxy_test.go b/engine/access/apiproxy/access_api_proxy_test.go index f8f2dce72e4..d20c5ee705d 100644 --- a/engine/access/apiproxy/access_api_proxy_test.go +++ b/engine/access/apiproxy/access_api_proxy_test.go @@ -11,8 +11,8 @@ import ( "google.golang.org/grpc" grpcinsecure "google.golang.org/grpc/credentials/insecure" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/forwarder" "github.com/onflow/flow-go/utils/grpcutils" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/engine/access/rest/apiproxy/forwarder.go b/engine/access/rest/apiproxy/forwarder.go index d5b65102196..164db705b47 100644 --- a/engine/access/rest/apiproxy/forwarder.go +++ b/engine/access/rest/apiproxy/forwarder.go @@ -15,9 +15,9 @@ import ( "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/routes" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/forwarder" accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 7e099f00bf3..61c4f75de0b 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -120,8 +120,7 @@ func (suite *RestAPITestSuite) SetupTest() { RESTListenAddr: unittest.DefaultAddress, } - backend, err := backend.NewBackend( - suite.log, + backend := backend.New( suite.state, suite.collClient, nil, @@ -133,14 +132,14 @@ func (suite *RestAPITestSuite) SetupTest() { suite.executionResults, suite.chainID, suite.metrics, - 0, - 0, + nil, false, 0, - config.BackendConfig, - ) - - require.NoError(suite.T(), err) + nil, + nil, + suite.log, + 0, + nil) rpcEngBuilder, err := rpc.NewBuilder( suite.log, diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index d5a29d04adb..97d23f9a5ab 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -201,28 +201,14 @@ func New( return b } -func NewBackend( +func NewCache( log zerolog.Logger, - state protocol.State, - collectionRPC accessproto.AccessAPIClient, - historicalAccessNodes []accessproto.AccessAPIClient, - blocks storage.Blocks, - headers storage.Headers, - collections storage.Collections, - transactions storage.Transactions, - executionReceipts storage.ExecutionReceipts, - executionResults storage.ExecutionResults, - chainID flow.ChainID, accessMetrics module.AccessMetrics, - collectionGRPCPort uint, - executionGRPCPort uint, - retryEnabled bool, - maxMsgSize uint, - config Config, -) (*Backend, error) { + connectionPoolSize uint, +) (*lru.Cache, uint, error) { var cache *lru.Cache - cacheSize := config.ConnectionPoolSize + cacheSize := connectionPoolSize if cacheSize > 0 { // TODO: remove this fallback after fixing issues with evictions // It was observed that evictions cause connection errors for in flight requests. This works around @@ -241,42 +227,10 @@ func NewBackend( } }) if err != nil { - return nil, fmt.Errorf("could not initialize connection pool cache: %w", err) + return nil, 0, fmt.Errorf("could not initialize connection pool cache: %w", err) } } - - connectionFactory := &ConnectionFactoryImpl{ - CollectionGRPCPort: collectionGRPCPort, - ExecutionGRPCPort: executionGRPCPort, - CollectionNodeGRPCTimeout: config.CollectionClientTimeout, - ExecutionNodeGRPCTimeout: config.ExecutionClientTimeout, - ConnectionsCache: cache, - CacheSize: cacheSize, - MaxMsgSize: maxMsgSize, - AccessMetrics: accessMetrics, - Log: log, - } - - return New(state, - collectionRPC, - historicalAccessNodes, - blocks, - headers, - collections, - transactions, - executionReceipts, - executionResults, - chainID, - accessMetrics, - connectionFactory, - retryEnabled, - config.MaxHeightRange, - config.PreferredExecutionNodeIDs, - config.FixedExecutionNodeIDs, - log, - DefaultSnapshotHistoryLimit, - config.ArchiveAddressList, - ), nil + return cache, cacheSize, nil } func identifierList(ids []string) (flow.IdentifierList, error) { diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index d0b31b19118..87551c96d5d 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -119,8 +119,7 @@ func (suite *RateLimitTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - backend, err := backend.NewBackend( - suite.log, + backend := backend.New( suite.state, suite.collClient, nil, @@ -132,12 +131,14 @@ func (suite *RateLimitTestSuite) SetupTest() { nil, suite.chainID, suite.metrics, - 0, - 0, + nil, false, 0, - config.BackendConfig) - require.NoError(suite.T(), err) + nil, + nil, + suite.log, + 0, + nil) rpcEngBuilder, err := NewBuilder( suite.log, diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 86d018e0548..a5975a7e92c 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -11,7 +11,6 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -110,8 +109,7 @@ func (suite *SecureGRPCTestSuite) SetupTest() { block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) - backend, err := backend.NewBackend( - suite.log, + backend := backend.New( suite.state, suite.collClient, nil, @@ -123,12 +121,14 @@ func (suite *SecureGRPCTestSuite) SetupTest() { nil, suite.chainID, suite.metrics, - 0, - 0, + nil, false, 0, - config.BackendConfig) - require.NoError(suite.T(), err) + nil, + nil, + suite.log, + 0, + nil) rpcEngBuilder, err := rpc.NewBuilder( suite.log, diff --git a/module/forwarder/forwarder.go b/engine/common/grpc/forwarder/forwarder.go similarity index 100% rename from module/forwarder/forwarder.go rename to engine/common/grpc/forwarder/forwarder.go From 4bfc6d4d59a3a5a1bb897caba9bb0a9fef9d48ce Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 15:46:37 -0400 Subject: [PATCH 063/169] moving slashing violations consumer to the network --- Makefile | 2 +- .../node_builder/access_node_builder.go | 4 -- cmd/observer/node_builder/observer_builder.go | 21 +++----- cmd/scaffold.go | 4 -- follower/follower_builder.go | 4 -- module/metrics.go | 3 +- module/mock/network_core_metrics.go | 12 +++++ network/middleware.go | 4 +- network/mocknetwork/middleware.go | 5 ++ network/mocknetwork/violations_consumer.go | 30 +++++------ network/p2p/middleware/middleware.go | 54 ++++++++++--------- network/p2p/network.go | 4 ++ network/slashing/consumer.go | 34 ++++++------ .../validator/authorized_sender_validator.go | 22 ++++---- network/{slashing => }/violations_consumer.go | 2 +- 15 files changed, 107 insertions(+), 98 deletions(-) rename network/{slashing => }/violations_consumer.go (98%) diff --git a/Makefile b/Makefile index 14625ddf649..09a2b1b9456 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,7 @@ generate-mocks: install-mock-generators rm -rf ./fvm/environment/mock mockery --name '.*' --dir=fvm/environment --case=underscore --output="./fvm/environment/mock" --outpkg="mock" mockery --name '.*' --dir=ledger --case=underscore --output="./ledger/mock" --outpkg="mock" - mockery --name 'ViolationsConsumer' --dir=network/slashing --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" + mockery --name 'ViolationsConsumer' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork" mockery --name '.*' --dir=network/p2p/ --case=underscore --output="./network/p2p/mock" --outpkg="mockp2p" mockery --name '.*' --dir=network/alsp --case=underscore --output="./network/alsp/mock" --outpkg="mockalsp" mockery --name 'Vertex' --dir="./module/forest" --case=underscore --output="./module/forest/mock" --outpkg="mock" diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 5196bcbba08..b15a4946aee 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -78,7 +78,6 @@ import ( "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" relaynet "github.com/onflow/flow-go/network/relay" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/state/protocol" @@ -1271,9 +1270,6 @@ func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, UnicastMessageTimeout: middleware.DefaultUnicastTimeout, IdTranslator: builder.IDTranslator, Codec: builder.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(logger, networkMetrics, func() network.MisbehaviorReportConsumer { - return builder.MisbehaviorReportConsumer - }), }, middleware.WithMessageValidators(validators...), // use default identifier provider ) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 5da4b2b17af..0991678abf5 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -63,7 +63,6 @@ import ( "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" stateprotocol "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" @@ -910,19 +909,15 @@ func (builder *ObserverServiceBuilder) initMiddleware(nodeID flow.Identifier, libp2pNode p2p.LibP2PNode, validators ...network.MessageValidator, ) network.Middleware { - slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network, func() network.MisbehaviorReportConsumer { - return builder.MisbehaviorReportConsumer - }) mw := middleware.NewMiddleware(&middleware.Config{ - Logger: builder.Logger, - Libp2pNode: libp2pNode, - FlowId: nodeID, - BitSwapMetrics: builder.Metrics.Bitswap, - RootBlockID: builder.SporkID, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - IdTranslator: builder.IDTranslator, - Codec: builder.CodecFactory(), - SlashingViolationsConsumer: slashingViolationsConsumer, + Logger: builder.Logger, + Libp2pNode: libp2pNode, + FlowId: nodeID, + BitSwapMetrics: builder.Metrics.Bitswap, + RootBlockID: builder.SporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: builder.IDTranslator, + Codec: builder.CodecFactory(), }, middleware.WithMessageValidators(validators...), // use default identifier provider ) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 07baddfeab0..0269cde565f 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -60,7 +60,6 @@ import ( "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/topology" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" @@ -456,9 +455,6 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory( UnicastMessageTimeout: fnb.FlowConfig.NetworkConfig.UnicastMessageTimeout, IdTranslator: fnb.IDTranslator, Codec: fnb.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(fnb.Logger, fnb.Metrics.Network, func() network.MisbehaviorReportConsumer { - return fnb.MisbehaviorReportConsumer - }), }, mwOpts...) diff --git a/follower/follower_builder.go b/follower/follower_builder.go index a9d6585b013..a389cfa503c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -57,7 +57,6 @@ import ( "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/state/protocol" badgerState "github.com/onflow/flow-go/state/protocol/badger" @@ -761,9 +760,6 @@ func (builder *FollowerServiceBuilder) initMiddleware(nodeID flow.Identifier, UnicastMessageTimeout: middleware.DefaultUnicastTimeout, IdTranslator: builder.IDTranslator, Codec: builder.CodecFactory(), - SlashingViolationsConsumer: slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network, func() network.MisbehaviorReportConsumer { - return builder.MisbehaviorReportConsumer - }), }, middleware.WithMessageValidators(validators...), ) diff --git a/module/metrics.go b/module/metrics.go index de40bd0072f..cf390d580b6 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -182,6 +182,8 @@ type NetworkInboundQueueMetrics interface { type NetworkCoreMetrics interface { NetworkInboundQueueMetrics AlspMetrics + NetworkSecurityMetrics + // OutboundMessageSent collects metrics related to a message sent by the node. OutboundMessageSent(sizeBytes int, topic string, protocol string, messageType string) // InboundMessageReceived collects metrics related to a message received by the node. @@ -223,7 +225,6 @@ type AlspMetrics interface { // NetworkMetrics is the blanket abstraction that encapsulates the metrics collectors for the networking layer. type NetworkMetrics interface { LibP2PMetrics - NetworkSecurityMetrics NetworkCoreMetrics } diff --git a/module/mock/network_core_metrics.go b/module/mock/network_core_metrics.go index 63c849fbf27..bd1ec9ec1a2 100644 --- a/module/mock/network_core_metrics.go +++ b/module/mock/network_core_metrics.go @@ -5,6 +5,8 @@ package mock import ( mock "github.com/stretchr/testify/mock" + peer "github.com/libp2p/go-libp2p/core/peer" + time "time" ) @@ -48,6 +50,16 @@ func (_m *NetworkCoreMetrics) OnMisbehaviorReported(channel string, misbehaviorT _m.Called(channel, misbehaviorType) } +// OnRateLimitedPeer provides a mock function with given fields: pid, role, msgType, topic, reason +func (_m *NetworkCoreMetrics) OnRateLimitedPeer(pid peer.ID, role string, msgType string, topic string, reason string) { + _m.Called(pid, role, msgType, topic, reason) +} + +// OnUnauthorizedMessage provides a mock function with given fields: role, msgType, topic, offense +func (_m *NetworkCoreMetrics) OnUnauthorizedMessage(role string, msgType string, topic string, offense string) { + _m.Called(role, msgType, topic, offense) +} + // OutboundMessageSent provides a mock function with given fields: sizeBytes, topic, protocol, messageType func (_m *NetworkCoreMetrics) OutboundMessageSent(sizeBytes int, topic string, protocol string, messageType string) { _m.Called(sizeBytes, topic, protocol, messageType) diff --git a/network/middleware.go b/network/middleware.go index c2eeef98905..060d87bef4d 100644 --- a/network/middleware.go +++ b/network/middleware.go @@ -6,7 +6,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/network/channels" @@ -22,6 +21,9 @@ type Middleware interface { // SetOverlay sets the overlay used by the middleware. This must be called before the middleware can be Started. SetOverlay(Overlay) + // SetSlashingViolationsConsumer sets the slashing violations consumer. + SetSlashingViolationsConsumer(ViolationsConsumer) + // SendDirect sends msg on a 1-1 direct connection to the target ID. It models a guaranteed delivery asynchronous // direct one-to-one connection on the underlying network. No intermediate node on the overlay is utilized // as the router. diff --git a/network/mocknetwork/middleware.go b/network/mocknetwork/middleware.go index 64167ce9ed8..18cdaed21b0 100644 --- a/network/mocknetwork/middleware.go +++ b/network/mocknetwork/middleware.go @@ -160,6 +160,11 @@ func (_m *Middleware) SetOverlay(_a0 network.Overlay) { _m.Called(_a0) } +// SetSlashingViolationsConsumer provides a mock function with given fields: _a0 +func (_m *Middleware) SetSlashingViolationsConsumer(_a0 network.ViolationsConsumer) { + _m.Called(_a0) +} + // Start provides a mock function with given fields: _a0 func (_m *Middleware) Start(_a0 irrecoverable.SignalerContext) { _m.Called(_a0) diff --git a/network/mocknetwork/violations_consumer.go b/network/mocknetwork/violations_consumer.go index c02f2ed94cc..f281402564e 100644 --- a/network/mocknetwork/violations_consumer.go +++ b/network/mocknetwork/violations_consumer.go @@ -3,7 +3,7 @@ package mocknetwork import ( - slashing "github.com/onflow/flow-go/network/slashing" + network "github.com/onflow/flow-go/network" mock "github.com/stretchr/testify/mock" ) @@ -13,11 +13,11 @@ type ViolationsConsumer struct { } // OnInvalidMsgError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnInvalidMsgError(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnInvalidMsgError(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -27,11 +27,11 @@ func (_m *ViolationsConsumer) OnInvalidMsgError(violation *slashing.Violation) e } // OnSenderEjectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnSenderEjectedError(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnSenderEjectedError(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -41,11 +41,11 @@ func (_m *ViolationsConsumer) OnSenderEjectedError(violation *slashing.Violation } // OnUnAuthorizedSenderError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -55,11 +55,11 @@ func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *slashing.Viol } // OnUnauthorizedPublishOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -69,11 +69,11 @@ func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *slashing } // OnUnauthorizedUnicastOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -83,11 +83,11 @@ func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *slashing } // OnUnexpectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnexpectedError(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnUnexpectedError(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) @@ -97,11 +97,11 @@ func (_m *ViolationsConsumer) OnUnexpectedError(violation *slashing.Violation) e } // OnUnknownMsgTypeError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *slashing.Violation) error { +func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *network.Violation) error { ret := _m.Called(violation) var r0 error - if rf, ok := ret.Get(0).(func(*slashing.Violation) error); ok { + if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { r0 = rf(violation) } else { r0 = ret.Error(0) diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index f260d434ca8..7b0f275d537 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -35,7 +35,6 @@ import ( "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" _ "github.com/onflow/flow-go/utils/binstat" @@ -104,7 +103,7 @@ type Middleware struct { idTranslator p2p.IDTranslator previousProtocolStatePeers []peer.AddrInfo codec network.Codec - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer unicastRateLimiters *ratelimit.RateLimiters authorizedSenderValidator *validator.AuthorizedSenderValidator } @@ -140,15 +139,14 @@ func WithUnicastRateLimiters(rateLimiters *ratelimit.RateLimiters) OptionFn { // Config is the configuration for the middleware. type Config struct { - Logger zerolog.Logger - Libp2pNode p2p.LibP2PNode - FlowId flow.Identifier // This node's Flow ID - BitSwapMetrics module.BitswapMetrics - RootBlockID flow.Identifier - UnicastMessageTimeout time.Duration - IdTranslator p2p.IDTranslator - Codec network.Codec - SlashingViolationsConsumer slashing.ViolationsConsumer + Logger zerolog.Logger + Libp2pNode p2p.LibP2PNode + FlowId flow.Identifier // This node's Flow ID + BitSwapMetrics module.BitswapMetrics + RootBlockID flow.Identifier + UnicastMessageTimeout time.Duration + IdTranslator p2p.IDTranslator + Codec network.Codec } // Validate validates the configuration, and sets default values for any missing fields. @@ -172,16 +170,15 @@ func NewMiddleware(cfg *Config, opts ...OptionFn) *Middleware { // create the node entity and inject dependencies & config mw := &Middleware{ - log: cfg.Logger, - libP2PNode: cfg.Libp2pNode, - bitswapMetrics: cfg.BitSwapMetrics, - rootBlockID: cfg.RootBlockID, - validators: DefaultValidators(cfg.Logger, cfg.FlowId), - unicastMessageTimeout: cfg.UnicastMessageTimeout, - idTranslator: cfg.IdTranslator, - codec: cfg.Codec, - slashingViolationsConsumer: cfg.SlashingViolationsConsumer, - unicastRateLimiters: ratelimit.NoopRateLimiters(), + log: cfg.Logger, + libP2PNode: cfg.Libp2pNode, + bitswapMetrics: cfg.BitSwapMetrics, + rootBlockID: cfg.RootBlockID, + validators: DefaultValidators(cfg.Logger, cfg.FlowId), + unicastMessageTimeout: cfg.UnicastMessageTimeout, + idTranslator: cfg.IdTranslator, + codec: cfg.Codec, + unicastRateLimiters: ratelimit.NoopRateLimiters(), } for _, opt := range opts { @@ -304,6 +301,11 @@ func (m *Middleware) SetOverlay(ov network.Overlay) { m.ov = ov } +// SetSlashingViolationsConsumer sets the slashing violations consumer. +func (m *Middleware) SetSlashingViolationsConsumer(consumer network.ViolationsConsumer) { + m.slashingViolationsConsumer = consumer +} + // authorizedPeers is a peer manager callback used by the underlying libp2p node that updates who can connect to this node (as // well as who this node can connect to). // and who is not allowed to connect to this node. This function is called by the peer manager and connection gater components @@ -518,7 +520,7 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { // ignore messages if node does not have subscription to topic if !m.libP2PNode.HasSubscription(topic) { - violation := &slashing.Violation{ + violation := &network.Violation{ Identity: nil, PeerID: remotePeer.String(), Channel: channel, Protocol: message.ProtocolTypeUnicast, } @@ -654,7 +656,7 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag // we can remove this check maxSize, err := unicastMaxMsgSizeByCode(msg.Payload) if err != nil { - svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(&slashing.Violation{ + svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(&network.Violation{ Identity: nil, PeerID: remotePeer.String(), MsgType: "", Channel: channel, Protocol: message.ProtocolTypeUnicast, Err: err, }) m.checkSlashingViolationsConsumerErr(svcErr) @@ -709,7 +711,7 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe switch { case codec.IsErrUnknownMsgCode(err): // slash peer if message contains unknown message code byte - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) @@ -717,7 +719,7 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe return case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } svcErr := m.slashingViolationsConsumer.OnInvalidMsgError(violation) @@ -728,7 +730,7 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe // don't crash as a result of external inputs since that creates a DoS vector // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) - violation := &slashing.Violation{ + violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } svcErr := m.slashingViolationsConsumer.OnUnexpectedError(violation) diff --git a/network/p2p/network.go b/network/p2p/network.go index 384fad3ab59..5b9d0b4df45 100644 --- a/network/p2p/network.go +++ b/network/p2p/network.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/queue" + "github.com/onflow/flow-go/network/slashing" _ "github.com/onflow/flow-go/utils/binstat" "github.com/onflow/flow-go/utils/logging" ) @@ -53,6 +54,7 @@ type Network struct { registerEngineRequests chan *registerEngineRequest registerBlobServiceRequests chan *registerBlobServiceRequest misbehaviorReportManager network.MisbehaviorReportManager + slashingViolationsConsumer network.ViolationsConsumer } var _ network.Network = &Network{} @@ -171,6 +173,8 @@ func NewNetwork(param *NetworkConfig, opts ...NetworkOption) (*Network, error) { opt(n) } + n.slashingViolationsConsumer = slashing.NewSlashingViolationsConsumer(param.Logger, param.Metrics, n) + n.mw.SetSlashingViolationsConsumer(n.slashingViolationsConsumer) n.mw.SetOverlay(n) if err := n.conduitFactory.RegisterAdapter(n); err != nil { diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index b2c8c74ca85..3fdbe307564 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -19,22 +19,22 @@ const ( // Consumer is a struct that logs a message for any slashable offenses. // This struct will be updated in the future when slashing is implemented. type Consumer struct { - log zerolog.Logger - metrics module.NetworkSecurityMetrics - reportConsumerFactory func() network.MisbehaviorReportConsumer + log zerolog.Logger + metrics module.NetworkSecurityMetrics + misbehaviorReportConsumer network.MisbehaviorReportConsumer } // NewSlashingViolationsConsumer returns a new Consumer. -func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, reportConsumerFactory func() network.MisbehaviorReportConsumer) *Consumer { +func NewSlashingViolationsConsumer(log zerolog.Logger, metrics module.NetworkSecurityMetrics, misbehaviorReportConsumer network.MisbehaviorReportConsumer) *Consumer { return &Consumer{ - log: log.With().Str("module", "network_slashing_consumer").Logger(), - metrics: metrics, - reportConsumerFactory: reportConsumerFactory, + log: log.With().Str("module", "network_slashing_consumer").Logger(), + metrics: metrics, + misbehaviorReportConsumer: misbehaviorReportConsumer, } } // logOffense logs the slashing violation with details. -func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *Violation) { +func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *network.Violation) { // if violation fails before the message is decoded the violation.MsgType will be unknown if len(violation.MsgType) == 0 { violation.MsgType = unknown @@ -67,7 +67,7 @@ func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *Violat // reportMisbehavior reports the slashing violation to the alsp misbehavior report manager. When violation identity // is nil this indicates the misbehavior occurred either on a public network and the identity of the sender is unknown // we can skip reporting the misbehavior. -func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *Violation) error { +func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) error { if violation.Identity == nil { c.log.Debug().Str("peerID", violation.PeerID).Msg("violation identity unknown skipping misbehavior reporting") return nil @@ -76,50 +76,50 @@ func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation if err != nil { return fmt.Errorf("failed to create misbehavior report: %w", err) } - c.reportConsumerFactory().ReportMisbehaviorOnChannel(violation.Channel, report) + c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) return nil } // OnUnAuthorizedSenderError logs an error for unauthorized sender error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnAuthorizedSenderError(violation *Violation) error { +func (c *Consumer) OnUnAuthorizedSenderError(violation *network.Violation) error { c.logOffense(alsp.UnAuthorizedSender, violation) return c.reportMisbehavior(alsp.UnAuthorizedSender, violation) } // OnUnknownMsgTypeError logs an error for unknown message type error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnknownMsgTypeError(violation *Violation) error { +func (c *Consumer) OnUnknownMsgTypeError(violation *network.Violation) error { c.logOffense(alsp.UnknownMsgType, violation) return c.reportMisbehavior(alsp.UnknownMsgType, violation) } // OnInvalidMsgError logs an error for messages that contained payloads that could not // be unmarshalled into the message type denoted by message code byte and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnInvalidMsgError(violation *Violation) error { +func (c *Consumer) OnInvalidMsgError(violation *network.Violation) error { c.logOffense(alsp.InvalidMessage, violation) return c.reportMisbehavior(alsp.InvalidMessage, violation) } // OnSenderEjectedError logs an error for sender ejected error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnSenderEjectedError(violation *Violation) error { +func (c *Consumer) OnSenderEjectedError(violation *network.Violation) error { c.logOffense(alsp.SenderEjected, violation) return c.reportMisbehavior(alsp.SenderEjected, violation) } // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *Violation) error { +func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) error { c.logOffense(alsp.UnauthorizedUnicastOnChannel, violation) return c.reportMisbehavior(alsp.UnauthorizedUnicastOnChannel, violation) } // OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. -func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *Violation) error { +func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) error { c.logOffense(alsp.UnauthorizedPublishOnChannel, violation) return c.reportMisbehavior(alsp.UnauthorizedPublishOnChannel, violation) } // OnUnexpectedError logs an error for unexpected errors. This indicates message validation // has failed for an unknown reason and could potentially be n slashable offense and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnexpectedError(violation *Violation) error { +func (c *Consumer) OnUnexpectedError(violation *network.Violation) error { c.logOffense(alsp.UnExpectedValidationError, violation) return c.reportMisbehavior(alsp.UnExpectedValidationError, violation) } diff --git a/network/validator/authorized_sender_validator.go b/network/validator/authorized_sender_validator.go index 4e83b0762c7..9817e3a7d6f 100644 --- a/network/validator/authorized_sender_validator.go +++ b/network/validator/authorized_sender_validator.go @@ -8,11 +8,11 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/codec" "github.com/onflow/flow-go/network/message" "github.com/onflow/flow-go/network/p2p" - "github.com/onflow/flow-go/network/slashing" ) var ( @@ -25,12 +25,12 @@ type GetIdentityFunc func(peer.ID) (*flow.Identity, bool) // AuthorizedSenderValidator performs message authorization validation. type AuthorizedSenderValidator struct { log zerolog.Logger - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer getIdentity GetIdentityFunc } // NewAuthorizedSenderValidator returns a new AuthorizedSenderValidator -func NewAuthorizedSenderValidator(log zerolog.Logger, slashingViolationsConsumer slashing.ViolationsConsumer, getIdentity GetIdentityFunc) *AuthorizedSenderValidator { +func NewAuthorizedSenderValidator(log zerolog.Logger, slashingViolationsConsumer network.ViolationsConsumer, getIdentity GetIdentityFunc) *AuthorizedSenderValidator { return &AuthorizedSenderValidator{ log: log.With().Str("component", "authorized_sender_validator").Logger(), slashingViolationsConsumer: slashingViolationsConsumer, @@ -61,7 +61,7 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan // something terrible went wrong. identity, ok := av.getIdentity(from) if !ok { - violation := &slashing.Violation{PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} + violation := &network.Violation{PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} err := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) av.checkSlashingViolationsConsumerErr(err) return "", ErrIdentityUnverified @@ -69,7 +69,7 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan msgCode, err := codec.MessageCodeFromPayload(payload) if err != nil { - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) av.checkSlashingViolationsConsumerErr(svcErr) return "", err @@ -80,30 +80,30 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan case err == nil: return msgType, nil case message.IsUnknownMsgTypeErr(err) || codec.IsErrUnknownMsgCode(err): - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) av.checkSlashingViolationsConsumerErr(svcErr) return msgType, err case errors.Is(err, message.ErrUnauthorizedMessageOnChannel) || errors.Is(err, message.ErrUnauthorizedRole): - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) av.checkSlashingViolationsConsumerErr(svcErr) return msgType, err case errors.Is(err, ErrSenderEjected): - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnSenderEjectedError(violation) av.checkSlashingViolationsConsumerErr(svcErr) return msgType, err case errors.Is(err, message.ErrUnauthorizedUnicastOnChannel): - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) av.checkSlashingViolationsConsumerErr(svcErr) return msgType, err case errors.Is(err, message.ErrUnauthorizedPublishOnChannel): - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnauthorizedPublishOnChannel(violation) av.checkSlashingViolationsConsumerErr(svcErr) @@ -113,7 +113,7 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan // don't crash as a result of external inputs since that creates a DoS vector // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) - violation := &slashing.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} + violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} svcErr := av.slashingViolationsConsumer.OnUnexpectedError(violation) av.checkSlashingViolationsConsumerErr(svcErr) diff --git a/network/slashing/violations_consumer.go b/network/violations_consumer.go similarity index 98% rename from network/slashing/violations_consumer.go rename to network/violations_consumer.go index 981fa549089..0936e61efc9 100644 --- a/network/slashing/violations_consumer.go +++ b/network/violations_consumer.go @@ -1,4 +1,4 @@ -package slashing +package network import ( "github.com/onflow/flow-go/model/flow" From ee9f7140a38cd970c84806dba9f194928cf016dc Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 15:46:50 -0400 Subject: [PATCH 064/169] update unit tests --- network/internal/testutils/testUtil.go | 25 +++++++------ network/p2p/test/topic_validator_test.go | 9 ++--- network/test/middleware_test.go | 3 +- network/test/unicast_authorization_test.go | 35 ++++++++++--------- .../authorized_sender_validator_test.go | 24 ++++++------- utils/unittest/unittest.go | 10 ++---- 6 files changed, 51 insertions(+), 55 deletions(-) diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index 47533cb2677..eb8506fb182 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -48,7 +48,6 @@ import ( "github.com/onflow/flow-go/network/p2p/unicast" "github.com/onflow/flow-go/network/p2p/unicast/protocols" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -187,7 +186,7 @@ func GenerateMiddlewares(t *testing.T, identities flow.IdentityList, libP2PNodes []p2p.LibP2PNode, codec network.Codec, - consumer slashing.ViolationsConsumer, + consumer network.ViolationsConsumer, opts ...func(*optsConfig)) ([]network.Middleware, []*UpdatableIDProvider) { mws := make([]network.Middleware, len(identities)) idProviders := make([]*UpdatableIDProvider, len(identities)) @@ -213,18 +212,18 @@ func GenerateMiddlewares(t *testing.T, // creating middleware of nodes mws[i] = middleware.NewMiddleware(&middleware.Config{ - Logger: logger, - Libp2pNode: node, - FlowId: nodeId, - BitSwapMetrics: bitswapmet, - RootBlockID: sporkID, - UnicastMessageTimeout: middleware.DefaultUnicastTimeout, - IdTranslator: translator.NewIdentityProviderIDTranslator(idProviders[i]), - Codec: codec, - SlashingViolationsConsumer: consumer, + Logger: logger, + Libp2pNode: node, + FlowId: nodeId, + BitSwapMetrics: bitswapmet, + RootBlockID: sporkID, + UnicastMessageTimeout: middleware.DefaultUnicastTimeout, + IdTranslator: translator.NewIdentityProviderIDTranslator(idProviders[i]), + Codec: codec, }, middleware.WithUnicastRateLimiters(o.unicastRateLimiters), middleware.WithPeerManagerFilters(o.peerManagerFilters)) + mws[i].SetSlashingViolationsConsumer(consumer) } return mws, idProviders } @@ -300,7 +299,7 @@ func GenerateIDsAndMiddlewares(t *testing.T, n int, logger zerolog.Logger, codec network.Codec, - consumer slashing.ViolationsConsumer, + consumer network.ViolationsConsumer, opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []observable.Observable, []*UpdatableIDProvider) { ids, libP2PNodes, protectObservables := GenerateIDs(t, logger, n, opts...) @@ -380,7 +379,7 @@ func GenerateIDsMiddlewaresNetworks(t *testing.T, n int, log zerolog.Logger, codec network.Codec, - consumer slashing.ViolationsConsumer, + consumer network.ViolationsConsumer, opts ...func(*optsConfig)) (flow.IdentityList, []p2p.LibP2PNode, []network.Middleware, []network.Network, []observable.Observable) { ids, libp2pNodes, mws, observables, _ := GenerateIDsAndMiddlewares(t, n, log, codec, consumer, opts...) sms := GenerateSubscriptionManagers(t, mws) diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index 9348544174f..adbb3822daf 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -15,6 +15,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/alsp" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/p2pfixtures" @@ -297,7 +298,7 @@ func TestAuthorizedSenderValidator_Unauthorized(t *testing.T) { translatorFixture, err := translator.NewFixedTableIdentityTranslator(ids) require.NoError(t, err) - violation := &slashing.Violation{ + violation := &network.Violation{ Identity: &identity3, PeerID: an1.Host().ID().String(), OriginID: identity3.NodeID, @@ -417,7 +418,7 @@ func TestAuthorizedSenderValidator_InvalidMsg(t *testing.T) { require.NoError(t, err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { @@ -494,7 +495,7 @@ func TestAuthorizedSenderValidator_Ejected(t *testing.T) { require.NoError(t, err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { @@ -591,7 +592,7 @@ func TestAuthorizedSenderValidator_ClusterChannel(t *testing.T) { logger := unittest.Logger() misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(t) defer misbehaviorReportConsumer.AssertNotCalled(t, "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(logger, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentity := func(pid peer.ID) (*flow.Identity, bool) { fid, err := translatorFixture.GetFlowID(pid) if err != nil { diff --git a/network/test/middleware_test.go b/network/test/middleware_test.go index 811b100e52f..4b34e084eaf 100644 --- a/network/test/middleware_test.go +++ b/network/test/middleware_test.go @@ -38,7 +38,6 @@ import ( p2ptest "github.com/onflow/flow-go/network/p2p/test" "github.com/onflow/flow-go/network/p2p/unicast/ratelimit" "github.com/onflow/flow-go/network/p2p/utils/ratelimiter" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -85,7 +84,7 @@ type MiddlewareTestSuite struct { mwCancel context.CancelFunc mwCtx irrecoverable.SignalerContext - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer } // TestMiddlewareTestSuit runs all the test methods in this test suit diff --git a/network/test/unicast_authorization_test.go b/network/test/unicast_authorization_test.go index 6fe4d0b8b58..8d810a98e21 100644 --- a/network/test/unicast_authorization_test.go +++ b/network/test/unicast_authorization_test.go @@ -24,7 +24,6 @@ import ( "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/validator" "github.com/onflow/flow-go/utils/unittest" ) @@ -73,7 +72,7 @@ func (u *UnicastAuthorizationTestSuite) TearDownTest() { } // setupMiddlewaresAndProviders will setup 2 middlewares that will be used as a sender and receiver in each suite test. -func (u *UnicastAuthorizationTestSuite) setupMiddlewaresAndProviders(slashingViolationsConsumer slashing.ViolationsConsumer) { +func (u *UnicastAuthorizationTestSuite) setupMiddlewaresAndProviders(slashingViolationsConsumer network.ViolationsConsumer) { ids, libP2PNodes, _ := testutils.GenerateIDs(u.T(), u.logger, 2) mws, providers := testutils.GenerateMiddlewares(u.T(), u.logger, ids, libP2PNodes, unittest.NetworkCodec(), slashingViolationsConsumer) require.Len(u.T(), ids, 2) @@ -123,7 +122,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() require.NoError(u.T(), err) var nilID *flow.Identity - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nilID, // because the peer will be unverified this identity will be nil PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type @@ -134,7 +133,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnstakedPeer() slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -185,8 +184,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, // we expect this method to be called with the ejected identity + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "", // message will not be decoded before OnSenderEjectedError is logged, we won't log message type Channel: channels.TestNetworkChannel, // message will not be decoded before OnSenderEjectedError is logged, we won't log peer ID @@ -196,7 +196,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_EjectedPeer() { slashingViolationsConsumer.On( "OnSenderEjectedError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -244,8 +244,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*message.TestMessage", Channel: channels.ConsensusCommittee, @@ -256,7 +257,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedPee slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -307,7 +308,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( invalidMessageCode := codec.MessageCode(byte('X')) var nilID *flow.Identity - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nilID, PeerID: expectedSenderPeerID.String(), MsgType: "", @@ -319,7 +320,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnknownMsgCode( slashingViolationsConsumer.On( "OnUnknownMsgTypeError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -376,8 +377,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() modifiedMessageCode := codec.CodeDKGMessage - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*messages.DKGMessage", Channel: channels.TestNetworkChannel, @@ -388,7 +390,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_WrongMsgCode() slashingViolationsConsumer.On( "OnUnAuthorizedSenderError", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -501,8 +503,9 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: u.senderID, + OriginID: u.senderID.NodeID, PeerID: expectedSenderPeerID.String(), MsgType: "*messages.BlockProposal", Channel: channels.ConsensusCommittee, @@ -513,7 +516,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_UnauthorizedUni slashingViolationsConsumer.On( "OnUnauthorizedUnicastOnChannel", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) @@ -564,7 +567,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu expectedSenderPeerID, err := unittest.PeerIDFromFlowID(u.senderID) require.NoError(u.T(), err) - expectedViolation := &slashing.Violation{ + expectedViolation := &network.Violation{ Identity: nil, PeerID: expectedSenderPeerID.String(), MsgType: "*message.TestMessage", @@ -576,7 +579,7 @@ func (u *UnicastAuthorizationTestSuite) TestUnicastAuthorization_ReceiverHasNoSu slashingViolationsConsumer.On( "OnUnauthorizedUnicastOnChannel", expectedViolation, - ).Once().Run(func(args mockery.Arguments) { + ).Return(nil).Return(nil).Once().Run(func(args mockery.Arguments) { close(u.waitCh) }) diff --git a/network/validator/authorized_sender_validator_test.go b/network/validator/authorized_sender_validator_test.go index 7ba2fb80e2d..8a9cd138cbb 100644 --- a/network/validator/authorized_sender_validator_test.go +++ b/network/validator/authorized_sender_validator_test.go @@ -47,7 +47,7 @@ type TestAuthorizedSenderValidatorSuite struct { unauthorizedUnicastOnChannel []TestCase authorizedUnicastOnChannel []TestCase log zerolog.Logger - slashingViolationsConsumer slashing.ViolationsConsumer + slashingViolationsConsumer network.ViolationsConsumer allMsgConfigs []message.MsgAuthConfig codec network.Codec } @@ -69,7 +69,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen s.Run(str, func() { misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) validateUnicast := authorizedSenderValidator.Validate validatePubsub := authorizedSenderValidator.PubSubMessageValidator(c.Channel) @@ -105,7 +105,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedSen identity, _ := unittest.IdentityWithNetworkingKeyFixture(unittest.WithRole(flow.RoleCollection)) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) getIdentityFunc := s.getIdentity(identity) pid, err := unittest.PeerIDFromFlowID(identity) require.NoError(s.T(), err) @@ -139,7 +139,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedS require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) payload, err := s.codec.Encode(c.Message) @@ -165,7 +165,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_AuthorizedUni require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -187,7 +187,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedU require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -209,7 +209,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnAuthorizedM require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypeUnicast) @@ -245,7 +245,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ClusterPrefix misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCluster(clusterID), expectedMisbehaviorReport).Once() misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCluster(clusterID), expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) // validate collection sync cluster SyncRequest is not allowed to be sent on channel via unicast @@ -294,7 +294,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.SyncCommittee, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) @@ -323,7 +323,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", channels.ConsensusCommittee, expectedMisbehaviorReport).Twice() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) validatePubsub := authorizedSenderValidator.PubSubMessageValidator(channels.ConsensusCommittee) @@ -355,7 +355,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_ValidationFai misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) // we cannot penalize a peer if identity is not known, in this case we don't expect any misbehavior reports to be reported defer misbehaviorReportConsumer.AssertNotCalled(s.T(), "ReportMisbehaviorOnChannel", mock.AnythingOfType("channels.Channel"), mock.AnythingOfType("*alsp.MisbehaviorReport")) - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, getIdentityFunc) msgType, err := authorizedSenderValidator.Validate(pid, []byte{codec.CodeSyncRequest.Uint8()}, channels.SyncCommittee, message.ProtocolTypeUnicast) @@ -389,7 +389,7 @@ func (s *TestAuthorizedSenderValidatorSuite) TestValidatorCallback_UnauthorizedP require.NoError(s.T(), err) misbehaviorReportConsumer := mocknetwork.NewMisbehaviorReportConsumer(s.T()) misbehaviorReportConsumer.On("ReportMisbehaviorOnChannel", c.Channel, expectedMisbehaviorReport).Once() - violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), unittest.MisbehaviorReportConsumerFactory(misbehaviorReportConsumer)) + violationsConsumer := slashing.NewSlashingViolationsConsumer(s.log, metrics.NewNoopCollector(), misbehaviorReportConsumer) authorizedSenderValidator := NewAuthorizedSenderValidator(s.log, violationsConsumer, c.GetIdentity) msgType, err := authorizedSenderValidator.Validate(pid, []byte{c.MessageCode.Uint8()}, c.Channel, message.ProtocolTypePubSub) require.ErrorIs(s.T(), err, message.ErrUnauthorizedPublishOnChannel) diff --git a/utils/unittest/unittest.go b/utils/unittest/unittest.go index ea4c3c6c28e..0d9949ffc2d 100644 --- a/utils/unittest/unittest.go +++ b/utils/unittest/unittest.go @@ -439,8 +439,8 @@ func GenerateRandomStringWithLen(commentLen uint) string { } // NetworkSlashingViolationsConsumer returns a slashing violations consumer for network middleware -func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) slashing.ViolationsConsumer { - return slashing.NewSlashingViolationsConsumer(logger, metrics, MisbehaviorReportConsumerFactory(consumer)) +func NetworkSlashingViolationsConsumer(logger zerolog.Logger, metrics module.NetworkSecurityMetrics, consumer network.MisbehaviorReportConsumer) network.ViolationsConsumer { + return slashing.NewSlashingViolationsConsumer(logger, metrics, consumer) } type MisbehaviorReportConsumerFixture struct { @@ -454,9 +454,3 @@ func (c *MisbehaviorReportConsumerFixture) ReportMisbehaviorOnChannel(channel ch func NewMisbehaviorReportConsumerFixture(manager network.MisbehaviorReportManager) *MisbehaviorReportConsumerFixture { return &MisbehaviorReportConsumerFixture{manager} } - -func MisbehaviorReportConsumerFactory(consumer network.MisbehaviorReportConsumer) func() network.MisbehaviorReportConsumer { - return func() network.MisbehaviorReportConsumer { - return consumer - } -} From 4a22fa9761a3c9e5a29d9e3a042b9ebcd9d68d53 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 16:58:33 -0400 Subject: [PATCH 065/169] add err const --- network/p2p/middleware/middleware.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index 7b0f275d537..5a5c6180198 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -62,12 +62,7 @@ const ( // LargeMsgUnicastTimeout is the maximum time to wait for a unicast request to complete for large message size LargeMsgUnicastTimeout = 1000 * time.Second - // DisallowListCacheSize is the maximum number of peers that can be disallow-listed at a time. The recommended - // size is 100 * number of staked nodes. Note that when the cache is full, there is no eviction policy and - // disallow-listing a new peer will fail. Hence, the cache size should be set to a value that is large enough - // to accommodate all the peers that can be disallow-listed at a time. Also, note that this cache is only taking - // the staked (authorized) peers. Hence, Sybil attacks are not possible. - DisallowListCacheSize = 100 * 1000 + violationDisseminateErr = "failed to disseminate slashing violation on slashing violations consumer" ) var ( @@ -831,7 +826,7 @@ func (m *Middleware) IsConnected(nodeID flow.Identifier) (bool, error) { func (m *Middleware) checkSlashingViolationsConsumerErr(err error) { if err != nil { - m.log.Error().Err(err).Msg("failed to disseminate slashing violation on slashing violations consumer") + m.log.Error().Err(err).Msg(violationDisseminateErr) } } From 8a002e8646b8158b643e5433f2d2815e6acb4dd2 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 16:58:40 -0400 Subject: [PATCH 066/169] Update network/slashing/consumer.go Co-authored-by: Yahya Hassanzadeh --- network/slashing/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index 3fdbe307564..f2866a4bb9e 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -69,7 +69,7 @@ func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *networ // we can skip reporting the misbehavior. func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) error { if violation.Identity == nil { - c.log.Debug().Str("peerID", violation.PeerID).Msg("violation identity unknown skipping misbehavior reporting") + c.log.Debug().Str("peerID", violation.PeerID).Msg("violation identity unknown (or public) skipping misbehavior reporting") return nil } report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) From 726f4267a75e50a0eeaf0d248821e7d53354a1c0 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 17:29:28 -0400 Subject: [PATCH 067/169] add counter metric for amount of time a violation report has been skipped --- module/metrics.go | 4 ++++ module/metrics/namespaces.go | 1 + module/metrics/network.go | 18 +++++++++++++++++- module/metrics/noop.go | 1 + module/mock/network_core_metrics.go | 5 +++++ module/mock/network_metrics.go | 5 +++++ module/mock/network_security_metrics.go | 5 +++++ network/slashing/consumer.go | 3 ++- 8 files changed, 40 insertions(+), 2 deletions(-) diff --git a/module/metrics.go b/module/metrics.go index cf390d580b6..7c52ca2bb1f 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -42,6 +42,10 @@ type NetworkSecurityMetrics interface { // OnRateLimitedPeer tracks the number of rate limited unicast messages seen on the network. OnRateLimitedPeer(pid peer.ID, role, msgType, topic, reason string) + + // OnViolationReportSkipped tracks the number of slashing violations consumer violations that were not + // reported for misbehavior when the identity of the sender not known. + OnViolationReportSkipped() } // GossipSubRouterMetrics encapsulates the metrics collectors for GossipSubRouter module of the networking layer. diff --git a/module/metrics/namespaces.go b/module/metrics/namespaces.go index 31995538992..6fd0f2db82f 100644 --- a/module/metrics/namespaces.go +++ b/module/metrics/namespaces.go @@ -28,6 +28,7 @@ const ( subsystemAuth = "authorization" subsystemRateLimiting = "ratelimit" subsystemAlsp = "alsp" + subsystemSecurity = "security" ) // Storage subsystems represent the various components of the storage layer. diff --git a/module/metrics/network.go b/module/metrics/network.go index f064ca10f6e..a1a92c28274 100644 --- a/module/metrics/network.go +++ b/module/metrics/network.go @@ -45,9 +45,10 @@ type NetworkCollector struct { dnsLookupRequestDroppedCount prometheus.Counter routingTableSize prometheus.Gauge - // authorization, rate limiting metrics + // security metrics unAuthorizedMessagesCount *prometheus.CounterVec rateLimitedUnicastMessagesCount *prometheus.CounterVec + violationReportSkippedCount prometheus.Counter prefix string } @@ -245,6 +246,15 @@ func NewNetworkCollector(logger zerolog.Logger, opts ...NetworkCollectorOpt) *Ne }, []string{LabelNodeRole, LabelMessage, LabelChannel, LabelRateLimitReason}, ) + nc.violationReportSkippedCount = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: namespaceNetwork, + Subsystem: subsystemSecurity, + Name: nc.prefix + "slashing_violation_reports_skipped_count", + Help: "number of slashing violations consumer violations that were not reported for misbehavior when the identity of the sender not known", + }, + ) + return nc } @@ -358,3 +368,9 @@ func (nc *NetworkCollector) OnRateLimitedPeer(peerID peer.ID, role, msgType, top Msg("unicast peer rate limited") nc.rateLimitedUnicastMessagesCount.WithLabelValues(role, msgType, topic, reason).Inc() } + +// OnViolationReportSkipped tracks the number of slashing violations consumer violations that were not +// reported for misbehavior when the identity of the sender not known. +func (nc *NetworkCollector) OnViolationReportSkipped() { + nc.violationReportSkippedCount.Inc() +} diff --git a/module/metrics/noop.go b/module/metrics/noop.go index 83a3fe5ae10..4ce7cd96fc3 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -298,3 +298,4 @@ func (nc *NoopCollector) AsyncProcessingStarted(string) func (nc *NoopCollector) AsyncProcessingFinished(string, time.Duration) {} func (nc *NoopCollector) OnMisbehaviorReported(string, string) {} +func (nc *NoopCollector) OnViolationReportSkipped() {} diff --git a/module/mock/network_core_metrics.go b/module/mock/network_core_metrics.go index bd1ec9ec1a2..d78c3355449 100644 --- a/module/mock/network_core_metrics.go +++ b/module/mock/network_core_metrics.go @@ -60,6 +60,11 @@ func (_m *NetworkCoreMetrics) OnUnauthorizedMessage(role string, msgType string, _m.Called(role, msgType, topic, offense) } +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkCoreMetrics) OnViolationReportSkipped() { + _m.Called() +} + // OutboundMessageSent provides a mock function with given fields: sizeBytes, topic, protocol, messageType func (_m *NetworkCoreMetrics) OutboundMessageSent(sizeBytes int, topic string, protocol string, messageType string) { _m.Called(sizeBytes, topic, protocol, messageType) diff --git a/module/mock/network_metrics.go b/module/mock/network_metrics.go index 851565d5724..2909f7d677f 100644 --- a/module/mock/network_metrics.go +++ b/module/mock/network_metrics.go @@ -300,6 +300,11 @@ func (_m *NetworkMetrics) OnUnauthorizedMessage(role string, msgType string, top _m.Called(role, msgType, topic, offense) } +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkMetrics) OnViolationReportSkipped() { + _m.Called() +} + // OutboundConnections provides a mock function with given fields: connectionCount func (_m *NetworkMetrics) OutboundConnections(connectionCount uint) { _m.Called(connectionCount) diff --git a/module/mock/network_security_metrics.go b/module/mock/network_security_metrics.go index 51d045c2a12..a48a693c0ab 100644 --- a/module/mock/network_security_metrics.go +++ b/module/mock/network_security_metrics.go @@ -23,6 +23,11 @@ func (_m *NetworkSecurityMetrics) OnUnauthorizedMessage(role string, msgType str _m.Called(role, msgType, topic, offense) } +// OnViolationReportSkipped provides a mock function with given fields: +func (_m *NetworkSecurityMetrics) OnViolationReportSkipped() { + _m.Called() +} + type mockConstructorTestingTNewNetworkSecurityMetrics interface { mock.TestingT Cleanup(func()) diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index f2866a4bb9e..aff78176bb9 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -69,7 +69,8 @@ func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *networ // we can skip reporting the misbehavior. func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) error { if violation.Identity == nil { - c.log.Debug().Str("peerID", violation.PeerID).Msg("violation identity unknown (or public) skipping misbehavior reporting") + c.log.Debug().Bool(logging.KeySuspicious, true).Str("peerID", violation.PeerID).Msg("violation identity unknown (or public) skipping misbehavior reporting") + c.metrics.OnViolationReportSkipped() return nil } report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) From 05671930c29874ddaadf84c3bb4fc14c43490bef Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 18:10:56 -0400 Subject: [PATCH 068/169] remove error return emit fatal log instead --- network/mocknetwork/violations_consumer.go | 91 +++---------------- network/p2p/middleware/middleware.go | 21 ++--- network/slashing/consumer.go | 49 ++++++---- .../validator/authorized_sender_validator.go | 35 ++----- network/violations_consumer.go | 25 ++--- 5 files changed, 73 insertions(+), 148 deletions(-) diff --git a/network/mocknetwork/violations_consumer.go b/network/mocknetwork/violations_consumer.go index f281402564e..2af1bf2b80f 100644 --- a/network/mocknetwork/violations_consumer.go +++ b/network/mocknetwork/violations_consumer.go @@ -13,101 +13,38 @@ type ViolationsConsumer struct { } // OnInvalidMsgError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnInvalidMsgError(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnInvalidMsgError(violation *network.Violation) { + _m.Called(violation) } // OnSenderEjectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnSenderEjectedError(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnSenderEjectedError(violation *network.Violation) { + _m.Called(violation) } // OnUnAuthorizedSenderError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnUnAuthorizedSenderError(violation *network.Violation) { + _m.Called(violation) } // OnUnauthorizedPublishOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) { + _m.Called(violation) } // OnUnauthorizedUnicastOnChannel provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) { + _m.Called(violation) } // OnUnexpectedError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnexpectedError(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnUnexpectedError(violation *network.Violation) { + _m.Called(violation) } // OnUnknownMsgTypeError provides a mock function with given fields: violation -func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *network.Violation) error { - ret := _m.Called(violation) - - var r0 error - if rf, ok := ret.Get(0).(func(*network.Violation) error); ok { - r0 = rf(violation) - } else { - r0 = ret.Error(0) - } - - return r0 +func (_m *ViolationsConsumer) OnUnknownMsgTypeError(violation *network.Violation) { + _m.Called(violation) } type mockConstructorTestingTNewViolationsConsumer interface { diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index 5a5c6180198..0a71fc10c0c 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -522,8 +522,7 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { msgCode, err := codec.MessageCodeFromPayload(msg.Payload) if err != nil { violation.Err = err - svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return } @@ -531,15 +530,13 @@ func (m *Middleware) handleIncomingStream(s libp2pnetwork.Stream) { _, what, err := codec.InterfaceFromMessageCode(msgCode) if err != nil { violation.Err = err - svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return } violation.MsgType = what violation.Err = ErrUnicastMsgWithoutSub - svcErr := m.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) return } @@ -651,10 +648,9 @@ func (m *Middleware) processUnicastStreamMessage(remotePeer peer.ID, msg *messag // we can remove this check maxSize, err := unicastMaxMsgSizeByCode(msg.Payload) if err != nil { - svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(&network.Violation{ + m.slashingViolationsConsumer.OnUnknownMsgTypeError(&network.Violation{ Identity: nil, PeerID: remotePeer.String(), MsgType: "", Channel: channel, Protocol: message.ProtocolTypeUnicast, Err: err, }) - m.checkSlashingViolationsConsumerErr(svcErr) return } if msg.Size() > maxSize { @@ -709,16 +705,14 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - svcErr := m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return case codec.IsErrMsgUnmarshal(err) || codec.IsErrInvalidEncoding(err): // slash if peer sent a message that could not be marshalled into the message type denoted by the message code byte violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - svcErr := m.slashingViolationsConsumer.OnInvalidMsgError(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnInvalidMsgError(violation) return case err != nil: // this condition should never happen and indicates there's a bug @@ -728,8 +722,7 @@ func (m *Middleware) processAuthenticatedMessage(msg *message.Message, peerID pe violation := &network.Violation{ PeerID: peerID.String(), OriginID: originId, Channel: channel, Protocol: protocol, Err: err, } - svcErr := m.slashingViolationsConsumer.OnUnexpectedError(violation) - m.checkSlashingViolationsConsumerErr(svcErr) + m.slashingViolationsConsumer.OnUnexpectedError(violation) return } diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index aff78176bb9..765cdd657b1 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -67,60 +67,71 @@ func (c *Consumer) logOffense(misbehavior network.Misbehavior, violation *networ // reportMisbehavior reports the slashing violation to the alsp misbehavior report manager. When violation identity // is nil this indicates the misbehavior occurred either on a public network and the identity of the sender is unknown // we can skip reporting the misbehavior. -func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) error { +// Args: +// - misbehavior: the network misbehavior. +// - violation: the slashing violation. +// Any error encountered while creating the misbehavior report is considered irrecoverable and will result in a fatal log. +func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation *network.Violation) { if violation.Identity == nil { - c.log.Debug().Bool(logging.KeySuspicious, true).Str("peerID", violation.PeerID).Msg("violation identity unknown (or public) skipping misbehavior reporting") + c.log.Debug(). + Bool(logging.KeySuspicious, true). + Str("peerID", violation.PeerID). + Msg("violation identity unknown (or public) skipping misbehavior reporting") c.metrics.OnViolationReportSkipped() - return nil + return } report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) if err != nil { - return fmt.Errorf("failed to create misbehavior report: %w", err) + c.log.Fatal(). + Err(err). + Bool(logging.KeySuspicious, true). + Str("peerID", violation.PeerID). + Msg("failed to create misbehavior report") + } c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) - return nil } // OnUnAuthorizedSenderError logs an error for unauthorized sender error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnAuthorizedSenderError(violation *network.Violation) error { +func (c *Consumer) OnUnAuthorizedSenderError(violation *network.Violation) { c.logOffense(alsp.UnAuthorizedSender, violation) - return c.reportMisbehavior(alsp.UnAuthorizedSender, violation) + c.reportMisbehavior(alsp.UnAuthorizedSender, violation) } // OnUnknownMsgTypeError logs an error for unknown message type error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnknownMsgTypeError(violation *network.Violation) error { +func (c *Consumer) OnUnknownMsgTypeError(violation *network.Violation) { c.logOffense(alsp.UnknownMsgType, violation) - return c.reportMisbehavior(alsp.UnknownMsgType, violation) + c.reportMisbehavior(alsp.UnknownMsgType, violation) } // OnInvalidMsgError logs an error for messages that contained payloads that could not // be unmarshalled into the message type denoted by message code byte and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnInvalidMsgError(violation *network.Violation) error { +func (c *Consumer) OnInvalidMsgError(violation *network.Violation) { c.logOffense(alsp.InvalidMessage, violation) - return c.reportMisbehavior(alsp.InvalidMessage, violation) + c.reportMisbehavior(alsp.InvalidMessage, violation) } // OnSenderEjectedError logs an error for sender ejected error and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnSenderEjectedError(violation *network.Violation) error { +func (c *Consumer) OnSenderEjectedError(violation *network.Violation) { c.logOffense(alsp.SenderEjected, violation) - return c.reportMisbehavior(alsp.SenderEjected, violation) + c.reportMisbehavior(alsp.SenderEjected, violation) } // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) error { +func (c *Consumer) OnUnauthorizedUnicastOnChannel(violation *network.Violation) { c.logOffense(alsp.UnauthorizedUnicastOnChannel, violation) - return c.reportMisbehavior(alsp.UnauthorizedUnicastOnChannel, violation) + c.reportMisbehavior(alsp.UnauthorizedUnicastOnChannel, violation) } // OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. -func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) error { +func (c *Consumer) OnUnauthorizedPublishOnChannel(violation *network.Violation) { c.logOffense(alsp.UnauthorizedPublishOnChannel, violation) - return c.reportMisbehavior(alsp.UnauthorizedPublishOnChannel, violation) + c.reportMisbehavior(alsp.UnauthorizedPublishOnChannel, violation) } // OnUnexpectedError logs an error for unexpected errors. This indicates message validation // has failed for an unknown reason and could potentially be n slashable offense and reports a misbehavior to alsp misbehavior report manager. -func (c *Consumer) OnUnexpectedError(violation *network.Violation) error { +func (c *Consumer) OnUnexpectedError(violation *network.Violation) { c.logOffense(alsp.UnExpectedValidationError, violation) - return c.reportMisbehavior(alsp.UnExpectedValidationError, violation) + c.reportMisbehavior(alsp.UnExpectedValidationError, violation) } diff --git a/network/validator/authorized_sender_validator.go b/network/validator/authorized_sender_validator.go index 9817e3a7d6f..6841d69a9e6 100644 --- a/network/validator/authorized_sender_validator.go +++ b/network/validator/authorized_sender_validator.go @@ -62,16 +62,14 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan identity, ok := av.getIdentity(from) if !ok { violation := &network.Violation{PeerID: from.String(), Channel: channel, Protocol: protocol, Err: ErrIdentityUnverified} - err := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) - av.checkSlashingViolationsConsumerErr(err) + av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) return "", ErrIdentityUnverified } msgCode, err := codec.MessageCodeFromPayload(payload) if err != nil { violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - av.checkSlashingViolationsConsumerErr(svcErr) + av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return "", err } @@ -81,32 +79,23 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan return msgType, nil case message.IsUnknownMsgTypeErr(err) || codec.IsErrUnknownMsgCode(err): violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) - av.checkSlashingViolationsConsumerErr(svcErr) + av.slashingViolationsConsumer.OnUnknownMsgTypeError(violation) return msgType, err case errors.Is(err, message.ErrUnauthorizedMessageOnChannel) || errors.Is(err, message.ErrUnauthorizedRole): violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) - av.checkSlashingViolationsConsumerErr(svcErr) - + av.slashingViolationsConsumer.OnUnAuthorizedSenderError(violation) return msgType, err case errors.Is(err, ErrSenderEjected): violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnSenderEjectedError(violation) - av.checkSlashingViolationsConsumerErr(svcErr) - + av.slashingViolationsConsumer.OnSenderEjectedError(violation) return msgType, err case errors.Is(err, message.ErrUnauthorizedUnicastOnChannel): violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) - av.checkSlashingViolationsConsumerErr(svcErr) - + av.slashingViolationsConsumer.OnUnauthorizedUnicastOnChannel(violation) return msgType, err case errors.Is(err, message.ErrUnauthorizedPublishOnChannel): violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnauthorizedPublishOnChannel(violation) - av.checkSlashingViolationsConsumerErr(svcErr) - + av.slashingViolationsConsumer.OnUnauthorizedPublishOnChannel(violation) return msgType, err default: // this condition should never happen and indicates there's a bug @@ -114,9 +103,7 @@ func (av *AuthorizedSenderValidator) Validate(from peer.ID, payload []byte, chan // collect slashing data because this could potentially lead to slashing err = fmt.Errorf("unexpected error during message validation: %w", err) violation := &network.Violation{OriginID: identity.NodeID, Identity: identity, PeerID: from.String(), MsgType: msgType, Channel: channel, Protocol: protocol, Err: err} - svcErr := av.slashingViolationsConsumer.OnUnexpectedError(violation) - av.checkSlashingViolationsConsumerErr(svcErr) - + av.slashingViolationsConsumer.OnUnexpectedError(violation) return msgType, err } } @@ -162,9 +149,3 @@ func (av *AuthorizedSenderValidator) isAuthorizedSender(identity *flow.Identity, return what, nil } - -func (av *AuthorizedSenderValidator) checkSlashingViolationsConsumerErr(err error) { - if err != nil { - av.log.Error().Err(err).Msg("failed to disseminate slashing violation on slashing violations consumer") - } -} diff --git a/network/violations_consumer.go b/network/violations_consumer.go index 0936e61efc9..6c3de412c77 100644 --- a/network/violations_consumer.go +++ b/network/violations_consumer.go @@ -6,28 +6,31 @@ import ( "github.com/onflow/flow-go/network/message" ) +// ViolationsConsumer logs reported slashing violation errors and reports those violations as misbehavior's to the ALSP +// misbehavior report manager. Any errors encountered while reporting the misbehavior are considered irrecoverable and +// will result in a fatal level log. type ViolationsConsumer interface { - // OnUnAuthorizedSenderError logs an error for unauthorized sender error - OnUnAuthorizedSenderError(violation *Violation) error + // OnUnAuthorizedSenderError logs an error for unauthorized sender error. + OnUnAuthorizedSenderError(violation *Violation) - // OnUnknownMsgTypeError logs an error for unknown message type error - OnUnknownMsgTypeError(violation *Violation) error + // OnUnknownMsgTypeError logs an error for unknown message type error. + OnUnknownMsgTypeError(violation *Violation) // OnInvalidMsgError logs an error for messages that contained payloads that could not // be unmarshalled into the message type denoted by message code byte. - OnInvalidMsgError(violation *Violation) error + OnInvalidMsgError(violation *Violation) - // OnSenderEjectedError logs an error for sender ejected error - OnSenderEjectedError(violation *Violation) error + // OnSenderEjectedError logs an error for sender ejected error. + OnSenderEjectedError(violation *Violation) // OnUnauthorizedUnicastOnChannel logs an error for messages unauthorized to be sent via unicast. - OnUnauthorizedUnicastOnChannel(violation *Violation) error + OnUnauthorizedUnicastOnChannel(violation *Violation) // OnUnauthorizedPublishOnChannel logs an error for messages unauthorized to be sent via pubsub. - OnUnauthorizedPublishOnChannel(violation *Violation) error + OnUnauthorizedPublishOnChannel(violation *Violation) - // OnUnexpectedError logs an error for unknown errors - OnUnexpectedError(violation *Violation) error + // OnUnexpectedError logs an error for unknown errors. + OnUnexpectedError(violation *Violation) } type Violation struct { From 33bdad8686a0edf3d97aa09123279606948552d9 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 27 Jun 2023 18:17:47 -0400 Subject: [PATCH 069/169] fix lint --- network/middleware.go | 1 + network/p2p/middleware/middleware.go | 8 -------- network/p2p/test/topic_validator_test.go | 3 ++- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/network/middleware.go b/network/middleware.go index 060d87bef4d..d8e14ee82c1 100644 --- a/network/middleware.go +++ b/network/middleware.go @@ -6,6 +6,7 @@ import ( "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/network/channels" diff --git a/network/p2p/middleware/middleware.go b/network/p2p/middleware/middleware.go index 0a71fc10c0c..a2e6dab6c70 100644 --- a/network/p2p/middleware/middleware.go +++ b/network/p2p/middleware/middleware.go @@ -61,8 +61,6 @@ const ( // LargeMsgUnicastTimeout is the maximum time to wait for a unicast request to complete for large message size LargeMsgUnicastTimeout = 1000 * time.Second - - violationDisseminateErr = "failed to disseminate slashing violation on slashing violations consumer" ) var ( @@ -817,12 +815,6 @@ func (m *Middleware) IsConnected(nodeID flow.Identifier) (bool, error) { return m.libP2PNode.IsConnected(peerID) } -func (m *Middleware) checkSlashingViolationsConsumerErr(err error) { - if err != nil { - m.log.Error().Err(err).Msg(violationDisseminateErr) - } -} - // unicastMaxMsgSize returns the max permissible size for a unicast message func unicastMaxMsgSize(messageType string) int { switch messageType { diff --git a/network/p2p/test/topic_validator_test.go b/network/p2p/test/topic_validator_test.go index adbb3822daf..5a7e402b141 100644 --- a/network/p2p/test/topic_validator_test.go +++ b/network/p2p/test/topic_validator_test.go @@ -10,6 +10,8 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/mock" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/irrecoverable" @@ -29,7 +31,6 @@ import ( "github.com/onflow/flow-go/network/validator" flowpubsub "github.com/onflow/flow-go/network/validator/pubsub" "github.com/onflow/flow-go/utils/unittest" - "github.com/stretchr/testify/mock" ) // TestTopicValidator_Unstaked tests that the libP2P node topic validator rejects unauthenticated messages on non-public channels (unstaked) From 0e76e55a09a24c425cebd6e50ddc604a5a8f94ab Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 29 Jun 2023 12:31:05 +0300 Subject: [PATCH 070/169] Refactored request forwarding according to suggestions from code review --- .../node_builder/access_node_builder.go | 4 +- cmd/observer/node_builder/observer_builder.go | 25 +- engine/access/rest/README.md | 13 +- engine/access/rest/api/api.go | 61 +- engine/access/rest/apiproxy/forwarder.go | 604 ------------------ .../rest/apiproxy/rest_proxy_handler.go | 336 ++++++++++ engine/access/rest/apiproxy/router.go | 126 ---- engine/access/rest/handler.go | 10 +- engine/access/rest/mock/rest_backend_api.go | 505 +++++++++++++++ engine/access/rest/mock/rest_server_api.go | 380 ----------- engine/access/rest/models/collection.go | 47 -- engine/access/rest/models/event.go | 28 - engine/access/rest/models/network.go | 6 - .../access/rest/models/node_version_info.go | 11 - engine/access/rest/router.go | 4 +- engine/access/rest/routes/accounts.go | 20 +- engine/access/rest/routes/blocks.go | 211 +++--- engine/access/rest/routes/collections.go | 29 +- engine/access/rest/routes/events.go | 39 +- engine/access/rest/routes/execution_result.go | 42 +- engine/access/rest/routes/network.go | 8 +- .../access/rest/routes/node_version_info.go | 11 +- engine/access/rest/routes/scripts.go | 22 +- engine/access/rest/routes/transactions.go | 43 +- engine/access/rest/server.go | 2 +- engine/access/rest/server_request_handler.go | 346 ---------- engine/access/rest/tests/accounts_test.go | 180 +----- engine/access/rest/tests/blocks_test.go | 42 +- engine/access/rest/tests/collections_test.go | 7 +- engine/access/rest/tests/events_test.go | 3 +- .../rest/tests/execution_result_test.go | 18 +- engine/access/rest/tests/network_test.go | 3 +- .../rest/tests/node_version_info_test.go | 3 +- engine/access/rest/tests/scripts_test.go | 21 +- engine/access/rest/tests/test_helpers.go | 35 +- engine/access/rest/tests/transactions_test.go | 32 +- engine/access/rpc/backend/backend.go | 1 + engine/access/rpc/engine.go | 2 +- engine/access/rpc/engine_builder.go | 9 +- engine/common/rpc/convert/convert.go | 22 + 40 files changed, 1301 insertions(+), 2010 deletions(-) delete mode 100644 engine/access/rest/apiproxy/forwarder.go create mode 100644 engine/access/rest/apiproxy/rest_proxy_handler.go delete mode 100644 engine/access/rest/apiproxy/router.go create mode 100644 engine/access/rest/mock/rest_backend_api.go delete mode 100644 engine/access/rest/mock/rest_server_api.go delete mode 100644 engine/access/rest/server_request_handler.go diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index fc69272d16a..c13a6db6f40 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -994,7 +994,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { backendConfig := config.BackendConfig accessMetrics := builder.AccessMetrics - cache, cacheSize, err := backend.NewCache(node.Logger, + backendCache, cacheSize, err := backend.NewCache(node.Logger, accessMetrics, backendConfig.ConnectionPoolSize) if err != nil { @@ -1006,7 +1006,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ExecutionGRPCPort: builder.executionGRPCPort, CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, - ConnectionsCache: cache, + ConnectionsCache: backendCache, CacheSize: cacheSize, MaxMsgSize: config.MaxMsgSize, AccessMetrics: accessMetrics, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 7a93d9e2afa..5c8e4922c7e 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "strings" "time" @@ -29,11 +28,11 @@ import ( recovery "github.com/onflow/flow-go/consensus/recovery/protocol" "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/apiproxy" - "github.com/onflow/flow-go/engine/access/rest" restapiproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/follower" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/protocol" "github.com/onflow/flow-go/model/encodable" @@ -857,7 +856,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { config := builder.rpcConf backendConfig := config.BackendConfig - cache, cacheSize, err := backend.NewCache(node.Logger, + backendCache, cacheSize, err := backend.NewCache(node.Logger, accessMetrics, config.BackendConfig.ConnectionPoolSize) if err != nil { @@ -869,7 +868,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { ExecutionGRPCPort: 0, CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, - ConnectionsCache: cache, + ConnectionsCache: backendCache, CacheSize: cacheSize, MaxMsgSize: config.MaxMsgSize, AccessMetrics: accessMetrics, @@ -914,7 +913,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { } // upstream access node forwarder - forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) + rpcForwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) if err != nil { return nil, err } @@ -924,7 +923,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, Metrics: observerCollector, - Upstream: forwarder, + Upstream: rpcForwarder, Observer: protocol.NewHandler(protocol.New( node.State, node.Storage.Blocks, @@ -932,8 +931,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { backend.NewNetworkAPI(node.State, node.RootChainID, backend.DefaultSnapshotHistoryLimit), )), } - - restForwarder, err := restapiproxy.NewRestForwarder(builder.Logger, + frw, err := forwarder.NewForwarder( builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) @@ -941,12 +939,13 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return nil, err } - restHandler := &restapiproxy.RestRouter{ - Logger: builder.Logger, - Metrics: observerCollector, - Upstream: restForwarder, - Observer: rest.NewServerRequestHandler(builder.Logger, accessBackend), + restHandler := &restapiproxy.RestProxyHandler{ + Logger: builder.Logger, + Metrics: observerCollector, + Chain: node.RootChainID.Chain(), } + restHandler.API = accessBackend + restHandler.Forwarder = frw // build the rpc engine builder.RpcEng, err = engineBuilder. diff --git a/engine/access/rest/README.md b/engine/access/rest/README.md index a972fe8c385..633acf65707 100644 --- a/engine/access/rest/README.md +++ b/engine/access/rest/README.md @@ -12,8 +12,8 @@ available on our [docs site](https://docs.onflow.org/http-api/). - `request`: Implementation of API requests that provide validation for input data and build request models. - `routes`: The common HTTP handlers for all the requests. - `api`: The server API interface for REST service. -- `apiproxy`: Implementation of proxy router which splits requests for observer node between local and request -forwarding to upstream, implementation of handling request forwarding to an upstream access node using gRPC API. +- `apiproxy`: Implementation of proxy backend handler which includes the local backend and forwards the methods which +can't be handled locally to an upstream using gRPC API. - `tests`: Test for each request. ## Request lifecycle @@ -48,13 +48,12 @@ package that complies with function interfaced defined as: ```go type ApiHandlerFunc func ( r *request.Request, -srv api.RestServerApi, +backend api.RestBackendApi, generator models.LinkGenerator, ) (interface{}, error) ``` That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Then needs -to be added new request to `RestServerApi` interface and implemented it for local API service to the `RequestHandler` and for -request forwarding to the `RestForwarder`. After that new function needs to be added to the `RestRouter` for representing -the routing proxy algorithm. Adding a new API endpoint also requires for a new request builder to be implemented and added -in request package. Make sure to not forget about adding tests for each of the API handler. +to be added new request to `RestBackendApi` interface and overrides the method if it should be proxied to the backend +handler `RestProxyHandler` for request forwarding. Adding a new API endpoint also requires for a new request builder to +be implemented and added in request package. Make sure to not forget about adding tests for each of the API handler. diff --git a/engine/access/rest/api/api.go b/engine/access/rest/api/api.go index aea539e0571..81d45cccb6e 100644 --- a/engine/access/rest/api/api.go +++ b/engine/access/rest/api/api.go @@ -3,39 +3,36 @@ package api import ( "context" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/model/flow" ) -// RestServerApi is the server API for REST service. -type RestServerApi interface { - // GetTransactionByID gets a transaction by requested ID. - GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) - // CreateTransaction creates a new transaction from provided payload. - CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) - // GetTransactionResultByID retrieves transaction result by the transaction ID. - GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) - // GetBlocksByIDs gets blocks by provided ID or list of IDs. - GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) - // GetBlocksByHeight gets blocks by provided height. - GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) - // GetBlockPayloadByID gets block payload by ID - GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) - // GetExecutionResultByID gets execution result by the ID. - GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) - // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. - GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) - // GetCollectionByID retrieves a collection by ID and builds a response - GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) - // ExecuteScript handler sends the script from the request to be executed. - ExecuteScript(r request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) - // GetAccount handler retrieves account by address and returns the response. - GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) - // GetEvents for the provided block range or list of block IDs filtered by type. - GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) - // GetNetworkParameters returns network-wide parameters of the blockchain - GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) - // GetNodeVersionInfo returns node version information - GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) +// RestBackendApi is the backend API for REST service. +type RestBackendApi interface { + GetNetworkParameters(ctx context.Context) access.NetworkParameters + GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) + + GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) + + GetLatestBlock(ctx context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) + GetBlockByHeight(ctx context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) + GetBlockByID(ctx context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) + + GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) + + SendTransaction(ctx context.Context, tx *flow.TransactionBody) error + GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) + GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) + + GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) + + ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) + ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) + ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) + + GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) + GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) + + GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) + GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) } diff --git a/engine/access/rest/apiproxy/forwarder.go b/engine/access/rest/apiproxy/forwarder.go deleted file mode 100644 index 164db705b47..00000000000 --- a/engine/access/rest/apiproxy/forwarder.go +++ /dev/null @@ -1,604 +0,0 @@ -package apiproxy - -import ( - "context" - "fmt" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/engine/access/rest/api" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/access/rest/routes" - "github.com/onflow/flow-go/engine/common/grpc/forwarder" - "github.com/onflow/flow-go/engine/common/rpc/convert" - "github.com/onflow/flow-go/model/flow" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/entities" -) - -// RestForwarder - structure which handles requests to an upstream access node using gRPC API. -type RestForwarder struct { - log zerolog.Logger - *forwarder.Forwarder -} - -var _ api.RestServerApi = (*RestForwarder)(nil) - -// NewRestForwarder returns new RestForwarder. -func NewRestForwarder(log zerolog.Logger, identities flow.IdentityList, timeout time.Duration, maxMsgSize uint) (*RestForwarder, error) { - f, err := forwarder.NewForwarder(identities, timeout, maxMsgSize) - - restForwarder := &RestForwarder{ - log: log, - Forwarder: f, - } - return restForwarder, err -} - -// GetTransactionByID gets a transaction by requested ID. -func (f *RestForwarder) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { - var response models.Transaction - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getTransactionRequest := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - } - transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) - if err != nil { - return response, err - } - - var transactionResultResponse *accessproto.TransactionResultResponse - // only lookup result if transaction result is to be expanded - if r.ExpandsResult { - getTransactionResultRequest := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - BlockId: r.BlockID[:], - CollectionId: r.CollectionID[:], - } - transactionResultResponse, err = upstream.GetTransactionResult(context, getTransactionResultRequest) - if err != nil { - return response, err - } - } - flowTransaction, err := convert.MessageToTransaction(transactionResponse.Transaction, chain) - if err != nil { - return response, err - } - - flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) - - response.Build(&flowTransaction, flowTransactionResult, link) - return response, nil -} - -// CreateTransaction creates a new transaction from provided payload. -func (f *RestForwarder) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - var response models.Transaction - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - entitiesTransaction := convert.TransactionToMessage(r.Transaction) - sendTransactionRequest := &accessproto.SendTransactionRequest{ - Transaction: entitiesTransaction, - } - - _, err = upstream.SendTransaction(context, sendTransactionRequest) - if err != nil { - return response, err - } - - response.Build(&r.Transaction, nil, link) - return response, nil -} - -// GetTransactionResultByID retrieves transaction result by the transaction ID. -func (f *RestForwarder) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - var response models.TransactionResult - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getTransactionResult := &accessproto.GetTransactionRequest{ - Id: r.ID[:], - BlockId: r.BlockID[:], - CollectionId: r.CollectionID[:], - } - transactionResultResponse, err := upstream.GetTransactionResult(context, getTransactionResult) - if err != nil { - return response, err - } - - flowTransactionResult := access.MessageToTransactionResult(transactionResultResponse) - response.Build(flowTransactionResult, r.ID, link) - return response, nil -} - -// GetBlocksByIDs gets blocks by provided ID or list of IDs. -func (f *RestForwarder) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - blocks := make([]*models.Block, len(r.IDs)) - - upstream, err := f.FaultTolerantClient() - if err != nil { - return blocks, err - } - - for i, id := range r.IDs { - block, err := getBlockFromGrpc(routes.ForID(&id), context, expandFields, upstream, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil -} - -// GetBlocksByHeight gets blocks by provided height. -func (f *RestForwarder) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - req, err := r.GetBlockRequest() - if err != nil { - return nil, models.NewBadRequestError(err) - } - - upstream, err := f.FaultTolerantClient() - if err != nil { - return nil, err - } - - if req.FinalHeight || req.SealedHeight { - block, err := getBlockFromGrpc(routes.ForFinalized(req.Heights[0]), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - - return []*models.Block{block}, nil - } - - // if the query is /blocks/height=1000,1008,1049... - if req.HasHeights() { - blocks := make([]*models.Block, len(req.Heights)) - for i, height := range req.Heights { - block, err := getBlockFromGrpc(routes.ForHeight(height), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil - } - - // support providing end height as "sealed" or "final" - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ - IsSealed: req.EndHeight == request.SealedHeight, - } - blockResponse, err := upstream.GetLatestBlock(r.Context(), getLatestBlockRequest) - if err != nil { - return nil, err - } - - req.EndHeight = blockResponse.Block.BlockHeader.Height // overwrite special value height with fetched - - if req.StartHeight > req.EndHeight { - return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) - } - } - - blocks := make([]*models.Block, 0) - // start and end height inclusive - for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := getBlockFromGrpc(routes.ForHeight(i), r.Context(), r.ExpandFields, upstream, link) - if err != nil { - return nil, err - } - blocks = append(blocks, block) - } - - return blocks, nil -} - -// GetBlockPayloadByID gets block payload by ID -func (f *RestForwarder) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { - var payload models.BlockPayload - - upstream, err := f.FaultTolerantClient() - if err != nil { - return payload, err - } - - blkProvider := routes.NewBlockFromGrpcProvider(upstream, routes.ForID(&r.ID)) - block, _, statusErr := blkProvider.GetBlock(context) - if statusErr != nil { - return payload, statusErr - } - - flowPayload, err := convert.PayloadFromMessage(block) - if err != nil { - return payload, err - } - - err = payload.Build(flowPayload) - if err != nil { - return payload, err - } - - return payload, nil -} - -// GetExecutionResultByID gets execution result by the ID. -func (f *RestForwarder) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - var response models.ExecutionResult - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ - Id: r.ID[:], - } - - executionResultByIDResponse, err := upstream.GetExecutionResultByID(context, executionResultByIDRequest) - if err != nil { - return response, err - } - - if executionResultByIDResponse == nil { - err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) - return response, models.NewNotFoundError(err.Error(), err) - } - - flowExecResult, err := convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) - if err != nil { - return response, err - } - err = response.Build(flowExecResult, link) - if err != nil { - return response, err - } - - return response, nil -} - -// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func (f *RestForwarder) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - // for each block ID we retrieve execution result - results := make([]models.ExecutionResult, len(r.BlockIDs)) - - upstream, err := f.FaultTolerantClient() - if err != nil { - return results, err - } - - for i, id := range r.BlockIDs { - getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ - BlockId: id[:], - } - executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockID) - if err != nil { - return nil, err - } - - var response models.ExecutionResult - flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) - if err != nil { - return nil, err - } - err = response.Build(flowExecResult, link) - if err != nil { - return nil, err - } - results[i] = response - } - - return results, nil -} - -// GetCollectionByID retrieves a collection by ID and builds a response -func (f *RestForwarder) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { - var response models.Collection - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ - Id: r.ID[:], - } - - collectionResponse, err := upstream.GetCollectionByID(context, getCollectionByIDRequest) - if err != nil { - return response, err - } - - // if we expand transactions in the query retrieve each transaction data - transactions := make([]*entities.Transaction, 0) - if r.ExpandsTransactions { - for _, tid := range collectionResponse.Collection.TransactionIds { - getTransactionRequest := &accessproto.GetTransactionRequest{ - Id: tid, - } - transactionResponse, err := upstream.GetTransaction(context, getTransactionRequest) - if err != nil { - return response, err - } - - transactions = append(transactions, transactionResponse.Transaction) - } - } - - err = response.BuildFromGrpc(collectionResponse.Collection, transactions, link, expandFields, chain) - if err != nil { - return response, err - } - - return response, nil -} - -// ExecuteScript handler sends the script from the request to be executed. -func (f *RestForwarder) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { - upstream, err := f.FaultTolerantClient() - if err != nil { - return nil, err - } - - if r.BlockID != flow.ZeroID { - executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ - BlockId: r.BlockID[:], - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(context, executeScriptAtBlockIDRequest) - if err != nil { - return nil, err - } - return executeScriptAtBlockIDResponse.Value, nil - } - - // default to sealed height - if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { - executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(context, executeScriptAtLatestBlockRequest) - if err != nil { - return nil, err - } - return executeScriptAtLatestBlockResponse.Value, nil - } - - if r.BlockHeight == request.FinalHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: false, - } - getLatestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return nil, err - } - r.BlockHeight = getLatestBlockHeaderResponse.Block.Height - } - - executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ - BlockHeight: r.BlockHeight, - Script: r.Script.Source, - Arguments: r.Script.Args, - } - executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(context, executeScriptAtBlockHeightRequest) - if err != nil { - return nil, err - } - return executeScriptAtBlockHeightResponse.Value, nil -} - -// GetAccount handler retrieves account by address and returns the response. -func (f *RestForwarder) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - var response models.Account - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it - if r.Height == request.FinalHeight || r.Height == request.SealedHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: r.Height == request.SealedHeight, - } - blockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return response, err - } - r.Height = blockHeaderResponse.Block.Height - } - getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ - Address: r.Address.Bytes(), - BlockHeight: r.Height, - } - - accountResponse, err := upstream.GetAccountAtBlockHeight(context, getAccountAtBlockHeightRequest) - if err != nil { - return response, models.NewNotFoundError("not found account at block height", err) - } - - flowAccount, err := convert.MessageToAccount(accountResponse.Account) - if err != nil { - return response, err - } - - err = response.Build(flowAccount, link, expandFields) - return response, err -} - -// GetEvents for the provided block range or list of block IDs filtered by type. -func (f *RestForwarder) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { - // if the request has block IDs provided then return events for block IDs - var blocksEvents models.BlocksEvents - - upstream, err := f.FaultTolerantClient() - if err != nil { - return blocksEvents, err - } - - if len(r.BlockIDs) > 0 { - var blockIds [][]byte - for _, id := range r.BlockIDs { - blockIds = append(blockIds, id[:]) - } - getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ - Type: r.Type, - BlockIds: blockIds, - } - eventsResponse, err := upstream.GetEventsForBlockIDs(context, getEventsForBlockIDsRequest) - if err != nil { - return nil, err - } - - blocksEvents.BuildFromGrpc(eventsResponse.Results) - - return blocksEvents, nil - } - - // if end height is provided with special values then load the height - if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: r.EndHeight == request.SealedHeight, - } - latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(context, getLatestBlockHeaderRequest) - if err != nil { - return nil, err - } - - r.EndHeight = latestBlockHeaderResponse.Block.Height - // special check after we resolve special height value - if r.StartHeight > r.EndHeight { - return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) - } - } - - // if request provided block height range then return events for that range - getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ - Type: r.Type, - StartHeight: r.StartHeight, - EndHeight: r.EndHeight, - } - eventsResponse, err := upstream.GetEventsForHeightRange(context, getEventsForHeightRangeRequest) - if err != nil { - return nil, err - } - - blocksEvents.BuildFromGrpc(eventsResponse.Results) - return blocksEvents, nil -} - -// GetNetworkParameters returns network-wide parameters of the blockchain -func (f *RestForwarder) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { - var response models.NetworkParameters - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getNetworkParametersRequest := &accessproto.GetNetworkParametersRequest{} - getNetworkParametersResponse, err := upstream.GetNetworkParameters(r.Context(), getNetworkParametersRequest) - if err != nil { - return response, err - } - response.BuildFromGrpc(getNetworkParametersResponse) - return response, nil -} - -// GetNodeVersionInfo returns node version information -func (f *RestForwarder) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { - var response models.NodeVersionInfo - - upstream, err := f.FaultTolerantClient() - if err != nil { - return response, err - } - - getNodeVersionInfoRequest := &accessproto.GetNodeVersionInfoRequest{} - getNodeVersionInfoResponse, err := upstream.GetNodeVersionInfo(r.Context(), getNodeVersionInfoRequest) - if err != nil { - return response, err - } - - response.BuildFromGrpc(getNodeVersionInfoResponse.Info) - return response, nil -} - -func getBlockFromGrpc(option routes.BlockRequestOption, context context.Context, expandFields map[string]bool, upstream accessproto.AccessAPIClient, link models.LinkGenerator) (*models.Block, error) { - // lookup block - blkProvider := routes.NewBlockFromGrpcProvider(upstream, option) - blk, blockStatus, err := blkProvider.GetBlock(context) - if err != nil { - return nil, err - } - - // lookup execution result - // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) - var block models.Block - getExecutionResultForBlockIDRequest := &accessproto.GetExecutionResultForBlockIDRequest{ - BlockId: blk.Id, - } - - executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(context, getExecutionResultForBlockIDRequest) - if err != nil { - return nil, err - } - - flowExecResult, err := convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) - if err != nil { - return nil, err - } - - flowBlock, err := convert.MessageToBlock(blk) - if err != nil { - return nil, err - } - - flowBlockStatus, err := convert.MessagesToBlockStatus(blockStatus) - if err != nil { - return nil, err - } - - if err != nil { - // handle case where execution result is not yet available - if se, ok := status.FromError(err); ok { - if se.Code() == codes.NotFound { - err := block.Build(flowBlock, nil, link, flowBlockStatus, expandFields) - if err != nil { - return nil, err - } - return &block, nil - } - } - return nil, err - } - - err = block.Build(flowBlock, flowExecResult, link, flowBlockStatus, expandFields) - if err != nil { - return nil, err - } - return &block, nil -} diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go new file mode 100644 index 00000000000..d4bc6546a40 --- /dev/null +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -0,0 +1,336 @@ +package apiproxy + +import ( + "context" + + "google.golang.org/grpc/status" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/models" + "github.com/onflow/flow-go/engine/common/grpc/forwarder" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" +) + +// RestProxyHandler is a structure that represents the proxy algorithm for observer node. +// It includes the local backend and forwards the methods which can't be handled locally to an upstream using gRPC API. +type RestProxyHandler struct { + access.API + *forwarder.Forwarder + Logger zerolog.Logger + Metrics metrics.ObserverMetrics + Chain flow.Chain +} + +func (r *RestProxyHandler) log(handler, rpc string, err error) { + code := status.Code(err) + r.Metrics.RecordRPC(handler, rpc, code) + + logger := r.Logger.With(). + Str("handler", handler). + Str("rest_method", rpc). + Str("rest_code", code.String()). + Logger() + + if err != nil { + logger.Error().Err(err).Msg("request failed") + return + } + + logger.Info().Msg("request succeeded") +} + +// GetLatestBlockHeader returns the latest block header and block status, if isSealed = true - returns the latest seal header. +func (r *RestProxyHandler) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { //Uliana getAccount та GetEvents були з Upstream + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, flow.BlockStatusUnknown, err + } + + getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ + IsSealed: isSealed, + } + latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(ctx, getLatestBlockHeaderRequest) + if err != nil { + return nil, flow.BlockStatusUnknown, err + } + blockHeader, err := convert.MessageToBlockHeader(latestBlockHeaderResponse.Block) + if err != nil { + return nil, flow.BlockStatusUnknown, err + } + blockStatus, err := convert.MessagesToBlockStatus(latestBlockHeaderResponse.BlockStatus) + if err != nil { + return nil, flow.BlockStatusUnknown, err + } + + r.log("upstream", "GetLatestBlockHeader", err) + return blockHeader, blockStatus, nil +} + +// GetCollectionByID returns a collection by ID. +func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getCollectionByIDRequest := &accessproto.GetCollectionByIDRequest{ + Id: id[:], + } + + collectionResponse, err := upstream.GetCollectionByID(ctx, getCollectionByIDRequest) + if err != nil { + return nil, err + } + + transactions := make([]flow.Identifier, len(collectionResponse.Collection.TransactionIds)) + for _, txId := range collectionResponse.Collection.TransactionIds { + transactions = append(transactions, convert.MessageToIdentifier(txId)) + } + + r.log("upstream", "GetCollectionByID", err) + return &flow.LightCollection{ + Transactions: transactions, + }, nil +} + +// SendTransaction sends already created transaction. +func (r *RestProxyHandler) SendTransaction(ctx context.Context, tx *flow.TransactionBody) error { + upstream, err := r.FaultTolerantClient() + if err != nil { + return err + } + + transaction := convert.TransactionToMessage(*tx) + sendTransactionRequest := &accessproto.SendTransactionRequest{ + Transaction: transaction, + } + + _, err = upstream.SendTransaction(ctx, sendTransactionRequest) + if err != nil { + return err + } + + r.log("upstream", "SendTransaction", err) + return nil + +} + +// GetTransaction returns transaction by ID. +func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getTransactionRequest := &accessproto.GetTransactionRequest{ + Id: id[:], + } + transactionResponse, err := upstream.GetTransaction(ctx, getTransactionRequest) + if err != nil { + return nil, err + } + + transactionBody, err := convert.MessageToTransaction(transactionResponse.Transaction, r.Chain) + if err != nil { + return nil, err + } + + r.log("upstream", "GetTransaction", err) + return &transactionBody, nil +} + +// GetTransactionResult returns transaction result by the transaction ID. +func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getTransactionResultRequest := &accessproto.GetTransactionRequest{ + Id: id[:], + BlockId: blockID[:], + CollectionId: collectionID[:], + } + + transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "GetTransactionResult", err) + return access.MessageToTransactionResult(transactionResultResponse), nil +} + +// GetAccountAtBlockHeight returns account by account address and block height. +func (r *RestProxyHandler) GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getAccountAtBlockHeightRequest := &accessproto.GetAccountAtBlockHeightRequest{ + Address: address.Bytes(), + BlockHeight: height, + } + + accountResponse, err := upstream.GetAccountAtBlockHeight(ctx, getAccountAtBlockHeightRequest) + if err != nil { + return nil, models.NewNotFoundError("not found account at block height", err) + } + + r.log("upstream", "GetAccountAtBlockHeight", err) + return convert.MessageToAccount(accountResponse.Account) +} + +// ExecuteScriptAtLatestBlock executes script at latest block. +func (r *RestProxyHandler) ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtLatestBlockRequest := &accessproto.ExecuteScriptAtLatestBlockRequest{ + Script: script, + Arguments: arguments, + } + executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(ctx, executeScriptAtLatestBlockRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "ExecuteScriptAtLatestBlock", err) + return executeScriptAtLatestBlockResponse.Value, nil +} + +// ExecuteScriptAtBlockHeight executes script at the given block height . +func (r *RestProxyHandler) ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtBlockHeightRequest := &accessproto.ExecuteScriptAtBlockHeightRequest{ + BlockHeight: blockHeight, + Script: script, + Arguments: arguments, + } + executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(ctx, executeScriptAtBlockHeightRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "ExecuteScriptAtBlockHeight", err) + return executeScriptAtBlockHeightResponse.Value, nil +} + +// ExecuteScriptAtBlockID executes script at the given block id . +func (r *RestProxyHandler) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executeScriptAtBlockIDRequest := &accessproto.ExecuteScriptAtBlockIDRequest{ + BlockId: blockID[:], + Script: script, + Arguments: arguments, + } + executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(ctx, executeScriptAtBlockIDRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "ExecuteScriptAtBlockID", err) + return executeScriptAtBlockIDResponse.Value, nil +} + +// GetEventsForHeightRange returns events by their name in the specified blocks heights. +func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getEventsForHeightRangeRequest := &accessproto.GetEventsForHeightRangeRequest{ + Type: eventType, + StartHeight: startHeight, + EndHeight: endHeight, + } + eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "GetEventsForHeightRange", err) + return convert.MessagesToBlockEvents(eventsResponse.Results), nil +} + +// GetEventsForBlockIDs returns events by their name in the specified block IDs. +func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + var blockIds [][]byte + for _, id := range blockIDs { + blockIds = append(blockIds, id[:]) + } + + getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ + Type: eventType, + BlockIds: blockIds, + } + eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "GetEventsForBlockIDs", err) + return convert.MessagesToBlockEvents(eventsResponse.Results), nil +} + +// GetExecutionResultForBlockID gets execution result by provided block ID. +func (r *RestProxyHandler) GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + getExecutionResultForBlockID := &accessproto.GetExecutionResultForBlockIDRequest{ + BlockId: blockID[:], + } + executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(ctx, getExecutionResultForBlockID) + if err != nil { + return nil, err + } + + r.log("upsteram", "GetExecutionResultForBlockID", err) + return convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) +} + +// GetExecutionResultByID gets execution result by its ID. +func (r *RestProxyHandler) GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) { + upstream, err := r.FaultTolerantClient() + if err != nil { + return nil, err + } + + executionResultByIDRequest := &accessproto.GetExecutionResultByIDRequest{ + Id: id[:], + } + + executionResultByIDResponse, err := upstream.GetExecutionResultByID(ctx, executionResultByIDRequest) + if err != nil { + return nil, err + } + + r.log("upstream", "GetExecutionResultByID", err) + return convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) +} diff --git a/engine/access/rest/apiproxy/router.go b/engine/access/rest/apiproxy/router.go deleted file mode 100644 index b3a3e8db086..00000000000 --- a/engine/access/rest/apiproxy/router.go +++ /dev/null @@ -1,126 +0,0 @@ -package apiproxy - -import ( - "context" - - "google.golang.org/grpc/status" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/engine/access/rest/api" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/module/metrics" -) - -// RestRouter is a structure that represents the routing proxy algorithm for observer node. -// It splits requests between a local requests and forward requests which can't be handled locally to an upstream access node. -type RestRouter struct { - Logger zerolog.Logger - Metrics metrics.ObserverMetrics - Upstream api.RestServerApi - Observer api.RestServerApi -} - -func (r *RestRouter) log(handler, rpc string, err error) { - code := status.Code(err) - r.Metrics.RecordRPC(handler, rpc, code) - - logger := r.Logger.With(). - Str("handler", handler). - Str("rest_method", rpc). - Str("rest_code", code.String()). - Logger() - - if err != nil { - logger.Error().Err(err).Msg("request failed") - return - } - - logger.Info().Msg("request succeeded") -} - -func (r *RestRouter) GetTransactionByID(req request.GetTransaction, context context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { - res, err := r.Upstream.GetTransactionByID(req, context, link, chain) - r.log("upstream", "GetNodeVersionInfo", err) - return res, err -} - -func (r *RestRouter) CreateTransaction(req request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - res, err := r.Upstream.CreateTransaction(req, context, link) - r.log("upstream", "CreateTransaction", err) - return res, err -} - -func (r *RestRouter) GetTransactionResultByID(req request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - res, err := r.Upstream.GetTransactionResultByID(req, context, link) - r.log("upstream", "GetTransactionResultByID", err) - return res, err -} - -func (r *RestRouter) GetBlocksByIDs(req request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - res, err := r.Observer.GetBlocksByIDs(req, context, expandFields, link) - r.log("observer", "GetBlocksByIDs", err) - return res, err -} - -func (r *RestRouter) GetBlocksByHeight(req *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - res, err := r.Observer.GetBlocksByHeight(req, link) - r.log("observer", "GetBlocksByHeight", err) - return res, err -} - -func (r *RestRouter) GetBlockPayloadByID(req request.GetBlockPayload, context context.Context, link models.LinkGenerator) (models.BlockPayload, error) { - res, err := r.Observer.GetBlockPayloadByID(req, context, link) - r.log("observer", "GetBlockPayloadByID", err) - return res, err -} - -func (r *RestRouter) GetExecutionResultByID(req request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - res, err := r.Upstream.GetExecutionResultByID(req, context, link) - r.log("upstream", "GetExecutionResultByID", err) - return res, err -} - -func (r *RestRouter) GetExecutionResultsByBlockIDs(req request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - res, err := r.Upstream.GetExecutionResultsByBlockIDs(req, context, link) - r.log("upstream", "GetExecutionResultsByBlockIDs", err) - return res, err -} - -func (r *RestRouter) GetCollectionByID(req request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { - res, err := r.Upstream.GetCollectionByID(req, context, expandFields, link, chain) - r.log("upstream", "GetCollectionByID", err) - return res, err -} - -func (r *RestRouter) ExecuteScript(req request.GetScript, context context.Context, link models.LinkGenerator) ([]byte, error) { - res, err := r.Upstream.ExecuteScript(req, context, link) - r.log("upstream", "ExecuteScript", err) - return res, err -} - -func (r *RestRouter) GetAccount(req request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - res, err := r.Upstream.GetAccount(req, context, expandFields, link) - r.log("upstream", "GetAccount", err) - return res, err -} - -func (r *RestRouter) GetEvents(req request.GetEvents, context context.Context) (models.BlocksEvents, error) { - res, err := r.Upstream.GetEvents(req, context) - r.log("upstream", "GetEvents", err) - return res, err -} - -func (r *RestRouter) GetNetworkParameters(req *request.Request) (models.NetworkParameters, error) { - res, err := r.Observer.GetNetworkParameters(req) - r.log("observer", "GetNetworkParameters", err) - return res, err -} - -func (r *RestRouter) GetNodeVersionInfo(req *request.Request) (models.NodeVersionInfo, error) { - res, err := r.Observer.GetNodeVersionInfo(req) - r.log("observer", "GetNodeVersionInfo", err) - return res, err -} diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index c56b5ccf114..c025a0c4c24 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -25,7 +25,7 @@ const MaxRequestSize = 2 << 20 // 2MB // it fetches necessary resources and returns an error or response model. type ApiHandlerFunc func( r *request.Request, - srv api.RestServerApi, + backend api.RestBackendApi, generator models.LinkGenerator, ) (interface{}, error) @@ -34,7 +34,7 @@ type ApiHandlerFunc func( // wraps functionality for handling error and responses outside of endpoint handling. type Handler struct { logger zerolog.Logger - restServerAPI api.RestServerApi + backend api.RestBackendApi linkGenerator models.LinkGenerator apiHandlerFunc ApiHandlerFunc chain flow.Chain @@ -42,14 +42,14 @@ type Handler struct { func NewHandler( logger zerolog.Logger, - restServerAPI api.RestServerApi, + backend api.RestBackendApi, handlerFunc ApiHandlerFunc, generator models.LinkGenerator, chain flow.Chain, ) *Handler { return &Handler{ logger: logger, - restServerAPI: restServerAPI, + backend: backend, apiHandlerFunc: handlerFunc, linkGenerator: generator, chain: chain, @@ -74,7 +74,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { decoratedRequest := request.Decorate(r, h.chain) // execute handler function and check for error - response, err := h.apiHandlerFunc(decoratedRequest, h.restServerAPI, h.linkGenerator) + response, err := h.apiHandlerFunc(decoratedRequest, h.backend, h.linkGenerator) if err != nil { h.errorHandler(w, err, errLog) return diff --git a/engine/access/rest/mock/rest_backend_api.go b/engine/access/rest/mock/rest_backend_api.go new file mode 100644 index 00000000000..61c3a938efa --- /dev/null +++ b/engine/access/rest/mock/rest_backend_api.go @@ -0,0 +1,505 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import ( + access "github.com/onflow/flow-go/access" + + context "context" + + flow "github.com/onflow/flow-go/model/flow" + + mock "github.com/stretchr/testify/mock" +) + +// RestBackendApi is an autogenerated mock type for the RestBackendApi type +type RestBackendApi struct { + mock.Mock +} + +// ExecuteScriptAtBlockHeight provides a mock function with given fields: ctx, blockHeight, script, arguments +func (_m *RestBackendApi) ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) { + ret := _m.Called(ctx, blockHeight, script, arguments) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, uint64, []byte, [][]byte) ([]byte, error)); ok { + return rf(ctx, blockHeight, script, arguments) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64, []byte, [][]byte) []byte); ok { + r0 = rf(ctx, blockHeight, script, arguments) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64, []byte, [][]byte) error); ok { + r1 = rf(ctx, blockHeight, script, arguments) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ExecuteScriptAtBlockID provides a mock function with given fields: ctx, blockID, script, arguments +func (_m *RestBackendApi) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) { + ret := _m.Called(ctx, blockID, script, arguments) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, []byte, [][]byte) ([]byte, error)); ok { + return rf(ctx, blockID, script, arguments) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, []byte, [][]byte) []byte); ok { + r0 = rf(ctx, blockID, script, arguments) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, []byte, [][]byte) error); ok { + r1 = rf(ctx, blockID, script, arguments) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ExecuteScriptAtLatestBlock provides a mock function with given fields: ctx, script, arguments +func (_m *RestBackendApi) ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) { + ret := _m.Called(ctx, script, arguments) + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte) ([]byte, error)); ok { + return rf(ctx, script, arguments) + } + if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte) []byte); ok { + r0 = rf(ctx, script, arguments) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []byte, [][]byte) error); ok { + r1 = rf(ctx, script, arguments) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAccountAtBlockHeight provides a mock function with given fields: ctx, address, height +func (_m *RestBackendApi) GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) { + ret := _m.Called(ctx, address, height) + + var r0 *flow.Account + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Address, uint64) (*flow.Account, error)); ok { + return rf(ctx, address, height) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Address, uint64) *flow.Account); ok { + r0 = rf(ctx, address, height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Account) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Address, uint64) error); ok { + r1 = rf(ctx, address, height) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetBlockByHeight provides a mock function with given fields: ctx, height +func (_m *RestBackendApi) GetBlockByHeight(ctx context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) { + ret := _m.Called(ctx, height) + + var r0 *flow.Block + var r1 flow.BlockStatus + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, uint64) (*flow.Block, flow.BlockStatus, error)); ok { + return rf(ctx, height) + } + if rf, ok := ret.Get(0).(func(context.Context, uint64) *flow.Block); ok { + r0 = rf(ctx, height) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Block) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, uint64) flow.BlockStatus); ok { + r1 = rf(ctx, height) + } else { + r1 = ret.Get(1).(flow.BlockStatus) + } + + if rf, ok := ret.Get(2).(func(context.Context, uint64) error); ok { + r2 = rf(ctx, height) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetBlockByID provides a mock function with given fields: ctx, id +func (_m *RestBackendApi) GetBlockByID(ctx context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) { + ret := _m.Called(ctx, id) + + var r0 *flow.Block + var r1 flow.BlockStatus + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.Block, flow.BlockStatus, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.Block); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Block) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) flow.BlockStatus); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Get(1).(flow.BlockStatus) + } + + if rf, ok := ret.Get(2).(func(context.Context, flow.Identifier) error); ok { + r2 = rf(ctx, id) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetCollectionByID provides a mock function with given fields: ctx, id +func (_m *RestBackendApi) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) { + ret := _m.Called(ctx, id) + + var r0 *flow.LightCollection + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.LightCollection, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.LightCollection); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.LightCollection) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs +func (_m *RestBackendApi) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, blockIDs) + + var r0 []flow.BlockEvents + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, blockIDs) + } + if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, blockIDs) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow.BlockEvents) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier) error); ok { + r1 = rf(ctx, eventType, blockIDs) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight +func (_m *RestBackendApi) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64) ([]flow.BlockEvents, error) { + ret := _m.Called(ctx, eventType, startHeight, endHeight) + + var r0 []flow.BlockEvents + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) ([]flow.BlockEvents, error)); ok { + return rf(ctx, eventType, startHeight, endHeight) + } + if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) []flow.BlockEvents); ok { + r0 = rf(ctx, eventType, startHeight, endHeight) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow.BlockEvents) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64) error); ok { + r1 = rf(ctx, eventType, startHeight, endHeight) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetExecutionResultByID provides a mock function with given fields: ctx, id +func (_m *RestBackendApi) GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) { + ret := _m.Called(ctx, id) + + var r0 *flow.ExecutionResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.ExecutionResult, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.ExecutionResult); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.ExecutionResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetExecutionResultForBlockID provides a mock function with given fields: ctx, blockID +func (_m *RestBackendApi) GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) { + ret := _m.Called(ctx, blockID) + + var r0 *flow.ExecutionResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.ExecutionResult, error)); ok { + return rf(ctx, blockID) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.ExecutionResult); ok { + r0 = rf(ctx, blockID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.ExecutionResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(ctx, blockID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetLatestBlock provides a mock function with given fields: ctx, isSealed +func (_m *RestBackendApi) GetLatestBlock(ctx context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { + ret := _m.Called(ctx, isSealed) + + var r0 *flow.Block + var r1 flow.BlockStatus + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, bool) (*flow.Block, flow.BlockStatus, error)); ok { + return rf(ctx, isSealed) + } + if rf, ok := ret.Get(0).(func(context.Context, bool) *flow.Block); ok { + r0 = rf(ctx, isSealed) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Block) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, bool) flow.BlockStatus); ok { + r1 = rf(ctx, isSealed) + } else { + r1 = ret.Get(1).(flow.BlockStatus) + } + + if rf, ok := ret.Get(2).(func(context.Context, bool) error); ok { + r2 = rf(ctx, isSealed) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetLatestBlockHeader provides a mock function with given fields: ctx, isSealed +func (_m *RestBackendApi) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { + ret := _m.Called(ctx, isSealed) + + var r0 *flow.Header + var r1 flow.BlockStatus + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, bool) (*flow.Header, flow.BlockStatus, error)); ok { + return rf(ctx, isSealed) + } + if rf, ok := ret.Get(0).(func(context.Context, bool) *flow.Header); ok { + r0 = rf(ctx, isSealed) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.Header) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, bool) flow.BlockStatus); ok { + r1 = rf(ctx, isSealed) + } else { + r1 = ret.Get(1).(flow.BlockStatus) + } + + if rf, ok := ret.Get(2).(func(context.Context, bool) error); ok { + r2 = rf(ctx, isSealed) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetNetworkParameters provides a mock function with given fields: ctx +func (_m *RestBackendApi) GetNetworkParameters(ctx context.Context) access.NetworkParameters { + ret := _m.Called(ctx) + + var r0 access.NetworkParameters + if rf, ok := ret.Get(0).(func(context.Context) access.NetworkParameters); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(access.NetworkParameters) + } + + return r0 +} + +// GetNodeVersionInfo provides a mock function with given fields: ctx +func (_m *RestBackendApi) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { + ret := _m.Called(ctx) + + var r0 *access.NodeVersionInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*access.NodeVersionInfo, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *access.NodeVersionInfo); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.NodeVersionInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTransaction provides a mock function with given fields: ctx, id +func (_m *RestBackendApi) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) { + ret := _m.Called(ctx, id) + + var r0 *flow.TransactionBody + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.TransactionBody, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.TransactionBody); ok { + r0 = rf(ctx, id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.TransactionBody) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID +func (_m *RestBackendApi) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { + ret := _m.Called(ctx, id, blockID, collectionID) + + var r0 *access.TransactionResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) (*access.TransactionResult, error)); ok { + return rf(ctx, id, blockID, collectionID) + } + if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) *access.TransactionResult); ok { + r0 = rf(ctx, id, blockID, collectionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*access.TransactionResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) error); ok { + r1 = rf(ctx, id, blockID, collectionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SendTransaction provides a mock function with given fields: ctx, tx +func (_m *RestBackendApi) SendTransaction(ctx context.Context, tx *flow.TransactionBody) error { + ret := _m.Called(ctx, tx) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, *flow.TransactionBody) error); ok { + r0 = rf(ctx, tx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +type mockConstructorTestingTNewRestBackendApi interface { + mock.TestingT + Cleanup(func()) +} + +// NewRestBackendApi creates a new instance of RestBackendApi. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRestBackendApi(t mockConstructorTestingTNewRestBackendApi) *RestBackendApi { + mock := &RestBackendApi{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/engine/access/rest/mock/rest_server_api.go b/engine/access/rest/mock/rest_server_api.go deleted file mode 100644 index fb87307189c..00000000000 --- a/engine/access/rest/mock/rest_server_api.go +++ /dev/null @@ -1,380 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mock - -import ( - context "context" - - flow "github.com/onflow/flow-go/model/flow" - mock "github.com/stretchr/testify/mock" - - models "github.com/onflow/flow-go/engine/access/rest/models" - - request "github.com/onflow/flow-go/engine/access/rest/request" -) - -// RestServerApi is an autogenerated mock type for the RestServerApi type -type RestServerApi struct { - mock.Mock -} - -// CreateTransaction provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) CreateTransaction(r request.CreateTransaction, _a1 context.Context, link models.LinkGenerator) (models.Transaction, error) { - ret := _m.Called(r, _a1, link) - - var r0 models.Transaction - var r1 error - if rf, ok := ret.Get(0).(func(request.CreateTransaction, context.Context, models.LinkGenerator) (models.Transaction, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.CreateTransaction, context.Context, models.LinkGenerator) models.Transaction); ok { - r0 = rf(r, _a1, link) - } else { - r0 = ret.Get(0).(models.Transaction) - } - - if rf, ok := ret.Get(1).(func(request.CreateTransaction, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ExecuteScript provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) ExecuteScript(r request.GetScript, _a1 context.Context, link models.LinkGenerator) ([]byte, error) { - ret := _m.Called(r, _a1, link) - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func(request.GetScript, context.Context, models.LinkGenerator) ([]byte, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.GetScript, context.Context, models.LinkGenerator) []byte); ok { - r0 = rf(r, _a1, link) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func(request.GetScript, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetAccount provides a mock function with given fields: r, _a1, expandFields, link -func (_m *RestServerApi) GetAccount(r request.GetAccount, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - ret := _m.Called(r, _a1, expandFields, link) - - var r0 models.Account - var r1 error - if rf, ok := ret.Get(0).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) (models.Account, error)); ok { - return rf(r, _a1, expandFields, link) - } - if rf, ok := ret.Get(0).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) models.Account); ok { - r0 = rf(r, _a1, expandFields, link) - } else { - r0 = ret.Get(0).(models.Account) - } - - if rf, ok := ret.Get(1).(func(request.GetAccount, context.Context, map[string]bool, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, expandFields, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetBlockPayloadByID provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) GetBlockPayloadByID(r request.GetBlockPayload, _a1 context.Context, link models.LinkGenerator) (models.BlockPayload, error) { - ret := _m.Called(r, _a1, link) - - var r0 models.BlockPayload - var r1 error - if rf, ok := ret.Get(0).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) (models.BlockPayload, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) models.BlockPayload); ok { - r0 = rf(r, _a1, link) - } else { - r0 = ret.Get(0).(models.BlockPayload) - } - - if rf, ok := ret.Get(1).(func(request.GetBlockPayload, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetBlocksByHeight provides a mock function with given fields: r, link -func (_m *RestServerApi) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - ret := _m.Called(r, link) - - var r0 []*models.Block - var r1 error - if rf, ok := ret.Get(0).(func(*request.Request, models.LinkGenerator) ([]*models.Block, error)); ok { - return rf(r, link) - } - if rf, ok := ret.Get(0).(func(*request.Request, models.LinkGenerator) []*models.Block); ok { - r0 = rf(r, link) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*models.Block) - } - } - - if rf, ok := ret.Get(1).(func(*request.Request, models.LinkGenerator) error); ok { - r1 = rf(r, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetBlocksByIDs provides a mock function with given fields: r, _a1, expandFields, link -func (_m *RestServerApi) GetBlocksByIDs(r request.GetBlockByIDs, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - ret := _m.Called(r, _a1, expandFields, link) - - var r0 []*models.Block - var r1 error - if rf, ok := ret.Get(0).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) ([]*models.Block, error)); ok { - return rf(r, _a1, expandFields, link) - } - if rf, ok := ret.Get(0).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) []*models.Block); ok { - r0 = rf(r, _a1, expandFields, link) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*models.Block) - } - } - - if rf, ok := ret.Get(1).(func(request.GetBlockByIDs, context.Context, map[string]bool, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, expandFields, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetCollectionByID provides a mock function with given fields: r, _a1, expandFields, link, chain -func (_m *RestServerApi) GetCollectionByID(r request.GetCollection, _a1 context.Context, expandFields map[string]bool, link models.LinkGenerator, chain flow.Chain) (models.Collection, error) { - ret := _m.Called(r, _a1, expandFields, link, chain) - - var r0 models.Collection - var r1 error - if rf, ok := ret.Get(0).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) (models.Collection, error)); ok { - return rf(r, _a1, expandFields, link, chain) - } - if rf, ok := ret.Get(0).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) models.Collection); ok { - r0 = rf(r, _a1, expandFields, link, chain) - } else { - r0 = ret.Get(0).(models.Collection) - } - - if rf, ok := ret.Get(1).(func(request.GetCollection, context.Context, map[string]bool, models.LinkGenerator, flow.Chain) error); ok { - r1 = rf(r, _a1, expandFields, link, chain) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetEvents provides a mock function with given fields: r, _a1 -func (_m *RestServerApi) GetEvents(r request.GetEvents, _a1 context.Context) (models.BlocksEvents, error) { - ret := _m.Called(r, _a1) - - var r0 models.BlocksEvents - var r1 error - if rf, ok := ret.Get(0).(func(request.GetEvents, context.Context) (models.BlocksEvents, error)); ok { - return rf(r, _a1) - } - if rf, ok := ret.Get(0).(func(request.GetEvents, context.Context) models.BlocksEvents); ok { - r0 = rf(r, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(models.BlocksEvents) - } - } - - if rf, ok := ret.Get(1).(func(request.GetEvents, context.Context) error); ok { - r1 = rf(r, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetExecutionResultByID provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) GetExecutionResultByID(r request.GetExecutionResult, _a1 context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - ret := _m.Called(r, _a1, link) - - var r0 models.ExecutionResult - var r1 error - if rf, ok := ret.Get(0).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) (models.ExecutionResult, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) models.ExecutionResult); ok { - r0 = rf(r, _a1, link) - } else { - r0 = ret.Get(0).(models.ExecutionResult) - } - - if rf, ok := ret.Get(1).(func(request.GetExecutionResult, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetExecutionResultsByBlockIDs provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, _a1 context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - ret := _m.Called(r, _a1, link) - - var r0 []models.ExecutionResult - var r1 error - if rf, ok := ret.Get(0).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) ([]models.ExecutionResult, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) []models.ExecutionResult); ok { - r0 = rf(r, _a1, link) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]models.ExecutionResult) - } - } - - if rf, ok := ret.Get(1).(func(request.GetExecutionResultByBlockIDs, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNetworkParameters provides a mock function with given fields: r -func (_m *RestServerApi) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { - ret := _m.Called(r) - - var r0 models.NetworkParameters - var r1 error - if rf, ok := ret.Get(0).(func(*request.Request) (models.NetworkParameters, error)); ok { - return rf(r) - } - if rf, ok := ret.Get(0).(func(*request.Request) models.NetworkParameters); ok { - r0 = rf(r) - } else { - r0 = ret.Get(0).(models.NetworkParameters) - } - - if rf, ok := ret.Get(1).(func(*request.Request) error); ok { - r1 = rf(r) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetNodeVersionInfo provides a mock function with given fields: r -func (_m *RestServerApi) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { - ret := _m.Called(r) - - var r0 models.NodeVersionInfo - var r1 error - if rf, ok := ret.Get(0).(func(*request.Request) (models.NodeVersionInfo, error)); ok { - return rf(r) - } - if rf, ok := ret.Get(0).(func(*request.Request) models.NodeVersionInfo); ok { - r0 = rf(r) - } else { - r0 = ret.Get(0).(models.NodeVersionInfo) - } - - if rf, ok := ret.Get(1).(func(*request.Request) error); ok { - r1 = rf(r) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetTransactionByID provides a mock function with given fields: r, _a1, link, chain -func (_m *RestServerApi) GetTransactionByID(r request.GetTransaction, _a1 context.Context, link models.LinkGenerator, chain flow.Chain) (models.Transaction, error) { - ret := _m.Called(r, _a1, link, chain) - - var r0 models.Transaction - var r1 error - if rf, ok := ret.Get(0).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) (models.Transaction, error)); ok { - return rf(r, _a1, link, chain) - } - if rf, ok := ret.Get(0).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) models.Transaction); ok { - r0 = rf(r, _a1, link, chain) - } else { - r0 = ret.Get(0).(models.Transaction) - } - - if rf, ok := ret.Get(1).(func(request.GetTransaction, context.Context, models.LinkGenerator, flow.Chain) error); ok { - r1 = rf(r, _a1, link, chain) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetTransactionResultByID provides a mock function with given fields: r, _a1, link -func (_m *RestServerApi) GetTransactionResultByID(r request.GetTransactionResult, _a1 context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - ret := _m.Called(r, _a1, link) - - var r0 models.TransactionResult - var r1 error - if rf, ok := ret.Get(0).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) (models.TransactionResult, error)); ok { - return rf(r, _a1, link) - } - if rf, ok := ret.Get(0).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) models.TransactionResult); ok { - r0 = rf(r, _a1, link) - } else { - r0 = ret.Get(0).(models.TransactionResult) - } - - if rf, ok := ret.Get(1).(func(request.GetTransactionResult, context.Context, models.LinkGenerator) error); ok { - r1 = rf(r, _a1, link) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewRestServerApi interface { - mock.TestingT - Cleanup(func()) -} - -// NewRestServerApi creates a new instance of RestServerApi. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewRestServerApi(t mockConstructorTestingTNewRestServerApi) *RestServerApi { - mock := &RestServerApi{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/engine/access/rest/models/collection.go b/engine/access/rest/models/collection.go index d4979146c7c..c5076fdc7db 100644 --- a/engine/access/rest/models/collection.go +++ b/engine/access/rest/models/collection.go @@ -1,14 +1,10 @@ package models import ( - "encoding/hex" "fmt" "github.com/onflow/flow-go/engine/access/rest/util" - "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" - - "github.com/onflow/flow/protobuf/go/flow/entities" ) const ExpandsTransactions = "transactions" @@ -46,49 +42,6 @@ func (c *Collection) Build( return nil } -func (c *Collection) BuildFromGrpc( - collection *entities.Collection, - txs []*entities.Transaction, - link LinkGenerator, - expand map[string]bool, - chain flow.Chain) error { - - self, err := SelfLink(convert.MessageToIdentifier(collection.Id), link.CollectionLink) - if err != nil { - return err - } - - transactionsBody := make([]*flow.TransactionBody, 0) - for _, tx := range txs { - flowTransaction, err := convert.MessageToTransaction(tx, chain) - if err != nil { - return err - } - transactionsBody = append(transactionsBody, &flowTransaction) - } - - var expandable CollectionExpandable - var transactions Transactions - if expand[ExpandsTransactions] { - transactions.Build(transactionsBody, link) - } else { - expandable.Transactions = make([]string, len(collection.TransactionIds)) - for i, id := range collection.TransactionIds { - expandable.Transactions[i], err = link.TransactionLink(convert.MessageToIdentifier(id)) - if err != nil { - return err - } - } - } - - c.Id = hex.EncodeToString(collection.Id) - c.Transactions = transactions - c.Links = self - c.Expandable = &expandable - - return nil -} - func (c *CollectionGuarantee) Build(guarantee *flow.CollectionGuarantee) { c.CollectionId = guarantee.CollectionID.String() c.SignerIndices = fmt.Sprintf("%x", guarantee.SignerIndices) diff --git a/engine/access/rest/models/event.go b/engine/access/rest/models/event.go index d829ec862dc..929dbb3f42c 100644 --- a/engine/access/rest/models/event.go +++ b/engine/access/rest/models/event.go @@ -1,13 +1,8 @@ package models import ( - "encoding/hex" - "github.com/onflow/flow-go/engine/access/rest/util" - "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func (e *Event) Build(event flow.Event) { @@ -53,26 +48,3 @@ func (b *BlocksEvents) Build(blocksEvents []flow.BlockEvents) { *b = evs } - -func (b *BlocksEvents) BuildFromGrpc(blocksEvents []*accessproto.EventsResponse_Result) { - evs := make([]BlockEvents, 0) - for _, ev := range blocksEvents { - var blockEvent BlockEvents - blockEvent.BuildFromGrpc(ev) - evs = append(evs, blockEvent) - } - - *b = evs -} - -func (b *BlockEvents) BuildFromGrpc(blockEvents *accessproto.EventsResponse_Result) { - b.BlockHeight = util.FromUint64(blockEvents.BlockHeight) - b.BlockId = hex.EncodeToString(blockEvents.BlockId) - b.BlockTimestamp = blockEvents.BlockTimestamp.AsTime() - - var events Events - flowEvents := convert.MessagesToEvents(blockEvents.Events) - events.Build(flowEvents) - b.Events = events - -} diff --git a/engine/access/rest/models/network.go b/engine/access/rest/models/network.go index 1a6dd9a9816..927b5a23362 100644 --- a/engine/access/rest/models/network.go +++ b/engine/access/rest/models/network.go @@ -2,14 +2,8 @@ package models import ( "github.com/onflow/flow-go/access" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" ) func (t *NetworkParameters) Build(params *access.NetworkParameters) { t.ChainId = params.ChainID.String() } - -func (t *NetworkParameters) BuildFromGrpc(response *accessproto.GetNetworkParametersResponse) { - t.ChainId = response.ChainId -} diff --git a/engine/access/rest/models/node_version_info.go b/engine/access/rest/models/node_version_info.go index 782493c0ec9..6a85e9f8d42 100644 --- a/engine/access/rest/models/node_version_info.go +++ b/engine/access/rest/models/node_version_info.go @@ -1,12 +1,8 @@ package models import ( - "encoding/hex" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/util" - - "github.com/onflow/flow/protobuf/go/flow/entities" ) func (t *NodeVersionInfo) Build(params *access.NodeVersionInfo) { @@ -15,10 +11,3 @@ func (t *NodeVersionInfo) Build(params *access.NodeVersionInfo) { t.SporkId = params.SporkId.String() t.ProtocolVersion = util.FromUint64(params.ProtocolVersion) } - -func (t *NodeVersionInfo) BuildFromGrpc(params *entities.NodeVersionInfo) { - t.Semver = params.Semver - t.Commit = params.Commit - t.SporkId = hex.EncodeToString(params.SporkId) - t.ProtocolVersion = util.FromUint64(params.ProtocolVersion) -} diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index 2253d56033c..ef1f9665a91 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -14,7 +14,7 @@ import ( "github.com/onflow/flow-go/module" ) -func NewRouter(serverAPI api.RestServerApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { +func NewRouter(backend api.RestBackendApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { router := mux.NewRouter().StrictSlash(true) v1SubRouter := router.PathPrefix("/v1").Subrouter() @@ -27,7 +27,7 @@ func NewRouter(serverAPI api.RestServerApi, logger zerolog.Logger, chain flow.Ch linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) for _, r := range Routes { - h := NewHandler(logger, serverAPI, r.Handler, linkGenerator, chain) + h := NewHandler(logger, backend, r.Handler, linkGenerator, chain) v1SubRouter. Methods(r.Method). Path(r.Pattern). diff --git a/engine/access/rest/routes/accounts.go b/engine/access/rest/routes/accounts.go index 76ff9d7fcb5..d9f16562f1d 100644 --- a/engine/access/rest/routes/accounts.go +++ b/engine/access/rest/routes/accounts.go @@ -7,11 +7,27 @@ import ( ) // GetAccount handler retrieves account by address and returns the response -func GetAccount(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetAccount(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetAccountRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetAccount(req, r.Context(), r.ExpandFields, link) + // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it + if req.Height == request.FinalHeight || req.Height == request.SealedHeight { + header, _, err := backend.GetLatestBlockHeader(r.Context(), req.Height == request.SealedHeight) + if err != nil { + return nil, err + } + req.Height = header.Height + } + + account, err := backend.GetAccountAtBlockHeight(r.Context(), req.Address, req.Height) + if err != nil { + return nil, err + } + + var response models.Account + err = response.Build(account, link, r.ExpandFields) + return response, err } diff --git a/engine/access/rest/routes/blocks.go b/engine/access/rest/routes/blocks.go index b6676d7076d..7f0537a0c07 100644 --- a/engine/access/rest/routes/blocks.go +++ b/engine/access/rest/routes/blocks.go @@ -8,45 +8,115 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" - - accessproto "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/entities" ) // GetBlocksByIDs gets blocks by provided ID or list of IDs. -func GetBlocksByIDs(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByIDs(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockByIDsRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetBlocksByIDs(req, r.Context(), r.ExpandFields, link) + blocks := make([]*models.Block, len(req.IDs)) + + for i, id := range req.IDs { + block, err := getBlock(forID(&id), r, backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil } // GetBlocksByHeight gets blocks by height. -func GetBlocksByHeight(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { - return srv.GetBlocksByHeight(r, link) +func GetBlocksByHeight(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { + req, err := r.GetBlockRequest() + if err != nil { + return nil, models.NewBadRequestError(err) + } + + if req.FinalHeight || req.SealedHeight { + block, err := getBlock(forFinalized(req.Heights[0]), r, backend, link) + if err != nil { + return nil, err + } + + return []*models.Block{block}, nil + } + + // if the query is /blocks/height=1000,1008,1049... + if req.HasHeights() { + blocks := make([]*models.Block, len(req.Heights)) + for i, h := range req.Heights { + block, err := getBlock(forHeight(h), r, backend, link) + if err != nil { + return nil, err + } + blocks[i] = block + } + + return blocks, nil + } + + // support providing end height as "sealed" or "final" + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + latest, _, err := backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + req.EndHeight = latest.Header.Height // overwrite special value height with fetched + + if req.StartHeight > req.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) + } + } + + blocks := make([]*models.Block, 0) + // start and end height inclusive + for i := req.StartHeight; i <= req.EndHeight; i++ { + block, err := getBlock(forHeight(i), r, backend, link) + if err != nil { + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil } // GetBlockPayloadByID gets block payload by ID -func GetBlockPayloadByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetBlockPayloadByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockPayloadRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetBlockPayloadByID(req, r.Context(), link) + blkProvider := NewBlockProvider(backend, forID(&req.ID)) + blk, _, statusErr := blkProvider.getBlock(r.Context()) + if statusErr != nil { + return nil, statusErr + } + + var payload models.BlockPayload + err = payload.Build(blk.Payload) + if err != nil { + return nil, err + } + + return payload, nil } -func GetBlock(option BlockRequestOption, context context.Context, expandFields map[string]bool, backend access.API, link models.LinkGenerator) (*models.Block, error) { +func getBlock(option blockProviderOption, req *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (*models.Block, error) { // lookup block - blkProvider := NewBlockRequestProvider(backend, option) - blk, blockStatus, err := blkProvider.GetBlock(context) + blkProvider := NewBlockProvider(backend, option) + blk, blockStatus, err := blkProvider.getBlock(req.Context()) if err != nil { return nil, err } @@ -54,12 +124,12 @@ func GetBlock(option BlockRequestOption, context context.Context, expandFields m // lookup execution result // (even if not specified as expandable, since we need the execution result ID to generate its expandable link) var block models.Block - executionResult, err := backend.GetExecutionResultForBlockID(context, blk.ID()) + executionResult, err := backend.GetExecutionResultForBlockID(req.Context(), blk.ID()) if err != nil { // handle case where execution result is not yet available if se, ok := status.FromError(err); ok { if se.Code() == codes.NotFound { - err := block.Build(blk, nil, link, blockStatus, expandFields) + err := block.Build(blk, nil, link, blockStatus, req.ExpandFields) if err != nil { return nil, err } @@ -69,64 +139,60 @@ func GetBlock(option BlockRequestOption, context context.Context, expandFields m return nil, err } - err = block.Build(blk, executionResult, link, blockStatus, expandFields) + err = block.Build(blk, executionResult, link, blockStatus, req.ExpandFields) if err != nil { return nil, err } return &block, nil } -type blockRequest struct { - id *flow.Identifier - height uint64 - latest bool - sealed bool +// blockProvider is a layer of abstraction on top of the backend api.RestBackendApi and provides a uniform way to +// look up a block or a block header either by ID or by height +type blockProvider struct { + id *flow.Identifier + height uint64 + latest bool + sealed bool + backend api.RestBackendApi } -type BlockRequestOption func(blkRequest *blockRequest) +type blockProviderOption func(blkProvider *blockProvider) -func ForID(id *flow.Identifier) BlockRequestOption { - return func(blockRequest *blockRequest) { - blockRequest.id = id +func forID(id *flow.Identifier) blockProviderOption { + return func(blkProvider *blockProvider) { + blkProvider.id = id } } -func ForHeight(height uint64) BlockRequestOption { - return func(blockRequest *blockRequest) { - blockRequest.height = height +func forHeight(height uint64) blockProviderOption { + return func(blkProvider *blockProvider) { + blkProvider.height = height } } -func ForFinalized(queryParam uint64) BlockRequestOption { - return func(blockRequest *blockRequest) { +func forFinalized(queryParam uint64) blockProviderOption { + return func(blkProvider *blockProvider) { switch queryParam { case request.SealedHeight: - blockRequest.sealed = true + blkProvider.sealed = true fallthrough case request.FinalHeight: - blockRequest.latest = true + blkProvider.latest = true } } } -// blockRequestProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to -// look up a block or a block header either by ID or by height -type blockRequestProvider struct { - blockRequest - backend access.API -} - -func NewBlockRequestProvider(backend access.API, options ...BlockRequestOption) *blockRequestProvider { - blockRequestProvider := &blockRequestProvider{ +func NewBlockProvider(backend api.RestBackendApi, options ...blockProviderOption) *blockProvider { + blkProvider := &blockProvider{ backend: backend, } for _, o := range options { - o(&blockRequestProvider.blockRequest) + o(blkProvider) } - return blockRequestProvider + return blkProvider } -func (blkProvider *blockRequestProvider) GetBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { +func (blkProvider *blockProvider) getBlock(ctx context.Context) (*flow.Block, flow.BlockStatus, error) { if blkProvider.id != nil { blk, _, err := blkProvider.backend.GetBlockByID(ctx, *blkProvider.id) if err != nil { // unfortunately backend returns internal error status if not found @@ -154,60 +220,3 @@ func (blkProvider *blockRequestProvider) GetBlock(ctx context.Context) (*flow.Bl } return blk, status, nil } - -// BlockFromGrpcProvider is a layer of abstraction on top of the accessproto.AccessAPIClient and provides a uniform way to -// look up a block or a block header either by ID or by height -type BlockFromGrpcProvider struct { - blockRequest - upstream accessproto.AccessAPIClient -} - -func NewBlockFromGrpcProvider(upstream accessproto.AccessAPIClient, options ...BlockRequestOption) *BlockFromGrpcProvider { - blockFromGrpcProvider := &BlockFromGrpcProvider{ - upstream: upstream, - } - - for _, o := range options { - o(&blockFromGrpcProvider.blockRequest) - } - return blockFromGrpcProvider -} - -func (blkProvider *BlockFromGrpcProvider) GetBlock(ctx context.Context) (*entities.Block, entities.BlockStatus, error) { - if blkProvider.id != nil { - getBlockByIdRequest := &accessproto.GetBlockByIDRequest{ - Id: []byte(blkProvider.id.String()), - } - blockResponse, err := blkProvider.upstream.GetBlockByID(ctx, getBlockByIdRequest) - if err != nil { // unfortunately grpc returns internal error status if not found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewNotFoundError( - fmt.Sprintf("error looking up block with ID %s", blkProvider.id.String()), err, - ) - } - return blockResponse.Block, entities.BlockStatus_BLOCK_UNKNOWN, nil - } - - if blkProvider.latest { - getLatestBlockRequest := &accessproto.GetLatestBlockRequest{ - IsSealed: blkProvider.sealed, - } - blockResponse, err := blkProvider.upstream.GetLatestBlock(ctx, getLatestBlockRequest) - if err != nil { - // cannot be a 'not found' error since final and sealed block should always be found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewRestError(http.StatusInternalServerError, "block lookup failed", err) - } - return blockResponse.Block, blockResponse.BlockStatus, nil - } - - getBlockByHeight := &accessproto.GetBlockByHeightRequest{ - Height: blkProvider.height, - FullBlockResponse: true, - } - blockResponse, err := blkProvider.upstream.GetBlockByHeight(ctx, getBlockByHeight) - if err != nil { // unfortunately grpc returns internal error status if not found - return nil, entities.BlockStatus_BLOCK_UNKNOWN, models.NewNotFoundError( - fmt.Sprintf("error looking up block at height %d", blkProvider.height), err, - ) - } - return blockResponse.Block, blockResponse.BlockStatus, nil -} diff --git a/engine/access/rest/routes/collections.go b/engine/access/rest/routes/collections.go index 97a38005961..182581f90f0 100644 --- a/engine/access/rest/routes/collections.go +++ b/engine/access/rest/routes/collections.go @@ -4,14 +4,39 @@ import ( "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/model/flow" ) // GetCollectionByID retrieves a collection by ID and builds a response -func GetCollectionByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetCollectionByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetCollectionRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetCollectionByID(req, r.Context(), r.ExpandFields, link, r.Chain) + collection, err := backend.GetCollectionByID(r.Context(), req.ID) + if err != nil { + return nil, err + } + + // if we expand transactions in the query retrieve each transaction data + transactions := make([]*flow.TransactionBody, 0) + if req.ExpandsTransactions { + for _, tid := range collection.Transactions { + tx, err := backend.GetTransaction(r.Context(), tid) + if err != nil { + return nil, err + } + + transactions = append(transactions, tx) + } + } + + var response models.Collection + err = response.Build(collection, transactions, link, r.ExpandFields) + if err != nil { + return nil, err + } + + return response, nil } diff --git a/engine/access/rest/routes/events.go b/engine/access/rest/routes/events.go index fd728bc1188..36eee728386 100644 --- a/engine/access/rest/routes/events.go +++ b/engine/access/rest/routes/events.go @@ -1,6 +1,8 @@ package routes import ( + "fmt" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" @@ -10,11 +12,44 @@ const BlockQueryParam = "block_ids" const EventTypeQuery = "type" // GetEvents for the provided block range or list of block IDs filtered by type. -func GetEvents(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { +func GetEvents(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetEventsRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetEvents(req, r.Context()) + // if the request has block IDs provided then return events for block IDs + var blocksEvents models.BlocksEvents + if len(req.BlockIDs) > 0 { + events, err := backend.GetEventsForBlockIDs(r.Context(), req.Type, req.BlockIDs) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil + } + + // if end height is provided with special values then load the height + if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { + latest, _, err := backend.GetLatestBlockHeader(r.Context(), req.EndHeight == request.SealedHeight) + if err != nil { + return nil, err + } + + req.EndHeight = latest.Height + // special check after we resolve special height value + if req.StartHeight > req.EndHeight { + return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) + } + } + + // if request provided block height range then return events for that range + events, err := backend.GetEventsForHeightRange(r.Context(), req.Type, req.StartHeight, req.EndHeight) + if err != nil { + return nil, err + } + + blocksEvents.Build(events) + return blocksEvents, nil } diff --git a/engine/access/rest/routes/execution_result.go b/engine/access/rest/routes/execution_result.go index 4c1ba8aab7e..f923ce7905c 100644 --- a/engine/access/rest/routes/execution_result.go +++ b/engine/access/rest/routes/execution_result.go @@ -1,27 +1,61 @@ package routes import ( + "fmt" + "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func GetExecutionResultsByBlockIDs(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultsByBlockIDs(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultByBlockIDsRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetExecutionResultsByBlockIDs(req, r.Context(), link) + // for each block ID we retrieve execution result + results := make([]models.ExecutionResult, len(req.BlockIDs)) + for i, id := range req.BlockIDs { + res, err := backend.GetExecutionResultForBlockID(r.Context(), id) + if err != nil { + return nil, err + } + + var response models.ExecutionResult + err = response.Build(res, link) + if err != nil { + return nil, err + } + results[i] = response + } + + return results, nil } // GetExecutionResultByID gets execution result by the ID. -func GetExecutionResultByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetExecutionResultByID(req, r.Context(), link) + res, err := backend.GetExecutionResultByID(r.Context(), req.ID) + if err != nil { + return nil, err + } + + if res == nil { + err := fmt.Errorf("execution result with ID: %s not found", req.ID.String()) + return nil, models.NewNotFoundError(err.Error(), err) + } + + var response models.ExecutionResult + err = response.Build(res, link) + if err != nil { + return nil, err + } + + return response, nil } diff --git a/engine/access/rest/routes/network.go b/engine/access/rest/routes/network.go index a409fd4b893..88aa787108e 100644 --- a/engine/access/rest/routes/network.go +++ b/engine/access/rest/routes/network.go @@ -7,6 +7,10 @@ import ( ) // GetNetworkParameters returns network-wide parameters of the blockchain -func GetNetworkParameters(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { - return srv.GetNetworkParameters(r) +func GetNetworkParameters(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { + params := backend.GetNetworkParameters(r.Context()) + + var response models.NetworkParameters + response.Build(¶ms) + return response, nil } diff --git a/engine/access/rest/routes/node_version_info.go b/engine/access/rest/routes/node_version_info.go index 14fbd8bbf26..daf658e8869 100644 --- a/engine/access/rest/routes/node_version_info.go +++ b/engine/access/rest/routes/node_version_info.go @@ -7,6 +7,13 @@ import ( ) // GetNodeVersionInfo returns node version information -func GetNodeVersionInfo(r *request.Request, srv api.RestServerApi, _ models.LinkGenerator) (interface{}, error) { - return srv.GetNodeVersionInfo(r) +func GetNodeVersionInfo(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { + params, err := backend.GetNodeVersionInfo(r.Context()) + if err != nil { + return nil, err + } + + var response models.NodeVersionInfo + response.Build(params) + return response, nil } diff --git a/engine/access/rest/routes/scripts.go b/engine/access/rest/routes/scripts.go index 1debbb07934..e991dd3a89e 100644 --- a/engine/access/rest/routes/scripts.go +++ b/engine/access/rest/routes/scripts.go @@ -4,14 +4,32 @@ import ( "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" + "github.com/onflow/flow-go/model/flow" ) // ExecuteScript handler sends the script from the request to be executed. -func ExecuteScript(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func ExecuteScript(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetScriptRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.ExecuteScript(req, r.Context(), link) + if req.BlockID != flow.ZeroID { + return backend.ExecuteScriptAtBlockID(r.Context(), req.BlockID, req.Script.Source, req.Script.Args) + } + + // default to sealed height + if req.BlockHeight == request.SealedHeight || req.BlockHeight == request.EmptyHeight { + return backend.ExecuteScriptAtLatestBlock(r.Context(), req.Script.Source, req.Script.Args) + } + + if req.BlockHeight == request.FinalHeight { + finalBlock, _, err := backend.GetLatestBlockHeader(r.Context(), false) + if err != nil { + return nil, err + } + req.BlockHeight = finalBlock.Height + } + + return backend.ExecuteScriptAtBlockHeight(r.Context(), req.BlockHeight, req.Script.Source, req.Script.Args) } diff --git a/engine/access/rest/routes/transactions.go b/engine/access/rest/routes/transactions.go index 9f9f49d8cf1..763e9098299 100644 --- a/engine/access/rest/routes/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -1,37 +1,68 @@ package routes import ( + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetTransactionByID gets a transaction by requested ID. -func GetTransactionByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetTransactionByID(req, r.Context(), link, r.Chain) + tx, err := backend.GetTransaction(r.Context(), req.ID) + if err != nil { + return nil, err + } + + var txr *access.TransactionResult + // only lookup result if transaction result is to be expanded + if req.ExpandsResult { + txr, err = backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) + if err != nil { + return nil, err + } + } + + var response models.Transaction + response.Build(tx, txr, link) + return response, nil } // GetTransactionResultByID retrieves transaction result by the transaction ID. -func GetTransactionResultByID(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionResultByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionResultRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.GetTransactionResultByID(req, r.Context(), link) + txr, err := backend.GetTransactionResult(r.Context(), req.ID, req.BlockID, req.CollectionID) + if err != nil { + return nil, err + } + + var response models.TransactionResult + response.Build(txr, req.ID, link) + return response, nil } // CreateTransaction creates a new transaction from provided payload. -func CreateTransaction(r *request.Request, srv api.RestServerApi, link models.LinkGenerator) (interface{}, error) { +func CreateTransaction(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { req, err := r.CreateTransactionRequest() if err != nil { return nil, models.NewBadRequestError(err) } - return srv.CreateTransaction(req, r.Context(), link) + err = backend.SendTransaction(r.Context(), &req.Transaction) + if err != nil { + return nil, err + } + + var response models.Transaction + response.Build(&req.Transaction, nil, link) + return response, nil } diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index 9dc7112e6b6..d07983abb64 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -13,7 +13,7 @@ import ( ) // NewServer returns an HTTP server initialized with the REST API handler -func NewServer(serverAPI api.RestServerApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { +func NewServer(serverAPI api.RestBackendApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { router, err := NewRouter(serverAPI, logger, chain, restCollector) if err != nil { return nil, err diff --git a/engine/access/rest/server_request_handler.go b/engine/access/rest/server_request_handler.go deleted file mode 100644 index 257c9f82deb..00000000000 --- a/engine/access/rest/server_request_handler.go +++ /dev/null @@ -1,346 +0,0 @@ -package rest - -import ( - "context" - "fmt" - - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/engine/access/rest/api" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/access/rest/routes" - "github.com/onflow/flow-go/model/flow" -) - -// ServerRequestHandler is a structure that represents handling local requests. -type ServerRequestHandler struct { - log zerolog.Logger - backend access.API -} - -var _ api.RestServerApi = (*ServerRequestHandler)(nil) - -// NewServerRequestHandler returns new ServerRequestHandler. -func NewServerRequestHandler(log zerolog.Logger, backend access.API) *ServerRequestHandler { - return &ServerRequestHandler{ - log: log, - backend: backend, - } -} - -// GetTransactionByID gets a transaction by requested ID. -func (h *ServerRequestHandler) GetTransactionByID(r request.GetTransaction, context context.Context, link models.LinkGenerator, _ flow.Chain) (models.Transaction, error) { - var response models.Transaction - - tx, err := h.backend.GetTransaction(context, r.ID) - if err != nil { - return response, err - } - - var txr *access.TransactionResult - // only lookup result if transaction result is to be expanded - if r.ExpandsResult { - txr, err = h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) - if err != nil { - return response, err - } - } - - response.Build(tx, txr, link) - return response, nil -} - -// CreateTransaction creates a new transaction from provided payload. -func (h *ServerRequestHandler) CreateTransaction(r request.CreateTransaction, context context.Context, link models.LinkGenerator) (models.Transaction, error) { - var response models.Transaction - - err := h.backend.SendTransaction(context, &r.Transaction) - if err != nil { - return response, err - } - - response.Build(&r.Transaction, nil, link) - return response, nil -} - -// GetTransactionResultByID retrieves transaction result by the transaction ID. -func (h *ServerRequestHandler) GetTransactionResultByID(r request.GetTransactionResult, context context.Context, link models.LinkGenerator) (models.TransactionResult, error) { - var response models.TransactionResult - - txr, err := h.backend.GetTransactionResult(context, r.ID, r.BlockID, r.CollectionID) - if err != nil { - return response, err - } - - response.Build(txr, r.ID, link) - return response, nil -} - -// GetBlocksByIDs gets blocks by provided ID or list of IDs. -func (h *ServerRequestHandler) GetBlocksByIDs(r request.GetBlockByIDs, context context.Context, expandFields map[string]bool, link models.LinkGenerator) ([]*models.Block, error) { - blocks := make([]*models.Block, len(r.IDs)) - - for i, id := range r.IDs { - block, err := routes.GetBlock(routes.ForID(&id), context, expandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil -} - -// GetBlocksByHeight gets blocks by provided height. -func (h *ServerRequestHandler) GetBlocksByHeight(r *request.Request, link models.LinkGenerator) ([]*models.Block, error) { - req, err := r.GetBlockRequest() - if err != nil { - return nil, models.NewBadRequestError(err) - } - - if req.FinalHeight || req.SealedHeight { - block, err := routes.GetBlock(routes.ForFinalized(req.Heights[0]), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - - return []*models.Block{block}, nil - } - - // if the query is /blocks/height=1000,1008,1049... - if req.HasHeights() { - blocks := make([]*models.Block, len(req.Heights)) - for i, height := range req.Heights { - block, err := routes.GetBlock(routes.ForHeight(height), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks[i] = block - } - - return blocks, nil - } - - // support providing end height as "sealed" or "final" - if req.EndHeight == request.FinalHeight || req.EndHeight == request.SealedHeight { - latest, _, err := h.backend.GetLatestBlock(r.Context(), req.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - req.EndHeight = latest.Header.Height // overwrite special value height with fetched - - if req.StartHeight > req.EndHeight { - return nil, models.NewBadRequestError(fmt.Errorf("start height must be less than or equal to end height")) - } - } - - blocks := make([]*models.Block, 0) - // start and end height inclusive - for i := req.StartHeight; i <= req.EndHeight; i++ { - block, err := routes.GetBlock(routes.ForHeight(i), r.Context(), r.ExpandFields, h.backend, link) - if err != nil { - return nil, err - } - blocks = append(blocks, block) - } - - return blocks, nil -} - -// GetBlockPayloadByID gets block payload by ID -func (h *ServerRequestHandler) GetBlockPayloadByID(r request.GetBlockPayload, context context.Context, _ models.LinkGenerator) (models.BlockPayload, error) { - var payload models.BlockPayload - - blkProvider := routes.NewBlockRequestProvider(h.backend, routes.ForID(&r.ID)) - blk, _, statusErr := blkProvider.GetBlock(context) - if statusErr != nil { - return payload, statusErr - } - - err := payload.Build(blk.Payload) - if err != nil { - return payload, err - } - - return payload, nil -} - -// GetExecutionResultByID gets execution result by the ID. -func (h *ServerRequestHandler) GetExecutionResultByID(r request.GetExecutionResult, context context.Context, link models.LinkGenerator) (models.ExecutionResult, error) { - var response models.ExecutionResult - - res, err := h.backend.GetExecutionResultByID(context, r.ID) - if err != nil { - return response, err - } - - if res == nil { - err := fmt.Errorf("execution result with ID: %s not found", r.ID.String()) - return response, models.NewNotFoundError(err.Error(), err) - } - - err = response.Build(res, link) - if err != nil { - return response, err - } - - return response, nil -} - -// GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func (h *ServerRequestHandler) GetExecutionResultsByBlockIDs(r request.GetExecutionResultByBlockIDs, context context.Context, link models.LinkGenerator) ([]models.ExecutionResult, error) { - // for each block ID we retrieve execution result - results := make([]models.ExecutionResult, len(r.BlockIDs)) - for i, id := range r.BlockIDs { - res, err := h.backend.GetExecutionResultForBlockID(context, id) - if err != nil { - return nil, err - } - - var response models.ExecutionResult - err = response.Build(res, link) - if err != nil { - return nil, err - } - results[i] = response - } - - return results, nil -} - -// GetCollectionByID retrieves a collection by ID and builds a response -func (h *ServerRequestHandler) GetCollectionByID(r request.GetCollection, context context.Context, expandFields map[string]bool, link models.LinkGenerator, _ flow.Chain) (models.Collection, error) { - var response models.Collection - - collection, err := h.backend.GetCollectionByID(context, r.ID) - if err != nil { - return response, err - } - - // if we expand transactions in the query retrieve each transaction data - transactions := make([]*flow.TransactionBody, 0) - if r.ExpandsTransactions { - for _, tid := range collection.Transactions { - tx, err := h.backend.GetTransaction(context, tid) - if err != nil { - return response, err - } - - transactions = append(transactions, tx) - } - } - - err = response.Build(collection, transactions, link, expandFields) - if err != nil { - return response, err - } - - return response, nil -} - -// ExecuteScript handler sends the script from the request to be executed. -func (h *ServerRequestHandler) ExecuteScript(r request.GetScript, context context.Context, _ models.LinkGenerator) ([]byte, error) { - if r.BlockID != flow.ZeroID { - return h.backend.ExecuteScriptAtBlockID(context, r.BlockID, r.Script.Source, r.Script.Args) - } - - // default to sealed height - if r.BlockHeight == request.SealedHeight || r.BlockHeight == request.EmptyHeight { - return h.backend.ExecuteScriptAtLatestBlock(context, r.Script.Source, r.Script.Args) - } - - if r.BlockHeight == request.FinalHeight { - finalBlock, _, err := h.backend.GetLatestBlockHeader(context, false) - if err != nil { - return nil, err - } - r.BlockHeight = finalBlock.Height - } - - return h.backend.ExecuteScriptAtBlockHeight(context, r.BlockHeight, r.Script.Source, r.Script.Args) -} - -// GetAccount handler retrieves account by address and returns the response. -func (h *ServerRequestHandler) GetAccount(r request.GetAccount, context context.Context, expandFields map[string]bool, link models.LinkGenerator) (models.Account, error) { - var response models.Account - - // in case we receive special height values 'final' and 'sealed', fetch that height and overwrite request with it - if r.Height == request.FinalHeight || r.Height == request.SealedHeight { - header, _, err := h.backend.GetLatestBlockHeader(context, r.Height == request.SealedHeight) - if err != nil { - return response, err - } - r.Height = header.Height - } - - account, err := h.backend.GetAccountAtBlockHeight(context, r.Address, r.Height) - if err != nil { - return response, models.NewNotFoundError("not found account at block height", err) - } - - err = response.Build(account, link, expandFields) - return response, err -} - -// GetEvents for the provided block range or list of block IDs filtered by type. -func (h *ServerRequestHandler) GetEvents(r request.GetEvents, context context.Context) (models.BlocksEvents, error) { - // if the request has block IDs provided then return events for block IDs - var blocksEvents models.BlocksEvents - if len(r.BlockIDs) > 0 { - events, err := h.backend.GetEventsForBlockIDs(context, r.Type, r.BlockIDs) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil - } - - // if end height is provided with special values then load the height - if r.EndHeight == request.FinalHeight || r.EndHeight == request.SealedHeight { - latest, _, err := h.backend.GetLatestBlockHeader(context, r.EndHeight == request.SealedHeight) - if err != nil { - return nil, err - } - - r.EndHeight = latest.Height - // special check after we resolve special height value - if r.StartHeight > r.EndHeight { - return nil, models.NewBadRequestError(fmt.Errorf("current retrieved end height value is lower than start height")) - } - } - - // if request provided block height range then return events for that range - events, err := h.backend.GetEventsForHeightRange(context, r.Type, r.StartHeight, r.EndHeight) - if err != nil { - return nil, err - } - - blocksEvents.Build(events) - return blocksEvents, nil -} - -// GetNetworkParameters returns network-wide parameters of the blockchain -func (h *ServerRequestHandler) GetNetworkParameters(r *request.Request) (models.NetworkParameters, error) { - params := h.backend.GetNetworkParameters(r.Context()) - - var response models.NetworkParameters - response.Build(¶ms) - return response, nil -} - -// GetNodeVersionInfo returns node version information -func (h *ServerRequestHandler) GetNodeVersionInfo(r *request.Request) (models.NodeVersionInfo, error) { - var response models.NodeVersionInfo - - params, err := h.backend.GetNodeVersionInfo(r.Context()) - if err != nil { - return response, err - } - - response.Build(params) - return response, nil -} diff --git a/engine/access/rest/tests/accounts_test.go b/engine/access/rest/tests/accounts_test.go index c9a247f8c15..0ff84f79075 100644 --- a/engine/access/rest/tests/accounts_test.go +++ b/engine/access/rest/tests/accounts_test.go @@ -14,9 +14,8 @@ import ( "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/middleware" restmock "github.com/onflow/flow-go/engine/access/rest/mock" - "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" ) @@ -46,7 +45,6 @@ func accountURL(t *testing.T, address string, height string) string { // 5. Get invalid account. func TestAccessGetAccount(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("get by address at latest sealed block", func(t *testing.T) { account := accountFixture(t) @@ -65,7 +63,7 @@ func TestAccessGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -85,7 +83,7 @@ func TestAccessGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -100,7 +98,7 @@ func TestAccessGetAccount(t *testing.T) { expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -115,7 +113,7 @@ func TestAccessGetAccount(t *testing.T) { expected := expectedCondensedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -130,7 +128,7 @@ func TestAccessGetAccount(t *testing.T) { for i, test := range tests { req, _ := http.NewRequest("GET", test.url, nil) - rr, err := executeRequest(req, restHandler) + rr, err := executeRequest(req, backend) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, rr.Code) @@ -148,186 +146,76 @@ func TestAccessGetAccount(t *testing.T) { // 4. Get account by address at height condensed. // 5. Get invalid account. func TestObserverGetAccount(t *testing.T) { - backend := &mock.API{} - restForwarder := &restmock.RestServerApi{} - - restHandler, err := newObserverRestHandler(backend, restForwarder) - assert.NoError(t, err) + backend := &restmock.RestBackendApi{} t.Run("get by address at latest sealed block", func(t *testing.T) { account := accountFixture(t) + var height uint64 = 100 + block := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) req := getAccountRequest(t, account, sealedHeightQueryParam, expandableFieldKeys, expandableFieldContracts) - accountKeys := make([]models.AccountPublicKey, 1) - - sigAlgo := models.SigningAlgorithm("ECDSA_P256") - hashAlgo := models.HashingAlgorithm("SHA3_256") - - accountKeys[0] = models.AccountPublicKey{ - Index: "0", - PublicKey: account.Keys[0].PublicKey.String(), - SigningAlgorithm: &sigAlgo, - HashingAlgorithm: &hashAlgo, - SequenceNumber: "0", - Weight: "1000", - Revoked: false, - } + backend.Mock. + On("GetLatestBlockHeader", mocktestify.Anything, true). + Return(block, flow.BlockStatusSealed, nil) - restForwarder.Mock.On("GetAccount", - request.GetAccount{ - Address: account.Address, - Height: request.SealedHeight, - }, - mocktestify.Anything, - mocktestify.Anything, - mocktestify.Anything). - Return(models.Account{ - Address: account.Address.String(), - Balance: fmt.Sprintf("%d", account.Balance), - Keys: accountKeys, - Contracts: map[string]string{ - "contract1": "Y29udHJhY3Qx", - "contract2": "Y29udHJhY3Qy", - }, - Expandable: &models.AccountExpandable{}, - Links: &models.Links{ - Self: fmt.Sprintf("/v1/accounts/%s", account.Address), - }, - }, nil) + backend.Mock. + On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). + Return(account, nil) expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) t.Run("get by address at latest finalized block", func(t *testing.T) { + var height uint64 = 100 + block := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) account := accountFixture(t) req := getAccountRequest(t, account, finalHeightQueryParam, expandableFieldKeys, expandableFieldContracts) - accountKeys := make([]models.AccountPublicKey, 1) - - sigAlgo := models.SigningAlgorithm("ECDSA_P256") - hashAlgo := models.HashingAlgorithm("SHA3_256") - - accountKeys[0] = models.AccountPublicKey{ - Index: "0", - PublicKey: account.Keys[0].PublicKey.String(), - SigningAlgorithm: &sigAlgo, - HashingAlgorithm: &hashAlgo, - SequenceNumber: "0", - Weight: "1000", - Revoked: false, - } - - restForwarder.Mock.On("GetAccount", - request.GetAccount{ - Address: account.Address, - Height: request.FinalHeight, - }, - mocktestify.Anything, - mocktestify.Anything, - mocktestify.Anything). - Return(models.Account{ - Address: account.Address.String(), - Balance: fmt.Sprintf("%d", account.Balance), - Keys: accountKeys, - Contracts: map[string]string{ - "contract1": "Y29udHJhY3Qx", - "contract2": "Y29udHJhY3Qy", - }, - Expandable: &models.AccountExpandable{}, - Links: &models.Links{ - Self: fmt.Sprintf("/v1/accounts/%s", account.Address), - }, - }, nil) + backend.Mock. + On("GetLatestBlockHeader", mocktestify.Anything, false). + Return(block, flow.BlockStatusFinalized, nil) + backend.Mock. + On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). + Return(account, nil) expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) t.Run("get by address at height", func(t *testing.T) { var height uint64 = 1337 account := accountFixture(t) - req := getAccountRequest(t, account, fmt.Sprintf("%d", height), expandableFieldKeys, expandableFieldContracts) - accountKeys := make([]models.AccountPublicKey, 1) - - sigAlgo := models.SigningAlgorithm("ECDSA_P256") - hashAlgo := models.HashingAlgorithm("SHA3_256") - - accountKeys[0] = models.AccountPublicKey{ - Index: "0", - PublicKey: account.Keys[0].PublicKey.String(), - SigningAlgorithm: &sigAlgo, - HashingAlgorithm: &hashAlgo, - SequenceNumber: "0", - Weight: "1000", - Revoked: false, - } - restForwarder.Mock.On("GetAccount", - request.GetAccount{ - Address: account.Address, - Height: height, - }, - mocktestify.Anything, - mocktestify.Anything, - mocktestify.Anything). - Return(models.Account{ - Address: account.Address.String(), - Balance: fmt.Sprintf("%d", account.Balance), - Keys: accountKeys, - Contracts: map[string]string{ - "contract1": "Y29udHJhY3Qx", - "contract2": "Y29udHJhY3Qy", - }, - Expandable: &models.AccountExpandable{}, - Links: &models.Links{ - Self: fmt.Sprintf("/v1/accounts/%s", account.Address), - }, - }, nil) + backend.Mock. + On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). + Return(account, nil) expected := expectedExpandedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) t.Run("get by address at height condensed", func(t *testing.T) { var height uint64 = 1337 account := accountFixture(t) - req := getAccountRequest(t, account, fmt.Sprintf("%d", height)) - restForwarder.Mock.On("GetAccount", - request.GetAccount{ - Address: account.Address, - Height: height, - }, - mocktestify.Anything, - mocktestify.Anything, - mocktestify.Anything). - Return(models.Account{ - Address: account.Address.String(), - Balance: fmt.Sprintf("%d", account.Balance), - Contracts: map[string]string{}, - Expandable: &models.AccountExpandable{ - Keys: expandableFieldKeys, - Contracts: expandableFieldContracts, - }, - Links: &models.Links{ - Self: fmt.Sprintf("/v1/accounts/%s", account.Address), - }, - }, nil) + backend.Mock. + On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). + Return(account, nil) expected := expectedCondensedResponse(account) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) @@ -342,7 +230,7 @@ func TestObserverGetAccount(t *testing.T) { for i, test := range tests { req, _ := http.NewRequest("GET", test.url, nil) - rr, err := executeRequest(req, restHandler) + rr, err := executeRequest(req, backend) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, rr.Code) diff --git a/engine/access/rest/tests/blocks_test.go b/engine/access/rest/tests/blocks_test.go index 0b8fd2c66c5..7b8b18cdc5f 100644 --- a/engine/access/rest/tests/blocks_test.go +++ b/engine/access/rest/tests/blocks_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" - restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/util" @@ -148,48 +147,9 @@ func TestAccessGetBlocks(t *testing.T) { blkCnt := 10 blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) - restHandler := newAccessRestHandler(backend) for _, tv := range testVectors { - responseRec, err := executeRequest(tv.request, restHandler) - assert.NoError(t, err) - require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) - actualResp := responseRec.Body.String() - require.JSONEq(t, tv.expectedResponse, actualResp, "Failed: %s: incorrect response body", tv.description) - } -} - -// TestObserverGetBlocks tests requests forwarding for get blocks by ID and get blocks by heights. -// -// Check the following cases: -// 1. Get single expanded block by ID. -// 2. Get multiple expanded blocks by IDs -// 3. Get single condensed block by ID. -// 4. Get multiple condensed blocks by IDs. -// 5. Get single expanded block by height. -// 6. Get multiple expanded blocks by heights. -// 7. Get multiple expanded blocks by start and end height. -// 8. Get block by ID not found. -// 9. Get block by height not found. -// 10. Get block by end height less than start height. -// 11. Get block by both heights and start and end height. -// 12. Get block with missing height param. -// 13. Get block with missing height values. -// 14. Get block by more than maximum permissible number of IDs. -func TestObserverGetBlocks(t *testing.T) { - backend := &mock.API{} - restForwarder := &restmock.RestServerApi{} - - blkCnt := 10 - blockIDs, heights, blocks, executionResults := generateMocks(backend, blkCnt) - - testVectors := prepareTestVectors(t, blockIDs, heights, blocks, executionResults, blkCnt) - - restHandler, err := newObserverRestHandler(backend, restForwarder) - assert.NoError(t, err) - - for _, tv := range testVectors { - responseRec, err := executeRequest(tv.request, restHandler) + responseRec, err := executeRequest(tv.request, backend) assert.NoError(t, err) require.Equal(t, tv.expectedStatus, responseRec.Code, "failed test %s: incorrect response code", tv.description) actualResp := responseRec.Body.String() diff --git a/engine/access/rest/tests/collections_test.go b/engine/access/rest/tests/collections_test.go index 77a97957ae8..db57c4101e5 100644 --- a/engine/access/rest/tests/collections_test.go +++ b/engine/access/rest/tests/collections_test.go @@ -31,7 +31,6 @@ func getCollectionReq(id string, expandTransactions bool) *http.Request { func TestGetCollections(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("get by ID", func(t *testing.T) { inputs := []flow.LightCollection{ @@ -63,7 +62,7 @@ func TestGetCollections(t *testing.T) { }`, col.ID(), col.ID(), transactionsStr) req := getCollectionReq(col.ID().String(), false) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocks.AssertExpectationsForObjects(t, backend) } }) @@ -88,7 +87,7 @@ func TestGetCollections(t *testing.T) { Once() req := getCollectionReq(col.ID().String(), true) - rr, err := executeRequest(req, restHandler) + rr, err := executeRequest(req, backend) assert.NoError(t, err) assert.Equal(t, http.StatusOK, rr.Code) @@ -147,7 +146,7 @@ func TestGetCollections(t *testing.T) { Return(test.mockValue, test.mockErr) } req := getCollectionReq(test.id, false) - assertResponse(t, req, test.status, test.response, restHandler) + assertResponse(t, req, test.status, test.response, backend) } }) } diff --git a/engine/access/rest/tests/events_test.go b/engine/access/rest/tests/events_test.go index 806b7860b96..2de8a4b0834 100644 --- a/engine/access/rest/tests/events_test.go +++ b/engine/access/rest/tests/events_test.go @@ -25,7 +25,6 @@ import ( func TestGetEvents(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) events := generateEventsMocks(backend, 5) allBlockIDs := make([]string, len(events)) @@ -128,7 +127,7 @@ func TestGetEvents(t *testing.T) { for _, test := range testVectors { t.Run(test.description, func(t *testing.T) { - assertResponse(t, test.request, test.expectedStatus, test.expectedResponse, restHandler) + assertResponse(t, test.request, test.expectedStatus, test.expectedResponse, backend) }) } diff --git a/engine/access/rest/tests/execution_result_test.go b/engine/access/rest/tests/execution_result_test.go index f978f9bd299..23038b1abdb 100644 --- a/engine/access/rest/tests/execution_result_test.go +++ b/engine/access/rest/tests/execution_result_test.go @@ -37,10 +37,8 @@ func getResultByIDReq(id string, blockIDs []string) *http.Request { } func TestGetResultByID(t *testing.T) { - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) - t.Run("get by ID", func(t *testing.T) { + backend := &mock.API{} result := unittest.ExecutionResultFixture() id := unittest.IdentifierFixture() backend.Mock. @@ -50,11 +48,12 @@ func TestGetResultByID(t *testing.T) { req := getResultByIDReq(id.String(), nil) expected := executionResultExpectedStr(result) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocks.AssertExpectationsForObjects(t, backend) }) t.Run("get by ID not found", func(t *testing.T) { + backend := &mock.API{} id := unittest.IdentifierFixture() backend.Mock. On("GetExecutionResultByID", mocks.Anything, id). @@ -62,17 +61,15 @@ func TestGetResultByID(t *testing.T) { Once() req := getResultByIDReq(id.String(), nil) - assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, restHandler) + assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, backend) mocks.AssertExpectationsForObjects(t, backend) }) } func TestGetResultBlockID(t *testing.T) { - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("get by block ID", func(t *testing.T) { - + backend := &mock.API{} blockID := unittest.IdentifierFixture() result := unittest.ExecutionResultFixture(unittest.WithExecutionResultBlockID(blockID)) @@ -84,11 +81,12 @@ func TestGetResultBlockID(t *testing.T) { req := getResultByIDReq("", []string{blockID.String()}) expected := fmt.Sprintf(`[%s]`, executionResultExpectedStr(result)) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocks.AssertExpectationsForObjects(t, backend) }) t.Run("get by block ID not found", func(t *testing.T) { + backend := &mock.API{} blockID := unittest.IdentifierFixture() backend.Mock. On("GetExecutionResultForBlockID", mocks.Anything, blockID). @@ -96,7 +94,7 @@ func TestGetResultBlockID(t *testing.T) { Once() req := getResultByIDReq("", []string{blockID.String()}) - assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, restHandler) + assertResponse(t, req, http.StatusNotFound, `{"code":404,"message":"Flow resource not found: block not found"}`, backend) mocks.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/tests/network_test.go b/engine/access/rest/tests/network_test.go index 4af5b501c06..a841f076f60 100644 --- a/engine/access/rest/tests/network_test.go +++ b/engine/access/rest/tests/network_test.go @@ -23,7 +23,6 @@ func networkURL(t *testing.T) string { func TestGetNetworkParameters(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("get network parameters on mainnet", func(t *testing.T) { @@ -39,7 +38,7 @@ func TestGetNetworkParameters(t *testing.T) { expected := networkParametersExpectedStr(flow.Mainnet) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/tests/node_version_info_test.go b/engine/access/rest/tests/node_version_info_test.go index a4935a29438..25213102934 100644 --- a/engine/access/rest/tests/node_version_info_test.go +++ b/engine/access/rest/tests/node_version_info_test.go @@ -24,7 +24,6 @@ func nodeVersionInfoURL(t *testing.T) string { func TestGetNodeVersionInfo(t *testing.T) { backend := mock.NewAPI(t) - restHandler := newAccessRestHandler(backend) t.Run("get node version info", func(t *testing.T) { req := getNodeVersionInfoRequest(t) @@ -42,7 +41,7 @@ func TestGetNodeVersionInfo(t *testing.T) { expected := nodeVersionInfoExpectedStr(params) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) mocktestify.AssertExpectationsForObjects(t, backend) }) } diff --git a/engine/access/rest/tests/scripts_test.go b/engine/access/rest/tests/scripts_test.go index 0c93106b38a..da498e59dc2 100644 --- a/engine/access/rest/tests/scripts_test.go +++ b/engine/access/rest/tests/scripts_test.go @@ -48,7 +48,6 @@ func TestScripts(t *testing.T) { t.Run("get by Latest height", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) backend.Mock. On("ExecuteScriptAtLatestBlock", mocks.Anything, validCode, [][]byte{validArgs}). @@ -58,14 +57,12 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), restHandler) + ), backend) }) t.Run("get by height", func(t *testing.T) { - height := uint64(1337) - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) + height := uint64(1337) backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, height, validCode, [][]byte{validArgs}). @@ -75,14 +72,12 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), restHandler) + ), backend) }) t.Run("get by ID", func(t *testing.T) { - id, _ := flow.HexStringToIdentifier("222dc5dd51b9e4910f687e475f892f495f3352362ba318b53e318b4d78131312") - backend := &mock.API{} - restHandler := newAccessRestHandler(backend) + id, _ := flow.HexStringToIdentifier("222dc5dd51b9e4910f687e475f892f495f3352362ba318b53e318b4d78131312") backend.Mock. On("ExecuteScriptAtBlockID", mocks.Anything, id, validCode, [][]byte{validArgs}). @@ -92,12 +87,11 @@ func TestScripts(t *testing.T) { assertOKResponse(t, req, fmt.Sprintf( "\"%s\"", base64.StdEncoding.EncodeToString([]byte(`hello world`)), - ), restHandler) + ), backend) }) t.Run("get error", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, uint64(1337), validCode, [][]byte{validArgs}). @@ -109,13 +103,12 @@ func TestScripts(t *testing.T) { req, http.StatusBadRequest, `{"code":400, "message":"Invalid Flow request: internal server error"}`, - restHandler, + backend, ) }) t.Run("get invalid", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, mocks.Anything, mocks.Anything, mocks.Anything). @@ -136,7 +129,7 @@ func TestScripts(t *testing.T) { for _, test := range tests { req := scriptReq(test.id, test.height, test.body) - assertResponse(t, req, http.StatusBadRequest, test.out, restHandler) + assertResponse(t, req, http.StatusBadRequest, test.out, backend) } }) } diff --git a/engine/access/rest/tests/test_helpers.go b/engine/access/rest/tests/test_helpers.go index 4ce4fbc2f50..388b5ca999d 100644 --- a/engine/access/rest/tests/test_helpers.go +++ b/engine/access/rest/tests/test_helpers.go @@ -11,11 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/api" - restproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" - restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" ) @@ -30,11 +27,11 @@ const ( heightQueryParam = "height" ) -func executeRequest(req *http.Request, restHandler api.RestServerApi) (*httptest.ResponseRecorder, error) { +func executeRequest(req *http.Request, backend api.RestBackendApi) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - router, err := rest.NewRouter(restHandler, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) + router, err := rest.NewRouter(backend, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) if err != nil { return nil, err } @@ -44,32 +41,12 @@ func executeRequest(req *http.Request, restHandler api.RestServerApi) (*httptest return rr, nil } -func newAccessRestHandler(backend *mock.API) api.RestServerApi { - var b bytes.Buffer - logger := zerolog.New(&b) - - return rest.NewServerRequestHandler(logger, backend) -} - -func newObserverRestHandler(backend *mock.API, restForwarder *restmock.RestServerApi) (api.RestServerApi, error) { - var b bytes.Buffer - logger := zerolog.New(&b) - observerCollector := metrics.NewNoopCollector() - - return &restproxy.RestRouter{ - Logger: logger, - Metrics: observerCollector, - Upstream: restForwarder, - Observer: rest.NewServerRequestHandler(logger, backend), - }, nil -} - -func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, restHandler api.RestServerApi) { - assertResponse(t, req, http.StatusOK, expectedRespBody, restHandler) +func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend api.RestBackendApi) { + assertResponse(t, req, http.StatusOK, expectedRespBody, backend) } -func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, restHandler api.RestServerApi) { - rr, err := executeRequest(req, restHandler) +func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend api.RestBackendApi) { + rr, err := executeRequest(req, backend) assert.NoError(t, err) actualResponseBody := rr.Body.String() require.JSONEq(t, diff --git a/engine/access/rest/tests/transactions_test.go b/engine/access/rest/tests/transactions_test.go index f4570d3a191..3d81a0b5eff 100644 --- a/engine/access/rest/tests/transactions_test.go +++ b/engine/access/rest/tests/transactions_test.go @@ -104,7 +104,6 @@ func validCreateBody(tx flow.TransactionBody) map[string]interface{} { func TestGetTransactions(t *testing.T) { t.Run("get by ID without results", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -146,12 +145,11 @@ func TestGetTransactions(t *testing.T) { }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("Get by ID with results", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) tx := unittest.TransactionFixture() txr := transactionResultFixture(tx) @@ -217,21 +215,19 @@ func TestGetTransactions(t *testing.T) { } }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ReferenceBlockID, txr.CollectionID, tx.ID(), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("get by ID Invalid", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) req := getTransactionReq("invalid", false, "", "") expected := `{"code":400, "message":"invalid ID format"}` - assertResponse(t, req, http.StatusBadRequest, expected, restHandler) + assertResponse(t, req, http.StatusBadRequest, expected, backend) }) t.Run("get by ID non-existing", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -241,7 +237,7 @@ func TestGetTransactions(t *testing.T) { Return(nil, status.Error(codes.NotFound, "transaction not found")) expected := `{"code":404, "message":"Flow resource not found: transaction not found"}` - assertResponse(t, req, http.StatusNotFound, expected, restHandler) + assertResponse(t, req, http.StatusNotFound, expected, backend) }) } @@ -284,7 +280,6 @@ func TestGetTransactionResult(t *testing.T) { t.Run("get by transaction ID", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) req := getTransactionResultReq(id.String(), "", "") @@ -292,12 +287,11 @@ func TestGetTransactionResult(t *testing.T) { On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, flow.ZeroID). Return(txr, nil) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("get by block ID", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) req := getTransactionResultReq(id.String(), bid.String(), "") @@ -305,12 +299,11 @@ func TestGetTransactionResult(t *testing.T) { On("GetTransactionResult", mocks.Anything, id, bid, flow.ZeroID). Return(txr, nil) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("get by collection ID", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) req := getTransactionResultReq(id.String(), "", cid.String()) @@ -318,12 +311,11 @@ func TestGetTransactionResult(t *testing.T) { On("GetTransactionResult", mocks.Anything, id, flow.ZeroID, cid). Return(txr, nil) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("get execution statuses", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) testVectors := map[*access.TransactionResult]string{{ Status: flow.TransactionStatusExpired, @@ -367,24 +359,22 @@ func TestGetTransactionResult(t *testing.T) { "_self": "/v1/transaction_results/%s" } }`, bid.String(), cid.String(), err, cases.Title(language.English).String(strings.ToLower(txResult.Status.String())), txResult.ErrorMessage, id.String()) - assertOKResponse(t, req, expectedResp, restHandler) + assertOKResponse(t, req, expectedResp, backend) } }) t.Run("get by ID Invalid", func(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) req := getTransactionResultReq("invalid", "", "") expected := `{"code":400, "message":"invalid ID format"}` - assertResponse(t, req, http.StatusBadRequest, expected, restHandler) + assertResponse(t, req, http.StatusBadRequest, expected, backend) }) } func TestCreateTransaction(t *testing.T) { backend := &mock.API{} - restHandler := newAccessRestHandler(backend) t.Run("create", func(t *testing.T) { tx := unittest.TransactionBodyFixture() @@ -434,7 +424,7 @@ func TestCreateTransaction(t *testing.T) { } }`, tx.ID(), tx.ReferenceBlockID, util.ToBase64(tx.PayloadSignatures[0].Signature), util.ToBase64(tx.EnvelopeSignatures[0].Signature), tx.ID(), tx.ID()) - assertOKResponse(t, req, expected, restHandler) + assertOKResponse(t, req, expected, backend) }) t.Run("post invalid transaction", func(t *testing.T) { @@ -461,7 +451,7 @@ func TestCreateTransaction(t *testing.T) { testTx[test.inputField] = test.inputValue req := createTransactionReq(testTx) - assertResponse(t, req, http.StatusBadRequest, test.output, restHandler) + assertResponse(t, req, http.StatusBadRequest, test.output, backend) } }) } diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 97d23f9a5ab..3e5a08a3570 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -201,6 +201,7 @@ func New( return b } +// NewCache constructs cache and its size. func NewCache( log zerolog.Logger, accessMetrics module.AccessMetrics, diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 42b4aec9021..7ef17d032c3 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -69,7 +69,7 @@ type Engine struct { secureGrpcAddress net.Addr restAPIAddress net.Addr - restHandler api.RestServerApi + restHandler api.RestBackendApi } type Option func(*RPCEngineBuilder) diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index 79db9853c29..7308a18597d 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -8,7 +8,6 @@ import ( "github.com/onflow/flow-go/access" legacyaccess "github.com/onflow/flow-go/access/legacy" "github.com/onflow/flow-go/consensus/hotstuff" - "github.com/onflow/flow-go/engine/access/rest" restapi "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/module" @@ -40,7 +39,7 @@ func (builder *RPCEngineBuilder) RpcHandler() accessproto.AccessAPIServer { return builder.rpcHandler } -func (builder *RPCEngineBuilder) RestHandler() restapi.RestServerApi { +func (builder *RPCEngineBuilder) RestHandler() restapi.RestBackendApi { return builder.restHandler } @@ -69,8 +68,8 @@ func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPISer return builder } -// WithRestHandler specifies that the given `RestServerApi` should be used for REST. -func (builder *RPCEngineBuilder) WithRestHandler(handler restapi.RestServerApi) *RPCEngineBuilder { +// WithRestHandler specifies that the given `RestBackendApi` should be used for REST. +func (builder *RPCEngineBuilder) WithRestHandler(handler restapi.RestBackendApi) *RPCEngineBuilder { builder.restHandler = handler return builder } @@ -117,7 +116,7 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { restHandler := builder.Engine.restHandler if restHandler == nil { - restHandler = rest.NewServerRequestHandler(builder.log, builder.backend) + restHandler = builder.backend } builder.Engine.restHandler = restHandler diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index d2f817df8c7..f2f7a0eee9c 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -7,8 +7,10 @@ import ( "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" @@ -651,6 +653,26 @@ func AccountKeyToMessage(a flow.AccountPublicKey) (*entities.AccountKey, error) }, nil } +func MessagesToBlockEvents(blocksEvents []*accessproto.EventsResponse_Result) []flow.BlockEvents { + evs := make([]flow.BlockEvents, len(blocksEvents)) + for _, ev := range blocksEvents { + var blockEvent flow.BlockEvents + MessageToBlockEvent(ev) + evs = append(evs, blockEvent) + } + + return evs +} + +func MessageToBlockEvent(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { + return flow.BlockEvents{ + BlockHeight: blockEvents.BlockHeight, + BlockID: MessageToIdentifier(blockEvents.BlockId), + BlockTimestamp: blockEvents.BlockTimestamp.AsTime(), + Events: MessagesToEvents(blockEvents.Events), + } +} + func MessageToEvent(m *entities.Event) flow.Event { return flow.Event{ Type: flow.EventType(m.GetType()), From 9deccb87b03c6aa09dca4026a4d0e88d7c34c1cf Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 29 Jun 2023 14:20:43 +0300 Subject: [PATCH 071/169] Added constructor to rest proxy handler, updated test --- cmd/observer/node_builder/observer_builder.go | 22 +++++------- .../rest/apiproxy/rest_proxy_handler.go | 34 ++++++++++++++++++- integration/tests/access/observer_test.go | 4 +-- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 5c8e4922c7e..56512e02609 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -32,7 +32,6 @@ import ( "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/follower" - "github.com/onflow/flow-go/engine/common/grpc/forwarder" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/protocol" "github.com/onflow/flow-go/model/encodable" @@ -913,7 +912,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { } // upstream access node forwarder - rpcForwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) + forwarder, err := apiproxy.NewFlowAccessAPIForwarder(builder.upstreamIdentities, builder.apiTimeout, config.MaxMsgSize) if err != nil { return nil, err } @@ -923,7 +922,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, Metrics: observerCollector, - Upstream: rpcForwarder, + Upstream: forwarder, Observer: protocol.NewHandler(protocol.New( node.State, node.Storage.Blocks, @@ -931,22 +930,19 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { backend.NewNetworkAPI(node.State, node.RootChainID, backend.DefaultSnapshotHistoryLimit), )), } - frw, err := forwarder.NewForwarder( + + restHandler, err := restapiproxy.NewRestProxyHandler( + accessBackend, builder.upstreamIdentities, builder.apiTimeout, - config.MaxMsgSize) + config.MaxMsgSize, + builder.Logger, + observerCollector, + node.RootChainID.Chain()) if err != nil { return nil, err } - restHandler := &restapiproxy.RestProxyHandler{ - Logger: builder.Logger, - Metrics: observerCollector, - Chain: node.RootChainID.Chain(), - } - restHandler.API = accessBackend - restHandler.Forwarder = frw - // build the rpc engine builder.RpcEng, err = engineBuilder. WithRpcHandler(rpcHandler). diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index d4bc6546a40..d2a4c6adbd5 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -2,6 +2,7 @@ package apiproxy import ( "context" + "time" "google.golang.org/grpc/status" @@ -27,6 +28,37 @@ type RestProxyHandler struct { Chain flow.Chain } +// NewRestProxyHandler returns a new rest proxy handler for observer node. +func NewRestProxyHandler( + api access.API, + identities flow.IdentityList, + timeout time.Duration, + maxMsgSize uint, + log zerolog.Logger, + metrics metrics.ObserverMetrics, + chain flow.Chain, +) (*RestProxyHandler, error) { + + forwarder, err := forwarder.NewForwarder( + identities, + timeout, + maxMsgSize) + if err != nil { + return nil, err + } + + restProxyHandler := &RestProxyHandler{ + Logger: log, + Metrics: metrics, + Chain: chain, + } + + restProxyHandler.API = api + restProxyHandler.Forwarder = forwarder + + return restProxyHandler, nil +} + func (r *RestProxyHandler) log(handler, rpc string, err error) { code := status.Code(err) r.Metrics.RecordRPC(handler, rpc, code) @@ -46,7 +78,7 @@ func (r *RestProxyHandler) log(handler, rpc string, err error) { } // GetLatestBlockHeader returns the latest block header and block status, if isSealed = true - returns the latest seal header. -func (r *RestProxyHandler) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { //Uliana getAccount та GetEvents були з Upstream +func (r *RestProxyHandler) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { upstream, err := r.FaultTolerantClient() if err != nil { return nil, flow.BlockStatusUnknown, err diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index e36642eb662..a70bf31c3d6 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -418,7 +418,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { { name: "getBlocksByHeight", method: http.MethodGet, - path: "/blocks?height=0", + path: "/blocks?height=1", }, { name: "getBlockPayloadByID", @@ -448,7 +448,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { { name: "getAccount", method: http.MethodGet, - path: "/accounts/" + account.Address.HexWithPrefix(), + path: "/accounts/" + account.Address.HexWithPrefix() + "?block_height=1", }, { name: "getEvents", From bbe70d0190d7de4c384f8e0598afb09c6db0746d Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 29 Jun 2023 15:49:33 +0300 Subject: [PATCH 072/169] Changed flow/protobuf/go/flow module to latest, removed lines with replacing mod --- go.mod | 5 +---- go.sum | 4 ++-- insecure/go.mod | 2 +- insecure/go.sum | 4 ++-- integration/go.mod | 5 +---- integration/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 46031eb3d96..9e6a8f81059 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.41.6 github.com/onflow/flow-go/crypto v0.24.7 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible @@ -282,6 +282,3 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.6 // indirect ) - -//TODO: Remove when branch UlyanaAndrukhiv/3138-rest-api-on-observers on onflow/flow will be merged -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 => github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 diff --git a/go.sum b/go.sum index fa433a6d696..613e1f90fd5 100644 --- a/go.sum +++ b/go.sum @@ -101,8 +101,6 @@ github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdII github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 h1:8l0uZ9ut9TowB1qNKbPFt/ar/5mqxhqcp0r+HWv1zps= -github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -1246,6 +1244,8 @@ github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6 github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/insecure/go.mod b/insecure/go.mod index e3748bc6c3d..e91291c8fff 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -190,7 +190,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-go-sdk v0.41.6 // indirect - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 // indirect + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce // indirect github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onflow/wal v0.0.0-20230529184820-bc9f8244608d // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 614d1513e79..5a16af690b0 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1192,8 +1192,8 @@ github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6 github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8/go.mod h1:73C8FlT4L/Qe4Cf5iXUNL8b2pvu4zs5dJMMJ5V2TjUI= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/go.mod b/integration/go.mod index be1de65103b..549e9dffb09 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -25,7 +25,7 @@ require ( github.com/onflow/flow-go-sdk v0.41.6 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 + github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce github.com/plus3it/gorecurcopy v0.0.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 @@ -330,6 +330,3 @@ require ( replace github.com/onflow/flow-go => ../ replace github.com/onflow/flow-go/insecure => ../insecure - -//TODO: Remove when branch UlyanaAndrukhiv/3138-rest-api-on-observers on onflow/flow will be merged -replace github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 => github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 diff --git a/integration/go.sum b/integration/go.sum index 91c55c2d7ae..15ed5a63d18 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -112,8 +112,6 @@ github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBY github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702 h1:8l0uZ9ut9TowB1qNKbPFt/ar/5mqxhqcp0r+HWv1zps= -github.com/UlyanaAndrukhiv/flow/protobuf/go/flow v0.0.0-20230612132933-04dabe947702/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= @@ -1382,6 +1380,8 @@ github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7 github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4 h1:5AnM9jIwkyHaY6+C3cWnt07oTOYctmwxvpiL25HRJws= github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e h1:RHaXPHvWCy3VM62+HTyu6DYq5T8rrK1gxxqogKuJ4S4= github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e/go.mod h1:CRX9eXtc9zHaRVTW1Xh4Cf5pZgKkQuu1NuSEVyHXr/0= github.com/onflow/go-bitswap v0.0.0-20221017184039-808c5791a8a8 h1:XcSR/n2aSVO7lOEsKScYALcpHlfowLwicZ9yVbL6bnA= From 276a101b04eebfd8bbb5975b86cf1c65e1b271d5 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 30 Jun 2023 12:26:34 +0300 Subject: [PATCH 073/169] Refactored according to commits --- .../node_builder/access_node_builder.go | 1 + cmd/observer/node_builder/observer_builder.go | 29 ++++++----- .../rest/apiproxy/rest_proxy_handler.go | 48 +++---------------- engine/access/rest/handler.go | 2 +- engine/access/rpc/engine.go | 2 + engine/access/rpc/engine_builder.go | 12 ----- engine/common/rpc/convert/convert.go | 21 ++++---- 7 files changed, 33 insertions(+), 82 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index c13a6db6f40..e5fe875d113 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1045,6 +1045,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.apiBurstlimits, builder.Me, backend, + backend, ) if err != nil { return nil, err diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 56512e02609..4070c277749 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -895,6 +895,19 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { backend.DefaultSnapshotHistoryLimit, backendConfig.ArchiveAddressList) + observerCollector := metrics.NewObserverCollector() + restHandler, err := restapiproxy.NewRestProxyHandler( + accessBackend, + builder.upstreamIdentities, + builder.apiTimeout, + config.MaxMsgSize, + builder.Logger, + observerCollector, + node.RootChainID.Chain()) + if err != nil { + return nil, err + } + engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, @@ -906,6 +919,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.apiBurstlimits, builder.Me, accessBackend, + restHandler, ) if err != nil { return nil, err @@ -917,8 +931,6 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { return nil, err } - observerCollector := metrics.NewObserverCollector() - rpcHandler := &apiproxy.FlowAccessAPIRouter{ Logger: builder.Logger, Metrics: observerCollector, @@ -931,22 +943,9 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { )), } - restHandler, err := restapiproxy.NewRestProxyHandler( - accessBackend, - builder.upstreamIdentities, - builder.apiTimeout, - config.MaxMsgSize, - builder.Logger, - observerCollector, - node.RootChainID.Chain()) - if err != nil { - return nil, err - } - // build the rpc engine builder.RpcEng, err = engineBuilder. WithRpcHandler(rpcHandler). - WithRestHandler(restHandler). WithLegacy(). Build() if err != nil { diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index d2a4c6adbd5..18e7cfe3d9e 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -77,33 +77,6 @@ func (r *RestProxyHandler) log(handler, rpc string, err error) { logger.Info().Msg("request succeeded") } -// GetLatestBlockHeader returns the latest block header and block status, if isSealed = true - returns the latest seal header. -func (r *RestProxyHandler) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { - upstream, err := r.FaultTolerantClient() - if err != nil { - return nil, flow.BlockStatusUnknown, err - } - - getLatestBlockHeaderRequest := &accessproto.GetLatestBlockHeaderRequest{ - IsSealed: isSealed, - } - latestBlockHeaderResponse, err := upstream.GetLatestBlockHeader(ctx, getLatestBlockHeaderRequest) - if err != nil { - return nil, flow.BlockStatusUnknown, err - } - blockHeader, err := convert.MessageToBlockHeader(latestBlockHeaderResponse.Block) - if err != nil { - return nil, flow.BlockStatusUnknown, err - } - blockStatus, err := convert.MessagesToBlockStatus(latestBlockHeaderResponse.BlockStatus) - if err != nil { - return nil, flow.BlockStatusUnknown, err - } - - r.log("upstream", "GetLatestBlockHeader", err) - return blockHeader, blockStatus, nil -} - // GetCollectionByID returns a collection by ID. func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) { upstream, err := r.FaultTolerantClient() @@ -120,15 +93,13 @@ func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identi return nil, err } - transactions := make([]flow.Identifier, len(collectionResponse.Collection.TransactionIds)) - for _, txId := range collectionResponse.Collection.TransactionIds { - transactions = append(transactions, convert.MessageToIdentifier(txId)) + transactions, err := convert.MessageToLightCollection(collectionResponse.Collection) + if err != nil { + return nil, err } r.log("upstream", "GetCollectionByID", err) - return &flow.LightCollection{ - Transactions: transactions, - }, nil + return transactions, nil } // SendTransaction sends already created transaction. @@ -144,13 +115,9 @@ func (r *RestProxyHandler) SendTransaction(ctx context.Context, tx *flow.Transac } _, err = upstream.SendTransaction(ctx, sendTransactionRequest) - if err != nil { - return err - } r.log("upstream", "SendTransaction", err) - return nil - + return err } // GetTransaction returns transaction by ID. @@ -310,10 +277,7 @@ func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType s return nil, err } - var blockIds [][]byte - for _, id := range blockIDs { - blockIds = append(blockIds, id[:]) - } + blockIds := convert.IdentifiersToMessages(blockIDs) getEventsForBlockIDsRequest := &accessproto.GetEventsForBlockIDsRequest{ Type: eventType, diff --git a/engine/access/rest/handler.go b/engine/access/rest/handler.go index c025a0c4c24..f2f7cc4a640 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/handler.go @@ -125,7 +125,7 @@ func (h *Handler) errorHandler(w http.ResponseWriter, err error, errorLogger zer return } if se.Code() == codes.Unavailable { - msg := fmt.Sprintf("Invalid Upstream request: %s", se.Message()) + msg := fmt.Sprintf("Failed to process request: %s", se.Message()) h.errorResponse(w, http.StatusServiceUnavailable, msg, errorLogger) return } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 7ef17d032c3..17f26cf0435 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -84,6 +84,7 @@ func NewBuilder(log zerolog.Logger, apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 me module.Local, backend *backend.Backend, + restHandler api.RestBackendApi, ) (*RPCEngineBuilder, error) { log = log.With().Str("engine", "rpc").Logger() @@ -139,6 +140,7 @@ func NewBuilder(log zerolog.Logger, config: config, chain: chainID.Chain(), restCollector: accessMetrics, + restHandler: restHandler, } backendNotifierActor, backendNotifierWorker := events.NewFinalizationActor(eng.notifyBackendOnBlockFinalized) eng.backendNotifierActor = backendNotifierActor diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index 7308a18597d..41c8bbdc192 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -68,12 +68,6 @@ func (builder *RPCEngineBuilder) WithRpcHandler(handler accessproto.AccessAPISer return builder } -// WithRestHandler specifies that the given `RestBackendApi` should be used for REST. -func (builder *RPCEngineBuilder) WithRestHandler(handler restapi.RestBackendApi) *RPCEngineBuilder { - builder.restHandler = handler - return builder -} - // WithLegacy specifies that a legacy access API should be instantiated // Returns self-reference for chaining. func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { @@ -114,11 +108,5 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer, rpcHandler) accessproto.RegisterAccessAPIServer(builder.secureGrpcServer, rpcHandler) - restHandler := builder.Engine.restHandler - if restHandler == nil { - restHandler = builder.backend - } - builder.Engine.restHandler = restHandler - return builder.Engine, nil } diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index f2f7a0eee9c..cfcae8ade60 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -379,17 +378,15 @@ func MessageToBlock(m *entities.Block) (*flow.Block, error) { }, nil } -func MessagesToBlockStatus(s entities.BlockStatus) (flow.BlockStatus, error) { - switch s { - case entities.BlockStatus_BLOCK_UNKNOWN: - return flow.BlockStatusUnknown, nil - case entities.BlockStatus_BLOCK_FINALIZED: - return flow.BlockStatusFinalized, nil - case entities.BlockStatus_BLOCK_SEALED: - return flow.BlockStatusSealed, nil +func MessageToLightCollection(m *entities.Collection) (*flow.LightCollection, error) { + transactions := make([]flow.Identifier, 0, len(m.TransactionIds)) + for _, txId := range m.TransactionIds { + transactions = append(transactions, MessageToIdentifier(txId)) } - return flow.BlockStatusUnknown, fmt.Errorf("failed to convert block status") + return &flow.LightCollection{ + Transactions: transactions, + }, nil } func MessagesToExecutionResultMetaList(m []*entities.ExecutionReceiptMeta) flow.ExecutionReceiptMetaList { @@ -657,14 +654,14 @@ func MessagesToBlockEvents(blocksEvents []*accessproto.EventsResponse_Result) [] evs := make([]flow.BlockEvents, len(blocksEvents)) for _, ev := range blocksEvents { var blockEvent flow.BlockEvents - MessageToBlockEvent(ev) + MessageToBlockEvents(ev) evs = append(evs, blockEvent) } return evs } -func MessageToBlockEvent(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { +func MessageToBlockEvents(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { return flow.BlockEvents{ BlockHeight: blockEvents.BlockHeight, BlockID: MessageToIdentifier(blockEvents.BlockId), From 74c47d597ab1da505b9a8b11d9bae6005f7a8b57 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 30 Jun 2023 13:38:03 +0300 Subject: [PATCH 074/169] Removed RestBackendApi and refactored rest using standard Access API's backend, refactored tests place --- engine/access/rest/api/api.go | 38 -- engine/access/rest/mock/rest_backend_api.go | 505 ------------------ engine/access/rest/routes/accounts.go | 4 +- .../rest/{tests => routes}/accounts_test.go | 105 +--- engine/access/rest/routes/blocks.go | 14 +- .../rest/{tests => routes}/blocks_test.go | 2 +- engine/access/rest/routes/collections.go | 4 +- .../{tests => routes}/collections_test.go | 2 +- engine/access/rest/routes/events.go | 4 +- .../rest/{tests => routes}/events_test.go | 7 +- engine/access/rest/routes/execution_result.go | 6 +- .../execution_result_test.go | 2 +- engine/access/rest/{ => routes}/handler.go | 10 +- engine/access/rest/routes/network.go | 4 +- .../rest/{tests => routes}/network_test.go | 2 +- .../access/rest/routes/node_version_info.go | 4 +- .../node_version_info_test.go | 2 +- engine/access/rest/{ => routes}/router.go | 35 +- engine/access/rest/routes/scripts.go | 4 +- .../rest/{tests => routes}/scripts_test.go | 2 +- .../rest/{tests => routes}/test_helpers.go | 13 +- engine/access/rest/routes/transactions.go | 7 +- .../{tests => routes}/transactions_test.go | 2 +- engine/access/rest/server.go | 7 +- engine/access/rest_api_test.go | 14 +- engine/access/rpc/engine.go | 6 +- engine/access/rpc/engine_builder.go | 5 - engine/access/rpc/rate_limit_test.go | 2 + engine/access/secure_grpcr_test.go | 1 + engine/common/rpc/convert/convert.go | 1 + 30 files changed, 82 insertions(+), 732 deletions(-) delete mode 100644 engine/access/rest/api/api.go delete mode 100644 engine/access/rest/mock/rest_backend_api.go rename engine/access/rest/{tests => routes}/accounts_test.go (61%) rename engine/access/rest/{tests => routes}/blocks_test.go (99%) rename engine/access/rest/{tests => routes}/collections_test.go (99%) rename engine/access/rest/{tests => routes}/events_test.go (98%) rename engine/access/rest/{tests => routes}/execution_result_test.go (99%) rename engine/access/rest/{ => routes}/handler.go (97%) rename engine/access/rest/{tests => routes}/network_test.go (98%) rename engine/access/rest/{tests => routes}/node_version_info_test.go (99%) rename engine/access/rest/{ => routes}/router.go (74%) rename engine/access/rest/{tests => routes}/scripts_test.go (99%) rename engine/access/rest/{tests => routes}/test_helpers.go (74%) rename engine/access/rest/{tests => routes}/transactions_test.go (99%) diff --git a/engine/access/rest/api/api.go b/engine/access/rest/api/api.go deleted file mode 100644 index 81d45cccb6e..00000000000 --- a/engine/access/rest/api/api.go +++ /dev/null @@ -1,38 +0,0 @@ -package api - -import ( - "context" - - "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/model/flow" -) - -// RestBackendApi is the backend API for REST service. -type RestBackendApi interface { - GetNetworkParameters(ctx context.Context) access.NetworkParameters - GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) - - GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) - - GetLatestBlock(ctx context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) - GetBlockByHeight(ctx context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) - GetBlockByID(ctx context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) - - GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) - - SendTransaction(ctx context.Context, tx *flow.TransactionBody) error - GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) - GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) - - GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) - - ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) - ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) - ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) - - GetEventsForHeightRange(ctx context.Context, eventType string, startHeight, endHeight uint64) ([]flow.BlockEvents, error) - GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) - - GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) - GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) -} diff --git a/engine/access/rest/mock/rest_backend_api.go b/engine/access/rest/mock/rest_backend_api.go deleted file mode 100644 index 61c3a938efa..00000000000 --- a/engine/access/rest/mock/rest_backend_api.go +++ /dev/null @@ -1,505 +0,0 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. - -package mock - -import ( - access "github.com/onflow/flow-go/access" - - context "context" - - flow "github.com/onflow/flow-go/model/flow" - - mock "github.com/stretchr/testify/mock" -) - -// RestBackendApi is an autogenerated mock type for the RestBackendApi type -type RestBackendApi struct { - mock.Mock -} - -// ExecuteScriptAtBlockHeight provides a mock function with given fields: ctx, blockHeight, script, arguments -func (_m *RestBackendApi) ExecuteScriptAtBlockHeight(ctx context.Context, blockHeight uint64, script []byte, arguments [][]byte) ([]byte, error) { - ret := _m.Called(ctx, blockHeight, script, arguments) - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, uint64, []byte, [][]byte) ([]byte, error)); ok { - return rf(ctx, blockHeight, script, arguments) - } - if rf, ok := ret.Get(0).(func(context.Context, uint64, []byte, [][]byte) []byte); ok { - r0 = rf(ctx, blockHeight, script, arguments) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, uint64, []byte, [][]byte) error); ok { - r1 = rf(ctx, blockHeight, script, arguments) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ExecuteScriptAtBlockID provides a mock function with given fields: ctx, blockID, script, arguments -func (_m *RestBackendApi) ExecuteScriptAtBlockID(ctx context.Context, blockID flow.Identifier, script []byte, arguments [][]byte) ([]byte, error) { - ret := _m.Called(ctx, blockID, script, arguments) - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, []byte, [][]byte) ([]byte, error)); ok { - return rf(ctx, blockID, script, arguments) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, []byte, [][]byte) []byte); ok { - r0 = rf(ctx, blockID, script, arguments) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, []byte, [][]byte) error); ok { - r1 = rf(ctx, blockID, script, arguments) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ExecuteScriptAtLatestBlock provides a mock function with given fields: ctx, script, arguments -func (_m *RestBackendApi) ExecuteScriptAtLatestBlock(ctx context.Context, script []byte, arguments [][]byte) ([]byte, error) { - ret := _m.Called(ctx, script, arguments) - - var r0 []byte - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte) ([]byte, error)); ok { - return rf(ctx, script, arguments) - } - if rf, ok := ret.Get(0).(func(context.Context, []byte, [][]byte) []byte); ok { - r0 = rf(ctx, script, arguments) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, []byte, [][]byte) error); ok { - r1 = rf(ctx, script, arguments) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetAccountAtBlockHeight provides a mock function with given fields: ctx, address, height -func (_m *RestBackendApi) GetAccountAtBlockHeight(ctx context.Context, address flow.Address, height uint64) (*flow.Account, error) { - ret := _m.Called(ctx, address, height) - - var r0 *flow.Account - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Address, uint64) (*flow.Account, error)); ok { - return rf(ctx, address, height) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Address, uint64) *flow.Account); ok { - r0 = rf(ctx, address, height) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Account) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Address, uint64) error); ok { - r1 = rf(ctx, address, height) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetBlockByHeight provides a mock function with given fields: ctx, height -func (_m *RestBackendApi) GetBlockByHeight(ctx context.Context, height uint64) (*flow.Block, flow.BlockStatus, error) { - ret := _m.Called(ctx, height) - - var r0 *flow.Block - var r1 flow.BlockStatus - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, uint64) (*flow.Block, flow.BlockStatus, error)); ok { - return rf(ctx, height) - } - if rf, ok := ret.Get(0).(func(context.Context, uint64) *flow.Block); ok { - r0 = rf(ctx, height) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Block) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, uint64) flow.BlockStatus); ok { - r1 = rf(ctx, height) - } else { - r1 = ret.Get(1).(flow.BlockStatus) - } - - if rf, ok := ret.Get(2).(func(context.Context, uint64) error); ok { - r2 = rf(ctx, height) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetBlockByID provides a mock function with given fields: ctx, id -func (_m *RestBackendApi) GetBlockByID(ctx context.Context, id flow.Identifier) (*flow.Block, flow.BlockStatus, error) { - ret := _m.Called(ctx, id) - - var r0 *flow.Block - var r1 flow.BlockStatus - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.Block, flow.BlockStatus, error)); ok { - return rf(ctx, id) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.Block); ok { - r0 = rf(ctx, id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Block) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) flow.BlockStatus); ok { - r1 = rf(ctx, id) - } else { - r1 = ret.Get(1).(flow.BlockStatus) - } - - if rf, ok := ret.Get(2).(func(context.Context, flow.Identifier) error); ok { - r2 = rf(ctx, id) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetCollectionByID provides a mock function with given fields: ctx, id -func (_m *RestBackendApi) GetCollectionByID(ctx context.Context, id flow.Identifier) (*flow.LightCollection, error) { - ret := _m.Called(ctx, id) - - var r0 *flow.LightCollection - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.LightCollection, error)); ok { - return rf(ctx, id) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.LightCollection); ok { - r0 = rf(ctx, id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.LightCollection) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, id) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetEventsForBlockIDs provides a mock function with given fields: ctx, eventType, blockIDs -func (_m *RestBackendApi) GetEventsForBlockIDs(ctx context.Context, eventType string, blockIDs []flow.Identifier) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, blockIDs) - - var r0 []flow.BlockEvents - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, blockIDs) - } - if rf, ok := ret.Get(0).(func(context.Context, string, []flow.Identifier) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, blockIDs) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.BlockEvents) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, []flow.Identifier) error); ok { - r1 = rf(ctx, eventType, blockIDs) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetEventsForHeightRange provides a mock function with given fields: ctx, eventType, startHeight, endHeight -func (_m *RestBackendApi) GetEventsForHeightRange(ctx context.Context, eventType string, startHeight uint64, endHeight uint64) ([]flow.BlockEvents, error) { - ret := _m.Called(ctx, eventType, startHeight, endHeight) - - var r0 []flow.BlockEvents - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) ([]flow.BlockEvents, error)); ok { - return rf(ctx, eventType, startHeight, endHeight) - } - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) []flow.BlockEvents); ok { - r0 = rf(ctx, eventType, startHeight, endHeight) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]flow.BlockEvents) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64) error); ok { - r1 = rf(ctx, eventType, startHeight, endHeight) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetExecutionResultByID provides a mock function with given fields: ctx, id -func (_m *RestBackendApi) GetExecutionResultByID(ctx context.Context, id flow.Identifier) (*flow.ExecutionResult, error) { - ret := _m.Called(ctx, id) - - var r0 *flow.ExecutionResult - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.ExecutionResult, error)); ok { - return rf(ctx, id) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.ExecutionResult); ok { - r0 = rf(ctx, id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.ExecutionResult) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, id) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetExecutionResultForBlockID provides a mock function with given fields: ctx, blockID -func (_m *RestBackendApi) GetExecutionResultForBlockID(ctx context.Context, blockID flow.Identifier) (*flow.ExecutionResult, error) { - ret := _m.Called(ctx, blockID) - - var r0 *flow.ExecutionResult - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.ExecutionResult, error)); ok { - return rf(ctx, blockID) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.ExecutionResult); ok { - r0 = rf(ctx, blockID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.ExecutionResult) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, blockID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetLatestBlock provides a mock function with given fields: ctx, isSealed -func (_m *RestBackendApi) GetLatestBlock(ctx context.Context, isSealed bool) (*flow.Block, flow.BlockStatus, error) { - ret := _m.Called(ctx, isSealed) - - var r0 *flow.Block - var r1 flow.BlockStatus - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, bool) (*flow.Block, flow.BlockStatus, error)); ok { - return rf(ctx, isSealed) - } - if rf, ok := ret.Get(0).(func(context.Context, bool) *flow.Block); ok { - r0 = rf(ctx, isSealed) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Block) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, bool) flow.BlockStatus); ok { - r1 = rf(ctx, isSealed) - } else { - r1 = ret.Get(1).(flow.BlockStatus) - } - - if rf, ok := ret.Get(2).(func(context.Context, bool) error); ok { - r2 = rf(ctx, isSealed) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetLatestBlockHeader provides a mock function with given fields: ctx, isSealed -func (_m *RestBackendApi) GetLatestBlockHeader(ctx context.Context, isSealed bool) (*flow.Header, flow.BlockStatus, error) { - ret := _m.Called(ctx, isSealed) - - var r0 *flow.Header - var r1 flow.BlockStatus - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, bool) (*flow.Header, flow.BlockStatus, error)); ok { - return rf(ctx, isSealed) - } - if rf, ok := ret.Get(0).(func(context.Context, bool) *flow.Header); ok { - r0 = rf(ctx, isSealed) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.Header) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, bool) flow.BlockStatus); ok { - r1 = rf(ctx, isSealed) - } else { - r1 = ret.Get(1).(flow.BlockStatus) - } - - if rf, ok := ret.Get(2).(func(context.Context, bool) error); ok { - r2 = rf(ctx, isSealed) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetNetworkParameters provides a mock function with given fields: ctx -func (_m *RestBackendApi) GetNetworkParameters(ctx context.Context) access.NetworkParameters { - ret := _m.Called(ctx) - - var r0 access.NetworkParameters - if rf, ok := ret.Get(0).(func(context.Context) access.NetworkParameters); ok { - r0 = rf(ctx) - } else { - r0 = ret.Get(0).(access.NetworkParameters) - } - - return r0 -} - -// GetNodeVersionInfo provides a mock function with given fields: ctx -func (_m *RestBackendApi) GetNodeVersionInfo(ctx context.Context) (*access.NodeVersionInfo, error) { - ret := _m.Called(ctx) - - var r0 *access.NodeVersionInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (*access.NodeVersionInfo, error)); ok { - return rf(ctx) - } - if rf, ok := ret.Get(0).(func(context.Context) *access.NodeVersionInfo); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*access.NodeVersionInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(ctx) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetTransaction provides a mock function with given fields: ctx, id -func (_m *RestBackendApi) GetTransaction(ctx context.Context, id flow.Identifier) (*flow.TransactionBody, error) { - ret := _m.Called(ctx, id) - - var r0 *flow.TransactionBody - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) (*flow.TransactionBody, error)); ok { - return rf(ctx, id) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier) *flow.TransactionBody); ok { - r0 = rf(ctx, id) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*flow.TransactionBody) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier) error); ok { - r1 = rf(ctx, id) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetTransactionResult provides a mock function with given fields: ctx, id, blockID, collectionID -func (_m *RestBackendApi) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { - ret := _m.Called(ctx, id, blockID, collectionID) - - var r0 *access.TransactionResult - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) (*access.TransactionResult, error)); ok { - return rf(ctx, id, blockID, collectionID) - } - if rf, ok := ret.Get(0).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) *access.TransactionResult); ok { - r0 = rf(ctx, id, blockID, collectionID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*access.TransactionResult) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, flow.Identifier, flow.Identifier, flow.Identifier) error); ok { - r1 = rf(ctx, id, blockID, collectionID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// SendTransaction provides a mock function with given fields: ctx, tx -func (_m *RestBackendApi) SendTransaction(ctx context.Context, tx *flow.TransactionBody) error { - ret := _m.Called(ctx, tx) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *flow.TransactionBody) error); ok { - r0 = rf(ctx, tx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewRestBackendApi interface { - mock.TestingT - Cleanup(func()) -} - -// NewRestBackendApi creates a new instance of RestBackendApi. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewRestBackendApi(t mockConstructorTestingTNewRestBackendApi) *RestBackendApi { - mock := &RestBackendApi{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/engine/access/rest/routes/accounts.go b/engine/access/rest/routes/accounts.go index d9f16562f1d..972c2ba68ac 100644 --- a/engine/access/rest/routes/accounts.go +++ b/engine/access/rest/routes/accounts.go @@ -1,13 +1,13 @@ package routes import ( - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetAccount handler retrieves account by address and returns the response -func GetAccount(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetAccount(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetAccountRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/accounts_test.go b/engine/access/rest/routes/accounts_test.go similarity index 61% rename from engine/access/rest/tests/accounts_test.go rename to engine/access/rest/routes/accounts_test.go index 0ff84f79075..b8bebea8e85 100644 --- a/engine/access/rest/tests/accounts_test.go +++ b/engine/access/rest/routes/accounts_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "fmt" @@ -13,7 +13,6 @@ import ( "github.com/onflow/flow-go/access/mock" "github.com/onflow/flow-go/engine/access/rest/middleware" - restmock "github.com/onflow/flow-go/engine/access/rest/mock" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -137,108 +136,6 @@ func TestAccessGetAccount(t *testing.T) { }) } -// TestObserverGetAccount tests the get account request forwarding to an upstream. -// -// Runs the following tests: -// 1. Get account by address at latest sealed block. -// 2. Get account by address at latest finalized block. -// 3. Get account by address at height. -// 4. Get account by address at height condensed. -// 5. Get invalid account. -func TestObserverGetAccount(t *testing.T) { - backend := &restmock.RestBackendApi{} - - t.Run("get by address at latest sealed block", func(t *testing.T) { - account := accountFixture(t) - var height uint64 = 100 - block := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) - - req := getAccountRequest(t, account, sealedHeightQueryParam, expandableFieldKeys, expandableFieldContracts) - - backend.Mock. - On("GetLatestBlockHeader", mocktestify.Anything, true). - Return(block, flow.BlockStatusSealed, nil) - - backend.Mock. - On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). - Return(account, nil) - - expected := expectedExpandedResponse(account) - - assertOKResponse(t, req, expected, backend) - mocktestify.AssertExpectationsForObjects(t, backend) - }) - - t.Run("get by address at latest finalized block", func(t *testing.T) { - var height uint64 = 100 - block := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) - account := accountFixture(t) - - req := getAccountRequest(t, account, finalHeightQueryParam, expandableFieldKeys, expandableFieldContracts) - - backend.Mock. - On("GetLatestBlockHeader", mocktestify.Anything, false). - Return(block, flow.BlockStatusFinalized, nil) - backend.Mock. - On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). - Return(account, nil) - - expected := expectedExpandedResponse(account) - - assertOKResponse(t, req, expected, backend) - mocktestify.AssertExpectationsForObjects(t, backend) - }) - - t.Run("get by address at height", func(t *testing.T) { - var height uint64 = 1337 - account := accountFixture(t) - req := getAccountRequest(t, account, fmt.Sprintf("%d", height), expandableFieldKeys, expandableFieldContracts) - - backend.Mock. - On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). - Return(account, nil) - - expected := expectedExpandedResponse(account) - - assertOKResponse(t, req, expected, backend) - mocktestify.AssertExpectationsForObjects(t, backend) - }) - - t.Run("get by address at height condensed", func(t *testing.T) { - var height uint64 = 1337 - account := accountFixture(t) - req := getAccountRequest(t, account, fmt.Sprintf("%d", height)) - - backend.Mock. - On("GetAccountAtBlockHeight", mocktestify.Anything, account.Address, height). - Return(account, nil) - - expected := expectedCondensedResponse(account) - - assertOKResponse(t, req, expected, backend) - mocktestify.AssertExpectationsForObjects(t, backend) - }) - - t.Run("get invalid", func(t *testing.T) { - tests := []struct { - url string - out string - }{ - {accountURL(t, "123", ""), `{"code":400, "message":"invalid address"}`}, - {accountURL(t, unittest.AddressFixture().String(), "foo"), `{"code":400, "message":"invalid height format"}`}, - } - - for i, test := range tests { - req, _ := http.NewRequest("GET", test.url, nil) - rr, err := executeRequest(req, backend) - assert.NoError(t, err) - - assert.Equal(t, http.StatusBadRequest, rr.Code) - assert.JSONEq(t, test.out, rr.Body.String(), fmt.Sprintf("test #%d failed: %v", i, test)) - } - }) -} - func expectedExpandedResponse(account *flow.Account) string { return fmt.Sprintf(`{ "address":"%s", diff --git a/engine/access/rest/routes/blocks.go b/engine/access/rest/routes/blocks.go index 7f0537a0c07..cd8547bca93 100644 --- a/engine/access/rest/routes/blocks.go +++ b/engine/access/rest/routes/blocks.go @@ -8,14 +8,14 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" ) // GetBlocksByIDs gets blocks by provided ID or list of IDs. -func GetBlocksByIDs(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockByIDsRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -35,7 +35,7 @@ func GetBlocksByIDs(r *request.Request, backend api.RestBackendApi, link models. } // GetBlocksByHeight gets blocks by height. -func GetBlocksByHeight(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetBlocksByHeight(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -92,7 +92,7 @@ func GetBlocksByHeight(r *request.Request, backend api.RestBackendApi, link mode } // GetBlockPayloadByID gets block payload by ID -func GetBlockPayloadByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetBlockPayloadByID(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetBlockPayloadRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -113,7 +113,7 @@ func GetBlockPayloadByID(r *request.Request, backend api.RestBackendApi, link mo return payload, nil } -func getBlock(option blockProviderOption, req *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (*models.Block, error) { +func getBlock(option blockProviderOption, req *request.Request, backend access.API, link models.LinkGenerator) (*models.Block, error) { // lookup block blkProvider := NewBlockProvider(backend, option) blk, blockStatus, err := blkProvider.getBlock(req.Context()) @@ -153,7 +153,7 @@ type blockProvider struct { height uint64 latest bool sealed bool - backend api.RestBackendApi + backend access.API } type blockProviderOption func(blkProvider *blockProvider) @@ -181,7 +181,7 @@ func forFinalized(queryParam uint64) blockProviderOption { } } -func NewBlockProvider(backend api.RestBackendApi, options ...blockProviderOption) *blockProvider { +func NewBlockProvider(backend access.API, options ...blockProviderOption) *blockProvider { blkProvider := &blockProvider{ backend: backend, } diff --git a/engine/access/rest/tests/blocks_test.go b/engine/access/rest/routes/blocks_test.go similarity index 99% rename from engine/access/rest/tests/blocks_test.go rename to engine/access/rest/routes/blocks_test.go index 7b8b18cdc5f..3abccc9c78a 100644 --- a/engine/access/rest/tests/blocks_test.go +++ b/engine/access/rest/routes/blocks_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "fmt" diff --git a/engine/access/rest/routes/collections.go b/engine/access/rest/routes/collections.go index 182581f90f0..47b6150f480 100644 --- a/engine/access/rest/routes/collections.go +++ b/engine/access/rest/routes/collections.go @@ -1,14 +1,14 @@ package routes import ( - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" ) // GetCollectionByID retrieves a collection by ID and builds a response -func GetCollectionByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetCollectionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetCollectionRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/collections_test.go b/engine/access/rest/routes/collections_test.go similarity index 99% rename from engine/access/rest/tests/collections_test.go rename to engine/access/rest/routes/collections_test.go index db57c4101e5..de05152b6d5 100644 --- a/engine/access/rest/tests/collections_test.go +++ b/engine/access/rest/routes/collections_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "encoding/json" diff --git a/engine/access/rest/routes/events.go b/engine/access/rest/routes/events.go index 36eee728386..4f03624c768 100644 --- a/engine/access/rest/routes/events.go +++ b/engine/access/rest/routes/events.go @@ -3,7 +3,7 @@ package routes import ( "fmt" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) @@ -12,7 +12,7 @@ const BlockQueryParam = "block_ids" const EventTypeQuery = "type" // GetEvents for the provided block range or list of block IDs filtered by type. -func GetEvents(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { +func GetEvents(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetEventsRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/events_test.go b/engine/access/rest/routes/events_test.go similarity index 98% rename from engine/access/rest/tests/events_test.go rename to engine/access/rest/routes/events_test.go index 2de8a4b0834..47d4d89fd52 100644 --- a/engine/access/rest/tests/events_test.go +++ b/engine/access/rest/routes/events_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "encoding/json" @@ -17,7 +17,6 @@ import ( "google.golang.org/grpc/status" "github.com/onflow/flow-go/access/mock" - "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -138,7 +137,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q := u.Query() if len(blockIDs) > 0 { - q.Add(routes.BlockQueryParam, strings.Join(blockIDs, ",")) + q.Add(BlockQueryParam, strings.Join(blockIDs, ",")) } if start != "" && end != "" { @@ -146,7 +145,7 @@ func getEventReq(t *testing.T, eventType string, start string, end string, block q.Add(endHeightQueryParam, end) } - q.Add(routes.EventTypeQuery, eventType) + q.Add(EventTypeQuery, eventType) u.RawQuery = q.Encode() diff --git a/engine/access/rest/routes/execution_result.go b/engine/access/rest/routes/execution_result.go index f923ce7905c..b999665b26b 100644 --- a/engine/access/rest/routes/execution_result.go +++ b/engine/access/rest/routes/execution_result.go @@ -3,13 +3,13 @@ package routes import ( "fmt" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetExecutionResultsByBlockIDs gets Execution Result payload by block IDs. -func GetExecutionResultsByBlockIDs(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultsByBlockIDs(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultByBlockIDsRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -35,7 +35,7 @@ func GetExecutionResultsByBlockIDs(r *request.Request, backend api.RestBackendAp } // GetExecutionResultByID gets execution result by the ID. -func GetExecutionResultByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetExecutionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetExecutionResultRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/execution_result_test.go b/engine/access/rest/routes/execution_result_test.go similarity index 99% rename from engine/access/rest/tests/execution_result_test.go rename to engine/access/rest/routes/execution_result_test.go index 23038b1abdb..ba74974af1a 100644 --- a/engine/access/rest/tests/execution_result_test.go +++ b/engine/access/rest/routes/execution_result_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "fmt" diff --git a/engine/access/rest/handler.go b/engine/access/rest/routes/handler.go similarity index 97% rename from engine/access/rest/handler.go rename to engine/access/rest/routes/handler.go index f2f7cc4a640..e323843e50e 100644 --- a/engine/access/rest/handler.go +++ b/engine/access/rest/routes/handler.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "encoding/json" @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/engine/access/rest/util" @@ -25,7 +25,7 @@ const MaxRequestSize = 2 << 20 // 2MB // it fetches necessary resources and returns an error or response model. type ApiHandlerFunc func( r *request.Request, - backend api.RestBackendApi, + backend access.API, generator models.LinkGenerator, ) (interface{}, error) @@ -34,7 +34,7 @@ type ApiHandlerFunc func( // wraps functionality for handling error and responses outside of endpoint handling. type Handler struct { logger zerolog.Logger - backend api.RestBackendApi + backend access.API linkGenerator models.LinkGenerator apiHandlerFunc ApiHandlerFunc chain flow.Chain @@ -42,7 +42,7 @@ type Handler struct { func NewHandler( logger zerolog.Logger, - backend api.RestBackendApi, + backend access.API, handlerFunc ApiHandlerFunc, generator models.LinkGenerator, chain flow.Chain, diff --git a/engine/access/rest/routes/network.go b/engine/access/rest/routes/network.go index 88aa787108e..82abcbb6d49 100644 --- a/engine/access/rest/routes/network.go +++ b/engine/access/rest/routes/network.go @@ -1,13 +1,13 @@ package routes import ( - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNetworkParameters returns network-wide parameters of the blockchain -func GetNetworkParameters(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { +func GetNetworkParameters(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { params := backend.GetNetworkParameters(r.Context()) var response models.NetworkParameters diff --git a/engine/access/rest/tests/network_test.go b/engine/access/rest/routes/network_test.go similarity index 98% rename from engine/access/rest/tests/network_test.go rename to engine/access/rest/routes/network_test.go index a841f076f60..00d0ca03944 100644 --- a/engine/access/rest/tests/network_test.go +++ b/engine/access/rest/routes/network_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "fmt" diff --git a/engine/access/rest/routes/node_version_info.go b/engine/access/rest/routes/node_version_info.go index daf658e8869..31e172bba9f 100644 --- a/engine/access/rest/routes/node_version_info.go +++ b/engine/access/rest/routes/node_version_info.go @@ -1,13 +1,13 @@ package routes import ( - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetNodeVersionInfo returns node version information -func GetNodeVersionInfo(r *request.Request, backend api.RestBackendApi, _ models.LinkGenerator) (interface{}, error) { +func GetNodeVersionInfo(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { params, err := backend.GetNodeVersionInfo(r.Context()) if err != nil { return nil, err diff --git a/engine/access/rest/tests/node_version_info_test.go b/engine/access/rest/routes/node_version_info_test.go similarity index 99% rename from engine/access/rest/tests/node_version_info_test.go rename to engine/access/rest/routes/node_version_info_test.go index 25213102934..25f19ae1f3c 100644 --- a/engine/access/rest/tests/node_version_info_test.go +++ b/engine/access/rest/routes/node_version_info_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "fmt" diff --git a/engine/access/rest/router.go b/engine/access/rest/routes/router.go similarity index 74% rename from engine/access/rest/router.go rename to engine/access/rest/routes/router.go index ef1f9665a91..b9b0b650183 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/routes/router.go @@ -1,4 +1,4 @@ -package rest +package routes import ( "net/http" @@ -6,15 +6,14 @@ import ( "github.com/gorilla/mux" "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/middleware" "github.com/onflow/flow-go/engine/access/rest/models" - "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) -func NewRouter(backend api.RestBackendApi, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { +func NewRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*mux.Router, error) { router := mux.NewRouter().StrictSlash(true) v1SubRouter := router.PathPrefix("/v1").Subrouter() @@ -48,70 +47,70 @@ var Routes = []route{{ Method: http.MethodGet, Pattern: "/transactions/{id}", Name: "getTransactionByID", - Handler: routes.GetTransactionByID, + Handler: GetTransactionByID, }, { Method: http.MethodPost, Pattern: "/transactions", Name: "createTransaction", - Handler: routes.CreateTransaction, + Handler: CreateTransaction, }, { Method: http.MethodGet, Pattern: "/transaction_results/{id}", Name: "getTransactionResultByID", - Handler: routes.GetTransactionResultByID, + Handler: GetTransactionResultByID, }, { Method: http.MethodGet, Pattern: "/blocks/{id}", Name: "getBlocksByIDs", - Handler: routes.GetBlocksByIDs, + Handler: GetBlocksByIDs, }, { Method: http.MethodGet, Pattern: "/blocks", Name: "getBlocksByHeight", - Handler: routes.GetBlocksByHeight, + Handler: GetBlocksByHeight, }, { Method: http.MethodGet, Pattern: "/blocks/{id}/payload", Name: "getBlockPayloadByID", - Handler: routes.GetBlockPayloadByID, + Handler: GetBlockPayloadByID, }, { Method: http.MethodGet, Pattern: "/execution_results/{id}", Name: "getExecutionResultByID", - Handler: routes.GetExecutionResultByID, + Handler: GetExecutionResultByID, }, { Method: http.MethodGet, Pattern: "/execution_results", Name: "getExecutionResultByBlockID", - Handler: routes.GetExecutionResultsByBlockIDs, + Handler: GetExecutionResultsByBlockIDs, }, { Method: http.MethodGet, Pattern: "/collections/{id}", Name: "getCollectionByID", - Handler: routes.GetCollectionByID, + Handler: GetCollectionByID, }, { Method: http.MethodPost, Pattern: "/scripts", Name: "executeScript", - Handler: routes.ExecuteScript, + Handler: ExecuteScript, }, { Method: http.MethodGet, Pattern: "/accounts/{address}", Name: "getAccount", - Handler: routes.GetAccount, + Handler: GetAccount, }, { Method: http.MethodGet, Pattern: "/events", Name: "getEvents", - Handler: routes.GetEvents, + Handler: GetEvents, }, { Method: http.MethodGet, Pattern: "/network/parameters", Name: "getNetworkParameters", - Handler: routes.GetNetworkParameters, + Handler: GetNetworkParameters, }, { Method: http.MethodGet, Pattern: "/node_version_info", Name: "getNodeVersionInfo", - Handler: routes.GetNodeVersionInfo, + Handler: GetNodeVersionInfo, }} diff --git a/engine/access/rest/routes/scripts.go b/engine/access/rest/routes/scripts.go index e991dd3a89e..8627470ab88 100644 --- a/engine/access/rest/routes/scripts.go +++ b/engine/access/rest/routes/scripts.go @@ -1,14 +1,14 @@ package routes import ( - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" "github.com/onflow/flow-go/model/flow" ) // ExecuteScript handler sends the script from the request to be executed. -func ExecuteScript(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func ExecuteScript(r *request.Request, backend access.API, _ models.LinkGenerator) (interface{}, error) { req, err := r.GetScriptRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/scripts_test.go b/engine/access/rest/routes/scripts_test.go similarity index 99% rename from engine/access/rest/tests/scripts_test.go rename to engine/access/rest/routes/scripts_test.go index da498e59dc2..a3f2a64663c 100644 --- a/engine/access/rest/tests/scripts_test.go +++ b/engine/access/rest/routes/scripts_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "bytes" diff --git a/engine/access/rest/tests/test_helpers.go b/engine/access/rest/routes/test_helpers.go similarity index 74% rename from engine/access/rest/tests/test_helpers.go rename to engine/access/rest/routes/test_helpers.go index 388b5ca999d..e512cc94434 100644 --- a/engine/access/rest/tests/test_helpers.go +++ b/engine/access/rest/routes/test_helpers.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "bytes" @@ -11,8 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/onflow/flow-go/engine/access/rest" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/metrics" ) @@ -27,11 +26,11 @@ const ( heightQueryParam = "height" ) -func executeRequest(req *http.Request, backend api.RestBackendApi) (*httptest.ResponseRecorder, error) { +func executeRequest(req *http.Request, backend access.API) (*httptest.ResponseRecorder, error) { var b bytes.Buffer logger := zerolog.New(&b) - router, err := rest.NewRouter(backend, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) + router, err := NewRouter(backend, logger, flow.Testnet.Chain(), metrics.NewNoopCollector()) if err != nil { return nil, err } @@ -41,11 +40,11 @@ func executeRequest(req *http.Request, backend api.RestBackendApi) (*httptest.Re return rr, nil } -func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend api.RestBackendApi) { +func assertOKResponse(t *testing.T, req *http.Request, expectedRespBody string, backend access.API) { assertResponse(t, req, http.StatusOK, expectedRespBody, backend) } -func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend api.RestBackendApi) { +func assertResponse(t *testing.T, req *http.Request, status int, expectedRespBody string, backend access.API) { rr, err := executeRequest(req, backend) assert.NoError(t, err) actualResponseBody := rr.Body.String() diff --git a/engine/access/rest/routes/transactions.go b/engine/access/rest/routes/transactions.go index 763e9098299..b77aead82b4 100644 --- a/engine/access/rest/routes/transactions.go +++ b/engine/access/rest/routes/transactions.go @@ -2,13 +2,12 @@ package routes import ( "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/access/rest/request" ) // GetTransactionByID gets a transaction by requested ID. -func GetTransactionByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -34,7 +33,7 @@ func GetTransactionByID(r *request.Request, backend api.RestBackendApi, link mod } // GetTransactionResultByID retrieves transaction result by the transaction ID. -func GetTransactionResultByID(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func GetTransactionResultByID(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.GetTransactionResultRequest() if err != nil { return nil, models.NewBadRequestError(err) @@ -51,7 +50,7 @@ func GetTransactionResultByID(r *request.Request, backend api.RestBackendApi, li } // CreateTransaction creates a new transaction from provided payload. -func CreateTransaction(r *request.Request, backend api.RestBackendApi, link models.LinkGenerator) (interface{}, error) { +func CreateTransaction(r *request.Request, backend access.API, link models.LinkGenerator) (interface{}, error) { req, err := r.CreateTransactionRequest() if err != nil { return nil, models.NewBadRequestError(err) diff --git a/engine/access/rest/tests/transactions_test.go b/engine/access/rest/routes/transactions_test.go similarity index 99% rename from engine/access/rest/tests/transactions_test.go rename to engine/access/rest/routes/transactions_test.go index 3d81a0b5eff..6adde49b245 100644 --- a/engine/access/rest/tests/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -1,4 +1,4 @@ -package tests +package routes import ( "bytes" diff --git a/engine/access/rest/server.go b/engine/access/rest/server.go index d07983abb64..4a4b1be6f0e 100644 --- a/engine/access/rest/server.go +++ b/engine/access/rest/server.go @@ -7,14 +7,15 @@ import ( "github.com/rs/cors" "github.com/rs/zerolog" - "github.com/onflow/flow-go/engine/access/rest/api" + "github.com/onflow/flow-go/access" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" ) // NewServer returns an HTTP server initialized with the REST API handler -func NewServer(serverAPI api.RestBackendApi, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { - router, err := NewRouter(serverAPI, logger, chain, restCollector) +func NewServer(serverAPI access.API, listenAddress string, logger zerolog.Logger, chain flow.Chain, restCollector module.RestMetrics) (*http.Server, error) { + router, err := routes.NewRouter(serverAPI, logger, chain, restCollector) if err != nil { return nil, err } diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 61c4f75de0b..1c09b77bac4 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -19,9 +19,8 @@ import ( "github.com/stretchr/testify/suite" accessmock "github.com/onflow/flow-go/engine/access/mock" - "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/request" - "github.com/onflow/flow-go/engine/access/rest/tests" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/model/flow" @@ -152,6 +151,7 @@ func (suite *RestAPITestSuite) SetupTest() { nil, suite.me, backend, + backend, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() @@ -369,7 +369,7 @@ func (suite *RestAPITestSuite) TestRequestSizeRestriction() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() // make a request of size larger than the max permitted size - requestBytes := make([]byte, rest.MaxRequestSize+1) + requestBytes := make([]byte, routes.MaxRequestSize+1) script := restclient.ScriptsBody{ Script: string(requestBytes), } @@ -396,13 +396,13 @@ func assertError(t *testing.T, resp *http.Response, err error, expectedCode int, func optionsForBlockByID() *restclient.BlocksApiBlocksIdGetOpts { return &restclient.BlocksApiBlocksIdGetOpts{ - Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id"}), } } func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), StartHeight: optional.NewInterface(startHeight), EndHeight: optional.NewInterface(endHeight), @@ -411,7 +411,7 @@ func optionsForBlockByStartEndHeight(startHeight, endHeight uint64) *restclient. func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(heights), } @@ -419,7 +419,7 @@ func optionsForBlockByHeights(heights []uint64) *restclient.BlocksApiBlocksGetOp func optionsForFinalizedBlock(finalOrSealed string) *restclient.BlocksApiBlocksGetOpts { return &restclient.BlocksApiBlocksGetOpts{ - Expand: optional.NewInterface([]string{tests.ExpandableFieldPayload}), + Expand: optional.NewInterface([]string{routes.ExpandableFieldPayload}), Select_: optional.NewInterface([]string{"header.id", "header.height"}), Height: optional.NewInterface(finalOrSealed), } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 17f26cf0435..8577a1ba676 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -13,9 +13,9 @@ import ( "github.com/rs/zerolog" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine/access/rest" - "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" @@ -69,7 +69,7 @@ type Engine struct { secureGrpcAddress net.Addr restAPIAddress net.Addr - restHandler api.RestBackendApi + restHandler access.API } type Option func(*RPCEngineBuilder) @@ -84,7 +84,7 @@ func NewBuilder(log zerolog.Logger, apiBurstLimits map[string]int, // the api burst limit (max calls at the same time) for each of the Access API e.g. Ping->50, GetTransaction->10 me module.Local, backend *backend.Backend, - restHandler api.RestBackendApi, + restHandler access.API, ) (*RPCEngineBuilder, error) { log = log.With().Str("engine", "rpc").Logger() diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index 41c8bbdc192..f63d30289b9 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -8,7 +8,6 @@ import ( "github.com/onflow/flow-go/access" legacyaccess "github.com/onflow/flow-go/access/legacy" "github.com/onflow/flow-go/consensus/hotstuff" - restapi "github.com/onflow/flow-go/engine/access/rest/api" "github.com/onflow/flow-go/module" accessproto "github.com/onflow/flow/protobuf/go/flow/access" @@ -39,10 +38,6 @@ func (builder *RPCEngineBuilder) RpcHandler() accessproto.AccessAPIServer { return builder.rpcHandler } -func (builder *RPCEngineBuilder) RestHandler() restapi.RestBackendApi { - return builder.restHandler -} - // WithBlockSignerDecoder specifies that signer indices in block headers should be translated // to full node IDs with the given decoder. // Caution: diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 87551c96d5d..ea25c9370db 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -150,7 +150,9 @@ func (suite *RateLimitTestSuite) SetupTest() { apiRateLimt, apiBurstLimt, suite.me, + backend, backend) + require.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() require.NoError(suite.T(), err) diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index a5975a7e92c..d8ed0169ce2 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -141,6 +141,7 @@ func (suite *SecureGRPCTestSuite) SetupTest() { nil, suite.me, backend, + backend, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() diff --git a/engine/common/rpc/convert/convert.go b/engine/common/rpc/convert/convert.go index cfcae8ade60..ed1c1c10b63 100644 --- a/engine/common/rpc/convert/convert.go +++ b/engine/common/rpc/convert/convert.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" accessproto "github.com/onflow/flow/protobuf/go/flow/access" From e878214e65b74a90949870df5ebdab28187fb34e Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 30 Jun 2023 18:05:03 +0300 Subject: [PATCH 075/169] Updated according to comments --- .../node_builder/access_node_builder.go | 35 +++++++++---------- cmd/observer/node_builder/observer_builder.go | 31 ++++++++-------- engine/access/rest_api_test.go | 20 +++++------ engine/access/rpc/engine.go | 12 +++---- engine/access/rpc/engine_builder.go | 12 +++---- engine/access/rpc/rate_limit_test.go | 17 ++++----- engine/access/secure_grpcr_test.go | 19 +++++----- engine/access/state_stream/engine.go | 7 ++-- module/grpcserver/server.go | 10 +++--- module/grpcserver/server_builder.go | 4 --- 10 files changed, 73 insertions(+), 94 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 1397d4bd732..b70d11c6acc 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -243,9 +243,9 @@ type FlowAccessNodeBuilder struct { SyncEng *synceng.Engine StateStreamEng *state_stream.Engine - // grpc server builders - secureGrpcServer *grpcserver.GrpcServerBuilder - unsecureGrpcServer *grpcserver.GrpcServerBuilder + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { @@ -992,20 +992,27 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("creating grpc servers", func(node *cmd.NodeConfig) error { - builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + var err error + builder.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(node.Logger, builder.rpcConf.SecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits, - grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() + if err != nil { + return err + } - builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(node.Logger, builder.rpcConf.UnsecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, - builder.apiBurstlimits) + builder.apiBurstlimits).Build() + if err != nil { + return err + } return nil }). @@ -1121,21 +1128,11 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { } builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - secureGrpcServer, err := builder.secureGrpcServer.Build() - if err != nil { - return nil, err - } - - return secureGrpcServer, nil + return builder.secureGrpcServer, nil }) builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - unsecureGrpcServer, err := builder.unsecureGrpcServer.Build() - if err != nil { - return nil, err - } - - return unsecureGrpcServer, nil + return builder.unsecureGrpcServer, nil }) builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 2dbfc0aaf6f..e4898cfeaa6 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -642,7 +642,9 @@ func (builder *ObserverServiceBuilder) Initialize() error { builder.enqueueConnectWithStakedAN() - builder.enqueueRPCServer() + if err := builder.enqueueRPCServer(); err != nil { + return err + } if builder.BaseConfig.MetricsEnabled { builder.EnqueueMetricsServerInit() @@ -846,21 +848,27 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { }) } -func (builder *ObserverServiceBuilder) enqueueRPCServer() { - secureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, +func (builder *ObserverServiceBuilder) enqueueRPCServer() error { + secureGrpcServer, err := grpcserver.NewGrpcServerBuilder(builder.Logger, builder.rpcConf.SecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits, - grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)) + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() + if err != nil { + return err + } - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(builder.Logger, + unsecureGrpcServer, err := grpcserver.NewGrpcServerBuilder(builder.Logger, builder.rpcConf.UnsecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, - builder.apiBurstlimits) + builder.apiBurstlimits).Build() + if err != nil { + return err + } builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { engineBuilder, err := rpc.NewBuilder( @@ -921,23 +929,14 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { // build secure grpc server builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - secureGrpcServer, err := secureGrpcServer.Build() - if err != nil { - return nil, err - } - return secureGrpcServer, nil }) // build unsecure grpc server builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - unsecureGrpcServer, err := unsecureGrpcServer.Build() - if err != nil { - return nil, err - } - return unsecureGrpcServer, nil }) + return nil } // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index f8d16172a45..2c6dc95ebcd 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -135,20 +135,22 @@ func (suite *RestAPITestSuite) SetupTest() { // set the transport credentials for the server to use config.TransportCredentials = credentials.NewTLS(tlsConfig) - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil, - grpcserver.WithTransportCredentials(config.TransportCredentials)) + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + assert.NoError(suite.T(), err) - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, - nil) + nil).Build() + assert.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder( suite.log, @@ -169,8 +171,8 @@ func (suite *RestAPITestSuite) SetupTest() { false, false, suite.me, - secureGrpcServer, - unsecureGrpcServer, + suite.secureGrpcServer, + suite.unsecureGrpcServer, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() @@ -178,12 +180,6 @@ func (suite *RestAPITestSuite) SetupTest() { suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) - suite.secureGrpcServer, err = secureGrpcServer.Build() - assert.NoError(suite.T(), err) - - suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() - assert.NoError(suite.T(), err) - suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 5f04c5a3d4a..0013b04618e 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -60,9 +60,9 @@ type Engine struct { log zerolog.Logger restCollector module.RestMetrics - backend *backend.Backend // the gRPC service implementation - unsecureGrpcServer *grpcserver.GrpcServerBuilder // the unsecure gRPC server - secureGrpcServer *grpcserver.GrpcServerBuilder // the secure gRPC server + backend *backend.Backend // the gRPC service implementation + unsecureGrpcServer *grpcserver.GrpcServer // the unsecure gRPC server + secureGrpcServer *grpcserver.GrpcServer // the secure gRPC server httpServer *http.Server restServer *http.Server config Config @@ -91,14 +91,14 @@ func NewBuilder(log zerolog.Logger, retryEnabled bool, rpcMetricsEnabled bool, me module.Local, - secureGrpcServer *grpcserver.GrpcServerBuilder, - unsecureGrpcServer *grpcserver.GrpcServerBuilder, + secureGrpcServer *grpcserver.GrpcServer, + unsecureGrpcServer *grpcserver.GrpcServer, ) (*RPCEngineBuilder, error) { log = log.With().Str("engine", "rpc").Logger() // wrap the unsecured server with an HTTP proxy server to serve HTTP clients - httpServer := newHTTPProxyServer(unsecureGrpcServer.Server()) + httpServer := newHTTPProxyServer(unsecureGrpcServer.Server) var cache *lru.Cache cacheSize := config.ConnectionPoolSize diff --git a/engine/access/rpc/engine_builder.go b/engine/access/rpc/engine_builder.go index e3e09495400..a2630d26df8 100644 --- a/engine/access/rpc/engine_builder.go +++ b/engine/access/rpc/engine_builder.go @@ -68,11 +68,11 @@ func (builder *RPCEngineBuilder) WithNewHandler(handler accessproto.AccessAPISer func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { // Register legacy gRPC handlers for backwards compatibility, to be removed at a later date legacyaccessproto.RegisterAccessAPIServer( - builder.unsecureGrpcServer.Server(), + builder.unsecureGrpcServer.Server, legacyaccess.NewHandler(builder.backend, builder.chain), ) legacyaccessproto.RegisterAccessAPIServer( - builder.secureGrpcServer.Server(), + builder.secureGrpcServer.Server, legacyaccess.NewHandler(builder.backend, builder.chain), ) return builder @@ -83,8 +83,8 @@ func (builder *RPCEngineBuilder) WithLegacy() *RPCEngineBuilder { func (builder *RPCEngineBuilder) WithMetrics() *RPCEngineBuilder { // Not interested in legacy metrics, so initialize here grpc_prometheus.EnableHandlingTimeHistogram() - grpc_prometheus.Register(builder.unsecureGrpcServer.Server()) - grpc_prometheus.Register(builder.secureGrpcServer.Server()) + grpc_prometheus.Register(builder.unsecureGrpcServer.Server) + grpc_prometheus.Register(builder.secureGrpcServer.Server) return builder } @@ -100,7 +100,7 @@ func (builder *RPCEngineBuilder) Build() (*Engine, error) { handler = access.NewHandler(builder.Engine.backend, builder.Engine.chain, builder.finalizedHeaderCache, builder.me, access.WithBlockSignerDecoder(builder.signerIndicesDecoder)) } } - accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer.Server(), handler) - accessproto.RegisterAccessAPIServer(builder.secureGrpcServer.Server(), handler) + accessproto.RegisterAccessAPIServer(builder.unsecureGrpcServer.Server, handler) + accessproto.RegisterAccessAPIServer(builder.secureGrpcServer.Server, handler) return builder.Engine, nil } diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index 3f9558756e5..da92dd2c42a 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -128,35 +128,32 @@ func (suite *RateLimitTestSuite) SetupTest() { "Ping": suite.rateLimit, } - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, apiRateLimt, apiBurstLimt, - grpcserver.WithTransportCredentials(config.TransportCredentials)) + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + require.NoError(suite.T(), err) - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, apiRateLimt, - apiBurstLimt) + apiBurstLimt).Build() + require.NoError(suite.T(), err) block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) rpcEngBuilder, err := NewBuilder(suite.log, suite.state, config, suite.collClient, nil, suite.blocks, suite.headers, suite.collections, suite.transactions, nil, - nil, suite.chainID, suite.metrics, 0, 0, false, false, suite.me, secureGrpcServer, unsecureGrpcServer) + nil, suite.chainID, suite.metrics, 0, 0, false, false, suite.me, suite.secureGrpcServer, suite.unsecureGrpcServer) require.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() require.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) - suite.secureGrpcServer, err = secureGrpcServer.Build() - assert.NoError(suite.T(), err) - - suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() - assert.NoError(suite.T(), err) suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index 9d2dde00cce..d7f3325d45d 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -111,20 +111,22 @@ func (suite *SecureGRPCTestSuite) SetupTest() { // save the public key to use later in tests later suite.publicKey = networkingKey.PublicKey() - secureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil, - grpcserver.WithTransportCredentials(config.TransportCredentials)) + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + assert.NoError(suite.T(), err) - unsecureGrpcServer := grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, - nil) + nil).Build() + assert.NoError(suite.T(), err) block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) @@ -148,18 +150,13 @@ func (suite *SecureGRPCTestSuite) SetupTest() { false, false, suite.me, - secureGrpcServer, - unsecureGrpcServer, + suite.secureGrpcServer, + suite.unsecureGrpcServer, ) assert.NoError(suite.T(), err) suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() assert.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) - suite.secureGrpcServer, err = secureGrpcServer.Build() - assert.NoError(suite.T(), err) - - suite.unsecureGrpcServer, err = unsecureGrpcServer.Build() - assert.NoError(suite.T(), err) suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index fb6ed4419fc..2fec00781e5 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -6,7 +6,6 @@ import ( access "github.com/onflow/flow/protobuf/go/flow/executiondata" "github.com/rs/zerolog" - "google.golang.org/grpc" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/model/flow" @@ -58,7 +57,6 @@ type Engine struct { *component.ComponentManager log zerolog.Logger backend *StateStreamBackend - server *grpc.Server config Config chain flow.Chain handler *Handler @@ -81,7 +79,7 @@ func NewEng( chainID flow.ChainID, initialBlockHeight uint64, highestBlockHeight uint64, - server *grpcserver.GrpcServerBuilder, + server *grpcserver.GrpcServer, ) (*Engine, error) { logger := log.With().Str("engine", "state_stream_rpc").Logger() @@ -107,7 +105,6 @@ func NewEng( e := &Engine{ log: logger, backend: backend, - server: server.Server(), headers: headers, chain: chainID.Chain(), config: config, @@ -119,7 +116,7 @@ func NewEng( e.ComponentManager = component.NewComponentManagerBuilder(). Build() - access.RegisterExecutionDataAPIServer(e.server, e.handler) + access.RegisterExecutionDataAPIServer(server.Server, e.handler) return e, nil } diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index 0b6368189c0..1fa71c3007b 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -16,8 +16,8 @@ import ( // It makes it easy to configure the node to use the same port for both APIs. type GrpcServer struct { component.Component - log zerolog.Logger - grpcServer *grpc.Server + log zerolog.Logger + Server *grpc.Server grpcListenAddr string // the GRPC server address as ip:port @@ -32,7 +32,7 @@ func NewGrpcServer(log zerolog.Logger, ) (*GrpcServer, error) { server := &GrpcServer{ log: log, - grpcServer: grpcServer, + Server: grpcServer, grpcListenAddr: grpcListenAddr, } server.Component = component.NewComponentManagerBuilder(). @@ -62,7 +62,7 @@ func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready co g.log.Debug().Str("grpc_address", g.grpcAddress.String()).Msg("listening on port") ready() - err = g.grpcServer.Serve(l) // blocking call + err = g.Server.Serve(l) // blocking call if err != nil { g.log.Err(err).Msg("fatal error in grpc server") ctx.Throw(err) @@ -81,5 +81,5 @@ func (g *GrpcServer) GRPCAddress() net.Addr { func (g *GrpcServer) shutdownWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { ready() <-ctx.Done() - g.grpcServer.GracefulStop() + g.Server.GracefulStop() } diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index 0079b781e13..3f71fdbe4c6 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -82,10 +82,6 @@ func NewGrpcServerBuilder(log zerolog.Logger, return grpcServerBuilder } -func (b *GrpcServerBuilder) Server() *grpc.Server { - return b.server -} - func (b *GrpcServerBuilder) Build() (*GrpcServer, error) { return NewGrpcServer(b.log, b.gRPCListenAddr, b.server) } From f9a15e89deb2bdbef5dba72cf5dd2b39a640477a Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 30 Jun 2023 19:13:51 +0300 Subject: [PATCH 076/169] Updated logs, added CLI flag and added access metrics initialization --- cmd/observer/node_builder/observer_builder.go | 3 ++- .../rest/apiproxy/rest_proxy_handler.go | 25 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 4070c277749..53671c3098d 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -452,6 +452,7 @@ func (builder *ObserverServiceBuilder) extraFlags() { flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)") flags.UintVar(&builder.rpcConf.MaxMsgSize, "rpc-max-message-size", defaultConfig.rpcConf.MaxMsgSize, "the maximum message size in bytes for messages sent or received over grpc") + flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.BackendConfig.MaxHeightRange, "maximum size for height range requests") flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") @@ -851,7 +852,7 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - accessMetrics := metrics.NewNoopCollector() + accessMetrics := metrics.NewAccessCollector() config := builder.rpcConf backendConfig := config.BackendConfig diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index 18e7cfe3d9e..9fb8cc64ada 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -90,6 +90,7 @@ func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identi collectionResponse, err := upstream.GetCollectionByID(ctx, getCollectionByIDRequest) if err != nil { + r.log("upstream", "GetCollectionByID", err) return nil, err } @@ -98,7 +99,6 @@ func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identi return nil, err } - r.log("upstream", "GetCollectionByID", err) return transactions, nil } @@ -115,8 +115,8 @@ func (r *RestProxyHandler) SendTransaction(ctx context.Context, tx *flow.Transac } _, err = upstream.SendTransaction(ctx, sendTransactionRequest) - r.log("upstream", "SendTransaction", err) + return err } @@ -132,6 +132,7 @@ func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifie } transactionResponse, err := upstream.GetTransaction(ctx, getTransactionRequest) if err != nil { + r.log("upstream", "GetTransaction", err) return nil, err } @@ -140,7 +141,6 @@ func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifie return nil, err } - r.log("upstream", "GetTransaction", err) return &transactionBody, nil } @@ -148,6 +148,7 @@ func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifie func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Identifier, blockID flow.Identifier, collectionID flow.Identifier) (*access.TransactionResult, error) { upstream, err := r.FaultTolerantClient() if err != nil { + return nil, err } @@ -159,10 +160,10 @@ func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Ide transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) if err != nil { + r.log("upstream", "GetTransactionResult", err) return nil, err } - r.log("upstream", "GetTransactionResult", err) return access.MessageToTransactionResult(transactionResultResponse), nil } @@ -180,10 +181,10 @@ func (r *RestProxyHandler) GetAccountAtBlockHeight(ctx context.Context, address accountResponse, err := upstream.GetAccountAtBlockHeight(ctx, getAccountAtBlockHeightRequest) if err != nil { + r.log("upstream", "GetAccountAtBlockHeight", err) return nil, models.NewNotFoundError("not found account at block height", err) } - r.log("upstream", "GetAccountAtBlockHeight", err) return convert.MessageToAccount(accountResponse.Account) } @@ -200,10 +201,10 @@ func (r *RestProxyHandler) ExecuteScriptAtLatestBlock(ctx context.Context, scrip } executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(ctx, executeScriptAtLatestBlockRequest) if err != nil { + r.log("upstream", "ExecuteScriptAtLatestBlock", err) return nil, err } - r.log("upstream", "ExecuteScriptAtLatestBlock", err) return executeScriptAtLatestBlockResponse.Value, nil } @@ -221,10 +222,10 @@ func (r *RestProxyHandler) ExecuteScriptAtBlockHeight(ctx context.Context, block } executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(ctx, executeScriptAtBlockHeightRequest) if err != nil { + r.log("upstream", "ExecuteScriptAtBlockHeight", err) return nil, err } - r.log("upstream", "ExecuteScriptAtBlockHeight", err) return executeScriptAtBlockHeightResponse.Value, nil } @@ -242,10 +243,10 @@ func (r *RestProxyHandler) ExecuteScriptAtBlockID(ctx context.Context, blockID f } executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(ctx, executeScriptAtBlockIDRequest) if err != nil { + r.log("upstream", "ExecuteScriptAtBlockID", err) return nil, err } - r.log("upstream", "ExecuteScriptAtBlockID", err) return executeScriptAtBlockIDResponse.Value, nil } @@ -263,10 +264,10 @@ func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventTyp } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) if err != nil { + r.log("upstream", "GetEventsForHeightRange", err) return nil, err } - r.log("upstream", "GetEventsForHeightRange", err) return convert.MessagesToBlockEvents(eventsResponse.Results), nil } @@ -285,10 +286,10 @@ func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType s } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) if err != nil { + r.log("upstream", "GetEventsForBlockIDs", err) return nil, err } - r.log("upstream", "GetEventsForBlockIDs", err) return convert.MessagesToBlockEvents(eventsResponse.Results), nil } @@ -304,10 +305,10 @@ func (r *RestProxyHandler) GetExecutionResultForBlockID(ctx context.Context, blo } executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(ctx, getExecutionResultForBlockID) if err != nil { + r.log("upstream", "GetExecutionResultForBlockID", err) return nil, err } - r.log("upsteram", "GetExecutionResultForBlockID", err) return convert.MessageToExecutionResult(executionResultForBlockIDResponse.ExecutionResult) } @@ -324,9 +325,9 @@ func (r *RestProxyHandler) GetExecutionResultByID(ctx context.Context, id flow.I executionResultByIDResponse, err := upstream.GetExecutionResultByID(ctx, executionResultByIDRequest) if err != nil { + r.log("upstream", "GetExecutionResultByID", err) return nil, err } - r.log("upstream", "GetExecutionResultByID", err) return convert.MessageToExecutionResult(executionResultByIDResponse.ExecutionResult) } From aee37a9b32425b8d0ba6ca0d31942188a57eae4c Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 30 Jun 2023 19:25:09 +0300 Subject: [PATCH 077/169] Updated README file, updated comment --- engine/access/rest/README.md | 14 ++++++-------- engine/access/rest/routes/blocks.go | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/engine/access/rest/README.md b/engine/access/rest/README.md index 633acf65707..68378fa19b1 100644 --- a/engine/access/rest/README.md +++ b/engine/access/rest/README.md @@ -10,11 +10,9 @@ available on our [docs site](https://docs.onflow.org/http-api/). - `middleware`: The common [middlewares](https://github.com/gorilla/mux#middleware) that all request pass through. - `models`: The generated models using openapi generators and implementation of model builders. - `request`: Implementation of API requests that provide validation for input data and build request models. -- `routes`: The common HTTP handlers for all the requests. -- `api`: The server API interface for REST service. +- `routes`: The common HTTP handlers for all the requests, tests for each request. - `apiproxy`: Implementation of proxy backend handler which includes the local backend and forwards the methods which can't be handled locally to an upstream using gRPC API. -- `tests`: Test for each request. ## Request lifecycle @@ -48,12 +46,12 @@ package that complies with function interfaced defined as: ```go type ApiHandlerFunc func ( r *request.Request, -backend api.RestBackendApi, +backend access.API, generator models.LinkGenerator, ) (interface{}, error) ``` -That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Then needs -to be added new request to `RestBackendApi` interface and overrides the method if it should be proxied to the backend -handler `RestProxyHandler` for request forwarding. Adding a new API endpoint also requires for a new request builder to -be implemented and added in request package. Make sure to not forget about adding tests for each of the API handler. +That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Also needs +to override the method if it should be proxied to the backend handler `RestProxyHandler` for request forwarding. Adding +a new API endpoint also requires for a new request builder to be implemented and added in request package. Make sure to +not forget about adding tests for each of the API handler. diff --git a/engine/access/rest/routes/blocks.go b/engine/access/rest/routes/blocks.go index cd8547bca93..c26f14dd8bf 100644 --- a/engine/access/rest/routes/blocks.go +++ b/engine/access/rest/routes/blocks.go @@ -146,7 +146,7 @@ func getBlock(option blockProviderOption, req *request.Request, backend access.A return &block, nil } -// blockProvider is a layer of abstraction on top of the backend api.RestBackendApi and provides a uniform way to +// blockProvider is a layer of abstraction on top of the backend access.API and provides a uniform way to // look up a block or a block header either by ID or by height type blockProvider struct { id *flow.Identifier From 4c2f32ce8df4d8915a5b3648f87c94582d25d8e9 Mon Sep 17 00:00:00 2001 From: Josh Hannan Date: Mon, 3 Jul 2023 11:58:26 -0500 Subject: [PATCH 078/169] update bootstrapping to include nft and metadata views --- fvm/blueprints/token.go | 38 ++++++++++++++++++++++++++++++++- fvm/bootstrap.go | 47 +++++++++++++++++++++++++++++++++++++++-- go.mod | 1 + go.sum | 2 ++ 4 files changed, 85 insertions(+), 3 deletions(-) diff --git a/fvm/blueprints/token.go b/fvm/blueprints/token.go index 92cc09e22c3..8971345751a 100644 --- a/fvm/blueprints/token.go +++ b/fvm/blueprints/token.go @@ -22,6 +22,42 @@ func DeployFungibleTokenContractTransaction(fungibleToken flow.Address) *flow.Tr contractName) } +func DeployNonFungibleTokenContractTransaction(nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.NonFungibleToken() + contractName := "NonFungibleToken" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployMetadataViewsContractTransaction(fungibleToken, nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.MetadataViews(fungibleToken.HexWithPrefix(), nonFungibleToken.HexWithPrefix()) + contractName := "MetadataViews" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployViewResolverContractTransaction(nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.ViewResolver() + contractName := "ViewResolver" + return DeployContractTransaction( + nonFungibleToken, + contract, + contractName) +} + +func DeployFungibleTokenMetadataViewsContractTransaction(fungibleToken, nonFungibleToken flow.Address) *flow.TransactionBody { + contract := contracts.FungibleTokenMetadataViews(fungibleToken.HexWithPrefix(), nonFungibleToken.HexWithPrefix()) + contractName := "FungibleTokenMetadataViews" + return DeployContractTransaction( + fungibleToken, + contract, + contractName) +} + //go:embed scripts/deployFlowTokenTransactionTemplate.cdc var deployFlowTokenTransactionTemplate string @@ -31,7 +67,7 @@ var createFlowTokenMinterTransactionTemplate string //go:embed scripts/mintFlowTokenTransactionTemplate.cdc var mintFlowTokenTransactionTemplate string -func DeployFlowTokenContractTransaction(service, fungibleToken, flowToken flow.Address) *flow.TransactionBody { +func DeployFlowTokenContractTransaction(service, fungibleToken, metadataViews, flowToken flow.Address) *flow.TransactionBody { contract := contracts.FlowToken(fungibleToken.HexWithPrefix()) return flow.NewTransactionBody(). diff --git a/fvm/bootstrap.go b/fvm/bootstrap.go index 72d75919927..a6f1b8a9e99 100644 --- a/fvm/bootstrap.go +++ b/fvm/bootstrap.go @@ -318,7 +318,9 @@ func (b *bootstrapExecutor) Execute() error { service := b.createServiceAccount() fungibleToken := b.deployFungibleToken() - flowToken := b.deployFlowToken(service, fungibleToken) + nonFungibleToken := b.deployNonFungibleToken() + b.deployMetadataViews(fungibleToken, nonFungibleToken) + flowToken := b.deployFlowToken(service, fungibleToken, nonFungibleToken) storageFees := b.deployStorageFees(service, fungibleToken, flowToken) feeContract := b.deployFlowFees(service, fungibleToken, flowToken, storageFees) @@ -411,7 +413,47 @@ func (b *bootstrapExecutor) deployFungibleToken() flow.Address { return fungibleToken } -func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken flow.Address) flow.Address { +func (b *bootstrapExecutor) deployNonFungibleToken() flow.Address { + nonFungibleToken := b.createAccount(b.accountKeys.FungibleTokenAccountPublicKeys) + + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployNonFungibleTokenContractTransaction(nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy non-fungible token contract: %s", txError, err) + return nonFungibleToken +} + +func (b *bootstrapExecutor) deployMetadataViews(fungibleToken, nonFungibleToken flow.Address) { + + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployMetadataViewsContractTransaction(fungibleToken, nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy metadata views contract: %s", txError, err) + + txError, err = b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployViewResolverContractTransaction(nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy view resolver contract: %s", txError, err) + + txError, err = b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployFungibleTokenMetadataViewsContractTransaction(fungibleToken, nonFungibleToken), + 0), + ) + panicOnMetaInvokeErrf("failed to deploy fungible token metadata views contract: %s", txError, err) +} + +func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken, metadataViews flow.Address) flow.Address { flowToken := b.createAccount(b.accountKeys.FlowTokenAccountPublicKeys) txError, err := b.invokeMetaTransaction( b.ctx, @@ -419,6 +461,7 @@ func (b *bootstrapExecutor) deployFlowToken(service, fungibleToken flow.Address) blueprints.DeployFlowTokenContractTransaction( service, fungibleToken, + metadataViews, flowToken), 0), ) diff --git a/go.mod b/go.mod index ef9c29a3a43..c04e92ca4e0 100644 --- a/go.mod +++ b/go.mod @@ -227,6 +227,7 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onsi/ginkgo/v2 v2.9.7 // indirect diff --git a/go.sum b/go.sum index ecf5dfae9f4..c13777464c7 100644 --- a/go.sum +++ b/go.sum @@ -1222,6 +1222,8 @@ github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= +github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579 h1:zhk2h88qhize5ktDGZmTRAFtwDuB8TaKymPyi58YaYQ= +github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579/go.mod h1:3xI/fQSig86ORLQ6DPTCmC/1y3y2f/k/tkE+lBFvB18= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= From 1583edfc28005d78b77ca6e2c6b11745d987df2a Mon Sep 17 00:00:00 2001 From: Josh Hannan Date: Mon, 3 Jul 2023 13:15:22 -0500 Subject: [PATCH 079/169] update dependencies --- fvm/blueprints/token.go | 4 ++-- go.mod | 4 ++-- go.sum | 53 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/fvm/blueprints/token.go b/fvm/blueprints/token.go index 8971345751a..4058feb6519 100644 --- a/fvm/blueprints/token.go +++ b/fvm/blueprints/token.go @@ -50,7 +50,7 @@ func DeployViewResolverContractTransaction(nonFungibleToken flow.Address) *flow. } func DeployFungibleTokenMetadataViewsContractTransaction(fungibleToken, nonFungibleToken flow.Address) *flow.TransactionBody { - contract := contracts.FungibleTokenMetadataViews(fungibleToken.HexWithPrefix(), nonFungibleToken.HexWithPrefix()) + contract := contracts.FungibleTokenMetadataViews(fungibleToken.Hex(), nonFungibleToken.Hex()) contractName := "FungibleTokenMetadataViews" return DeployContractTransaction( fungibleToken, @@ -68,7 +68,7 @@ var createFlowTokenMinterTransactionTemplate string var mintFlowTokenTransactionTemplate string func DeployFlowTokenContractTransaction(service, fungibleToken, metadataViews, flowToken flow.Address) *flow.TransactionBody { - contract := contracts.FlowToken(fungibleToken.HexWithPrefix()) + contract := contracts.FlowToken(fungibleToken.HexWithPrefix(), metadataViews.HexWithPrefix(), metadataViews.HexWithPrefix()) return flow.NewTransactionBody(). SetScript([]byte(deployFlowTokenTransactionTemplate)). diff --git a/go.mod b/go.mod index c04e92ca4e0..fb8e63f228d 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/onflow/atree v0.6.0 github.com/onflow/cadence v0.39.12 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.41.6 github.com/onflow/flow-go/crypto v0.24.7 @@ -227,8 +227,8 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect diff --git a/go.sum b/go.sum index c13777464c7..922deb0c8cb 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,7 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/profiler v0.3.0/go.mod h1:9wYk9eY4iZHsev8TQb61kh3wiOiSyz/xOYixWPzweCU= @@ -200,6 +201,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytecodealliance/wasmtime-go v0.22.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= +github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -319,6 +322,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.13 h1:rOPqjSngvs1VSYH2H+PMPiWt4VEulvNRbFgqiGqJM3E= github.com/ethereum/go-ethereum v1.9.13/go.mod h1:qwN9d1GLyDh0N7Ab8bMGd0H9knaji2jOBm2RrMGjXls= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -340,8 +344,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.2.1-0.20210927235116-3d6d5d1de29b/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c h1:5tm/Wbs9d9r+qZaUFXk59CWDD0+77PBqDREffYkyi5c= github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/circlehash v0.1.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= @@ -783,6 +789,7 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -1032,6 +1039,7 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= @@ -1053,6 +1061,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1064,6 +1074,7 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -1074,8 +1085,12 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1216,24 +1231,33 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x071HgCF/0v5hQcaE5qqjc2UqN5gCU8h5Mk6uqpOg= github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= +github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579 h1:zhk2h88qhize5ktDGZmTRAFtwDuB8TaKymPyi58YaYQ= -github.com/onflow/flow-core-contracts v0.0.0-20230703164930-ef65cf873579/go.mod h1:3xI/fQSig86ORLQ6DPTCmC/1y3y2f/k/tkE+lBFvB18= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703164930-ef65cf873579 h1:Hr/r/fIcfu3llr0FJMchLfc7M+YeKRC1BjT62/GxrfQ= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703164930-ef65cf873579/go.mod h1:AvMe24vvaDYbEO9T4p0Xxrtx7fyZ5ri+61FTdFNPpHA= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8 h1:sU/VR+UxyAjEej4Zo+JgepiQb7MPXIynCeJGWH4DTRo= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8/go.mod h1:azBRTidOq5lPPaZ1XGCIs+zOyADNyELuEgcnJpSMVR8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= +github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= +github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= @@ -1294,6 +1318,7 @@ github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= @@ -1363,6 +1388,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1382,6 +1408,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1484,6 +1511,7 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= @@ -1545,8 +1573,10 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.0/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1594,6 +1624,7 @@ go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1633,6 +1664,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1645,6 +1677,7 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= @@ -1818,6 +1851,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1826,12 +1860,14 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1853,7 +1889,10 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1883,6 +1922,8 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1904,6 +1945,7 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= @@ -1978,6 +2020,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -2035,6 +2078,7 @@ google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6 google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= @@ -2120,7 +2164,10 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -2217,8 +2264,10 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= From 9af82d3c9339b391fdf7c39862e8b288dc3299f3 Mon Sep 17 00:00:00 2001 From: Josh Hannan Date: Mon, 3 Jul 2023 15:05:24 -0500 Subject: [PATCH 080/169] running tests --- go.mod | 2 +- go.sum | 8 +- insecure/go.mod | 3 +- insecure/go.sum | 129 +++++++++++++++++++++++++- integration/go.mod | 4 +- integration/go.sum | 8 +- network/mocknetwork/connector.go | 11 +-- network/mocknetwork/connector_host.go | 11 +-- 8 files changed, 148 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index fb8e63f228d..286e1bb0d67 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/onflow/atree v0.6.0 github.com/onflow/cadence v0.39.12 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-go-sdk v0.41.6 github.com/onflow/flow-go/crypto v0.24.7 diff --git a/go.sum b/go.sum index 922deb0c8cb..38cf72d2690 100644 --- a/go.sum +++ b/go.sum @@ -1239,12 +1239,8 @@ github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703164930-ef65cf873579 h1:Hr/r/fIcfu3llr0FJMchLfc7M+YeKRC1BjT62/GxrfQ= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703164930-ef65cf873579/go.mod h1:AvMe24vvaDYbEO9T4p0Xxrtx7fyZ5ri+61FTdFNPpHA= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8 h1:sU/VR+UxyAjEej4Zo+JgepiQb7MPXIynCeJGWH4DTRo= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703180611-254d0a8d48b8/go.mod h1:azBRTidOq5lPPaZ1XGCIs+zOyADNyELuEgcnJpSMVR8= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= diff --git a/insecure/go.mod b/insecure/go.mod index fba888f2997..beb7100c1ce 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -182,10 +182,11 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect github.com/onflow/cadence v0.39.12 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-go-sdk v0.41.6 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 82276186e6f..6e54c805356 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -19,6 +19,16 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -35,6 +45,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -85,6 +96,7 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -179,6 +191,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytecodealliance/wasmtime-go v0.22.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= +github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -292,9 +306,11 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.9/go.mod h1:a9TqabFudpDu1nucId+k9S8R9whYaHnGBLKFouA5EAo= github.com/ethereum/go-ethereum v1.9.13 h1:rOPqjSngvs1VSYH2H+PMPiWt4VEulvNRbFgqiGqJM3E= github.com/ethereum/go-ethereum v1.9.13/go.mod h1:qwN9d1GLyDh0N7Ab8bMGd0H9knaji2jOBm2RrMGjXls= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -304,6 +320,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -314,8 +331,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.2.1-0.20210927235116-3d6d5d1de29b/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c h1:5tm/Wbs9d9r+qZaUFXk59CWDD0+77PBqDREffYkyi5c= github.com/fxamacker/cbor/v2 v2.4.1-0.20230228173756-c0c9f774e40c/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/circlehash v0.1.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/fxamacker/circlehash v0.3.0 h1:XKdvTtIJV9t7DDUtsf0RIpC1OcxZtPbmgIH7ekx28WA= github.com/fxamacker/circlehash v0.3.0/go.mod h1:3aq3OfVvsWtkWMb6A1owjOQFA+TLsD5FgJflnaQwtMM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= @@ -397,6 +416,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= @@ -417,6 +437,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -472,6 +493,7 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -498,6 +520,8 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -736,9 +760,11 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/kevinburke/go-bindata v3.22.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8= github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -988,6 +1014,7 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= @@ -1009,6 +1036,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -1020,6 +1049,7 @@ github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXT github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -1030,8 +1060,12 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1170,20 +1204,27 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x071HgCF/0v5hQcaE5qqjc2UqN5gCU8h5Mk6uqpOg= github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= +github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= +github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= +github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= @@ -1242,6 +1283,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= @@ -1311,6 +1353,7 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1330,6 +1373,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -1432,6 +1476,7 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.4/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA= @@ -1495,8 +1540,10 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.0/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1544,6 +1591,7 @@ go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1583,6 +1631,7 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1595,11 +1644,13 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -1626,6 +1677,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1708,6 +1760,12 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= @@ -1752,6 +1810,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1760,12 +1819,14 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1787,28 +1848,41 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1841,12 +1915,14 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1893,6 +1969,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1902,6 +1979,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= @@ -1936,6 +2016,17 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1991,7 +2082,31 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= @@ -2022,6 +2137,12 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= @@ -2059,8 +2180,10 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200316214253-d7b0ff38cac9/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -2097,7 +2220,9 @@ nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= +pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/integration/go.mod b/integration/go.mod index f59d02427b4..70fc90d757c 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -18,7 +18,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 github.com/onflow/cadence v0.39.12 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-emulator v0.50.6 github.com/onflow/flow-go v0.31.1-0.20230607185125-e75265a6c631 @@ -228,7 +228,7 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect - github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4 // indirect + github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead // indirect diff --git a/integration/go.sum b/integration/go.sum index 0b3079bed0f..34417b41507 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1356,8 +1356,8 @@ github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVF github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d h1:B7PdhdUNkve5MVrekWDuQf84XsGBxNZ/D3x+QQ8XeVs= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-emulator v0.50.6 h1:grt62kVQC5Jsma7bY8k65ts7BZ6+E0XBXroRA75RAPA= @@ -1370,8 +1370,8 @@ github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWk github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= -github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4 h1:5AnM9jIwkyHaY6+C3cWnt07oTOYctmwxvpiL25HRJws= -github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= +github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index deedbd4f815..17795f2da60 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.30.16. DO NOT EDIT. package mocknetwork @@ -20,13 +20,12 @@ func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) _m.Called(ctx, peerChan) } -type mockConstructorTestingTNewConnector interface { +// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConnector(t interface { mock.TestingT Cleanup(func()) -} - -// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnector(t mockConstructorTestingTNewConnector) *Connector { +}) *Connector { mock := &Connector{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_host.go b/network/mocknetwork/connector_host.go index e656391a11f..ec17d9d4221 100644 --- a/network/mocknetwork/connector_host.go +++ b/network/mocknetwork/connector_host.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.21.4. DO NOT EDIT. +// Code generated by mockery v2.30.16. DO NOT EDIT. package mocknetwork @@ -100,13 +100,12 @@ func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { return r0 } -type mockConstructorTestingTNewConnectorHost interface { +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewConnectorHost(t interface { mock.TestingT Cleanup(func()) -} - -// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { +}) *ConnectorHost { mock := &ConnectorHost{} mock.Mock.Test(t) From 70edc6cccc8181c2bdbe9269bbdc6ecd36aa8b19 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 4 Jul 2023 17:40:18 +0200 Subject: [PATCH 081/169] fix build and deploy NFT on service account --- .../cmd/execution-state-extract/export_report.json | 4 ++-- engine/execution/state/bootstrap/bootstrap_test.go | 2 +- fvm/bootstrap.go | 9 ++++----- network/mocknetwork/connector.go | 11 ++++++----- network/mocknetwork/connector_host.go | 11 ++++++----- utils/unittest/execution_state.go | 6 +++--- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json index 2cbadb698d0..3b403a86e10 100644 --- a/cmd/util/cmd/execution-state-extract/export_report.json +++ b/cmd/util/cmd/execution-state-extract/export_report.json @@ -1,6 +1,6 @@ { "EpochCounter": 0, - "PreviousStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf", - "CurrentStateCommitment": "18eb0e8beef7ce851e552ecd29c813fde0a9e6f0c5614d7615642076602a48cf", + "PreviousStateCommitment": "837a870961643c2cc0f7baa5c7b28d1514b4bc91c0817b01cffe5b9b9735d2df", + "CurrentStateCommitment": "837a870961643c2cc0f7baa5c7b28d1514b4bc91c0817b01cffe5b9b9735d2df", "ReportSucceeded": true } \ No newline at end of file diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index db44a03230f..1d84b2938db 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("60a1998dc3c2656758f76520d040e1612b14d80bae263dd0d1118aa7c7d6e4ee") + expectedStateCommitmentBytes, _ := hex.DecodeString("986c540657fdb3b4154311069d901223a3268492f678ae706010cd537cc328ad") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/fvm/bootstrap.go b/fvm/bootstrap.go index a6f1b8a9e99..dc938792d0a 100644 --- a/fvm/bootstrap.go +++ b/fvm/bootstrap.go @@ -318,7 +318,7 @@ func (b *bootstrapExecutor) Execute() error { service := b.createServiceAccount() fungibleToken := b.deployFungibleToken() - nonFungibleToken := b.deployNonFungibleToken() + nonFungibleToken := b.deployNonFungibleToken(service) b.deployMetadataViews(fungibleToken, nonFungibleToken) flowToken := b.deployFlowToken(service, fungibleToken, nonFungibleToken) storageFees := b.deployStorageFees(service, fungibleToken, flowToken) @@ -413,17 +413,16 @@ func (b *bootstrapExecutor) deployFungibleToken() flow.Address { return fungibleToken } -func (b *bootstrapExecutor) deployNonFungibleToken() flow.Address { - nonFungibleToken := b.createAccount(b.accountKeys.FungibleTokenAccountPublicKeys) +func (b *bootstrapExecutor) deployNonFungibleToken(deployTo flow.Address) flow.Address { txError, err := b.invokeMetaTransaction( b.ctx, Transaction( - blueprints.DeployNonFungibleTokenContractTransaction(nonFungibleToken), + blueprints.DeployNonFungibleTokenContractTransaction(deployTo), 0), ) panicOnMetaInvokeErrf("failed to deploy non-fungible token contract: %s", txError, err) - return nonFungibleToken + return deployTo } func (b *bootstrapExecutor) deployMetadataViews(fungibleToken, nonFungibleToken flow.Address) { diff --git a/network/mocknetwork/connector.go b/network/mocknetwork/connector.go index 17795f2da60..deedbd4f815 100644 --- a/network/mocknetwork/connector.go +++ b/network/mocknetwork/connector.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.30.16. DO NOT EDIT. +// Code generated by mockery v2.21.4. DO NOT EDIT. package mocknetwork @@ -20,12 +20,13 @@ func (_m *Connector) Connect(ctx context.Context, peerChan <-chan peer.AddrInfo) _m.Called(ctx, peerChan) } -// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewConnector(t interface { +type mockConstructorTestingTNewConnector interface { mock.TestingT Cleanup(func()) -}) *Connector { +} + +// NewConnector creates a new instance of Connector. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnector(t mockConstructorTestingTNewConnector) *Connector { mock := &Connector{} mock.Mock.Test(t) diff --git a/network/mocknetwork/connector_host.go b/network/mocknetwork/connector_host.go index ec17d9d4221..e656391a11f 100644 --- a/network/mocknetwork/connector_host.go +++ b/network/mocknetwork/connector_host.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.30.16. DO NOT EDIT. +// Code generated by mockery v2.21.4. DO NOT EDIT. package mocknetwork @@ -100,12 +100,13 @@ func (_m *ConnectorHost) PeerInfo(peerId peer.ID) peer.AddrInfo { return r0 } -// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewConnectorHost(t interface { +type mockConstructorTestingTNewConnectorHost interface { mock.TestingT Cleanup(func()) -}) *ConnectorHost { +} + +// NewConnectorHost creates a new instance of ConnectorHost. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewConnectorHost(t mockConstructorTestingTNewConnectorHost) *ConnectorHost { mock := &ConnectorHost{} mock.Mock.Test(t) diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 6b92b183bc0..9e843576195 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -24,7 +24,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "30e59679cca50d898cd2d6e6392fff0f11d0088d308a6aaa07682ce3665145ff" +const GenesisStateCommitmentHex = "517138d362602fb11b17524a654b00d8eecdfbf56406b1636a2c58dad7c5d144" var GenesisStateCommitment flow.StateCommitment @@ -88,10 +88,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "b5e7064526738b1909a082d0bb3eafd6ae4a853c56cd218690c50afa1b2179b6" + return "dd8c079b196fced93e4c541a8f6c49a0ee5fda01b2653c5a03cc165ab1015423" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "9f58134fe65fba4529e908ec21eba590f8f0e81b34ca810d4b1babc49ffe5a53" + return "c6e7f204c774f4208e67451acfdf9783932df06e5ab29b7afc56d548a1573769" } From 3418422eba42e574f9f79ebb67fadb46d1937c5f Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Tue, 4 Jul 2023 20:43:15 +0200 Subject: [PATCH 082/169] upgrade emulator --- integration/go.mod | 16 ++++++++-------- integration/go.sum | 34 +++++++++++++++++----------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/integration/go.mod b/integration/go.mod index 70fc90d757c..2bbe5de000a 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -3,7 +3,7 @@ module github.com/onflow/flow-go/integration go 1.19 require ( - cloud.google.com/go/bigquery v1.48.0 + cloud.google.com/go/bigquery v1.50.0 github.com/VividCortex/ewma v1.2.0 github.com/coreos/go-semver v0.3.0 github.com/dapperlabs/testingdock v0.4.4 @@ -20,8 +20,8 @@ require ( github.com/onflow/cadence v0.39.12 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 - github.com/onflow/flow-emulator v0.50.6 - github.com/onflow/flow-go v0.31.1-0.20230607185125-e75265a6c631 + github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7 + github.com/onflow/flow-go v0.31.1-0.20230704154018-87a84e9d36c2 github.com/onflow/flow-go-sdk v0.41.6 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 @@ -42,17 +42,17 @@ require ( require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect - cloud.google.com/go/storage v1.28.1 // indirect + cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/storage v1.29.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/hcsshim v0.8.7 // indirect github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/apache/arrow/go/v10 v10.0.1 // indirect + github.com/apache/arrow/go/v11 v11.0.0 // indirect github.com/apache/thrift v0.16.0 // indirect github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect @@ -316,7 +316,7 @@ require ( gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/integration/go.sum b/integration/go.sum index 34417b41507..c7601ab308f 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -40,19 +40,19 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.48.0 h1:u+fhS1jJOkPO9vdM84M8HO5VznTfVUicBeoXNKD26ho= -cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.12.0 h1:3uaYULZRLByPdbuUvacGeqneudztEM4xqKQsBcxbDnY= +cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= @@ -66,8 +66,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -136,8 +136,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= -github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= @@ -1360,8 +1360,8 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-5 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= -github.com/onflow/flow-emulator v0.50.6 h1:grt62kVQC5Jsma7bY8k65ts7BZ6+E0XBXroRA75RAPA= -github.com/onflow/flow-emulator v0.50.6/go.mod h1:0avs83tvFDt8vyMcm4AYOcHDSRJHY5eX/XXQW2F4jEg= +github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7 h1:UFcuL4WO1h41vTL6MVBNA6JSeCDrxgHXv2R7617ukeQ= +github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7/go.mod h1:lwMNonHdLvfTF+YIU3yjz9huy/KSjqu+5exZ/TT7Hvc= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= @@ -1995,8 +1995,8 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2367,8 +2367,8 @@ google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= From e5669efc455e16e94b428dc9eecd458d33471fc2 Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Wed, 5 Jul 2023 16:17:36 +0200 Subject: [PATCH 083/169] make tidy --- .../execution-state-extract/export_report.json | 4 ++-- go.sum | 16 ++++++++++++++++ insecure/go.sum | 17 +++++++++++++++++ integration/go.mod | 4 ++-- integration/go.sum | 4 ++-- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/util/cmd/execution-state-extract/export_report.json b/cmd/util/cmd/execution-state-extract/export_report.json index 3b403a86e10..4c8484e4396 100644 --- a/cmd/util/cmd/execution-state-extract/export_report.json +++ b/cmd/util/cmd/execution-state-extract/export_report.json @@ -1,6 +1,6 @@ { "EpochCounter": 0, - "PreviousStateCommitment": "837a870961643c2cc0f7baa5c7b28d1514b4bc91c0817b01cffe5b9b9735d2df", - "CurrentStateCommitment": "837a870961643c2cc0f7baa5c7b28d1514b4bc91c0817b01cffe5b9b9735d2df", + "PreviousStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b", + "CurrentStateCommitment": "1c9f9d343cb8d4610e0b2c1eb74d6ea2f2f8aef2d666281dc22870e3efaa607b", "ReportSucceeded": true } \ No newline at end of file diff --git a/go.sum b/go.sum index 38cf72d2690..55200e0247c 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,7 @@ github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -334,6 +335,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -408,6 +410,7 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -429,6 +432,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= @@ -786,6 +790,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= @@ -1678,7 +1683,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -1691,6 +1699,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1964,12 +1973,14 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -2039,8 +2050,12 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -2302,6 +2317,7 @@ nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0 pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/insecure/go.sum b/insecure/go.sum index 6e54c805356..5356d959ea6 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -395,6 +395,7 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -457,6 +458,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -505,6 +507,11 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -1649,6 +1656,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1663,6 +1672,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1746,6 +1756,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -1895,6 +1906,7 @@ golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= @@ -1991,8 +2003,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -2147,6 +2163,7 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/integration/go.mod b/integration/go.mod index 2bbe5de000a..2a18e99d8c1 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -36,7 +36,7 @@ require ( go.uber.org/atomic v1.11.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/sync v0.2.0 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.1 google.golang.org/protobuf v1.30.0 ) @@ -306,7 +306,7 @@ require ( golang.org/x/crypto v0.10.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/term v0.9.0 // indirect golang.org/x/text v0.10.0 // indirect diff --git a/integration/go.sum b/integration/go.sum index c7601ab308f..da3e939fdd0 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2403,8 +2403,8 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= From 4c635c5e37b4cfbc9bbbee92da5dbb60779ebf01 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 5 Jul 2023 12:34:45 -0400 Subject: [PATCH 084/169] add control message rpc sent tracker for use in gossip sub tracer --- network/p2p/inspector/internal/cache/cache.go | 4 +- .../p2p/p2pbuilder/inspector/suite/suite.go | 2 +- network/p2p/tracer/internal/cache.go | 86 +++++++ network/p2p/tracer/internal/cache_test.go | 225 ++++++++++++++++++ .../p2p/tracer/internal/rpc_send_entity.go | 37 +++ .../p2p/tracer/internal/rpc_sent_tracker.go | 64 +++++ .../tracer/internal/rpc_sent_tracker_test.go | 71 ++++++ 7 files changed, 485 insertions(+), 4 deletions(-) create mode 100644 network/p2p/tracer/internal/cache.go create mode 100644 network/p2p/tracer/internal/cache_test.go create mode 100644 network/p2p/tracer/internal/rpc_send_entity.go create mode 100644 network/p2p/tracer/internal/rpc_sent_tracker.go create mode 100644 network/p2p/tracer/internal/rpc_sent_tracker_test.go diff --git a/network/p2p/inspector/internal/cache/cache.go b/network/p2p/inspector/internal/cache/cache.go index 133fd0a9ac7..82d8f781a98 100644 --- a/network/p2p/inspector/internal/cache/cache.go +++ b/network/p2p/inspector/internal/cache/cache.go @@ -40,9 +40,7 @@ type RecordCache struct { // NewRecordCache creates a new *RecordCache. // Args: -// - sizeLimit: the maximum number of records that the cache can hold. -// - logger: the logger used by the cache. -// - collector: the metrics collector used by the cache. +// - config: record cache config. // - recordEntityFactory: a factory function that creates a new spam record. // Returns: // - *RecordCache, the created cache. diff --git a/network/p2p/p2pbuilder/inspector/suite/suite.go b/network/p2p/p2pbuilder/inspector/suite/suite.go index 6271d627cf4..2aa7532ad41 100644 --- a/network/p2p/p2pbuilder/inspector/suite/suite.go +++ b/network/p2p/p2pbuilder/inspector/suite/suite.go @@ -62,7 +62,7 @@ func (s *GossipSubInspectorSuite) InspectFunc() func(peer.ID, *pubsub.RPC) error return s.aggregatedInspector.Inspect } -// AddInvalidCtrlMsgNotificationConsumer adds a consumer to the invalid control message notification distributor. +// AddInvCtrlMsgNotifConsumer adds a consumer to the invalid control message notification distributor. // This consumer is notified when a misbehaving peer regarding gossipsub control messages is detected. This follows a pub/sub // pattern where the consumer is notified when a new notification is published. // A consumer is only notified once for each notification, and only receives notifications that were published after it was added. diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go new file mode 100644 index 00000000000..597f25c1534 --- /dev/null +++ b/network/p2p/tracer/internal/cache.go @@ -0,0 +1,86 @@ +package internal + +import ( + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" + "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" + "github.com/onflow/flow-go/module/mempool/stdmap" + "github.com/onflow/flow-go/network/p2p" +) + +type entityFactory func(id flow.Identifier, controlMsgType p2p.ControlMessageType) rpcSentEntity + +type RPCSentCacheConfig struct { + sizeLimit uint32 + logger zerolog.Logger + collector module.HeroCacheMetrics +} + +// rpcSentCache cache that stores rpcSentEntity. These entity's represent RPC control messages sent from the local node. +type rpcSentCache struct { + // c is the underlying cache. + c *stdmap.Backend +} + +// newRPCSentCache creates a new *rpcSentCache. +// Args: +// - config: record cache config. +// Returns: +// - *rpcSentCache: the created cache. +// Note that this cache is intended to track control messages sent by the local node, +// it stores a RPCSendEntity using an Id which should uniquely identifies the message being tracked. +func newRPCSentCache(config *RPCSentCacheConfig) (*rpcSentCache, error) { + backData := herocache.NewCache(config.sizeLimit, + herocache.DefaultOversizeFactor, + heropool.LRUEjection, + config.logger.With().Str("mempool", "gossipsub=rpc-control-messages-sent").Logger(), + config.collector) + return &rpcSentCache{ + c: stdmap.NewBackend(stdmap.WithBackData(backData)), + }, nil +} + +// init initializes the record cached for the given messageID if it does not exist. +// Returns true if the record is initialized, false otherwise (i.e.: the record already exists). +// Args: +// - flow.Identifier: the messageID to store the rpc control message. +// - p2p.ControlMessageType: the rpc control message type. +// Returns: +// - bool: true if the record is initialized, false otherwise (i.e.: the record already exists). +// Note that if init is called multiple times for the same messageID, the record is initialized only once, and the +// subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). +func (r *rpcSentCache) init(messageID flow.Identifier, controlMsgType p2p.ControlMessageType) bool { + return r.c.Add(newRPCSentEntity(messageID, controlMsgType)) +} + +// has checks if the RPC message has been cached indicating it has been sent. +// Args: +// - flow.Identifier: the messageID to store the rpc control message. +// Returns: +// - bool: true if the RPC has been cache indicating it was sent from the local node. +func (r *rpcSentCache) has(messageId flow.Identifier) bool { + return r.c.Has(messageId) +} + +// ids returns the list of ids of each rpcSentEntity stored. +func (r *rpcSentCache) ids() []flow.Identifier { + return flow.GetIDs(r.c.All()) +} + +// remove the record of the given messageID from the cache. +// Returns true if the record is removed, false otherwise (i.e., the record does not exist). +// Args: +// - flow.Identifier: the messageID to store the rpc control message. +// Returns: +// - true if the record is removed, false otherwise (i.e., the record does not exist). +func (r *rpcSentCache) remove(messageID flow.Identifier) bool { + return r.c.Remove(messageID) +} + +// size returns the number of records in the cache. +func (r *rpcSentCache) size() uint { + return r.c.Size() +} diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go new file mode 100644 index 00000000000..2d4017c54e9 --- /dev/null +++ b/network/p2p/tracer/internal/cache_test.go @@ -0,0 +1,225 @@ +package internal + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "go.uber.org/atomic" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestCache_Init tests the init method of the rpcSentCache. +// It ensures that the method returns true when a new record is initialized +// and false when an existing record is initialized. +func TestCache_Init(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + id1 := unittest.IdentifierFixture() + id2 := unittest.IdentifierFixture() + + // test initializing a record for an ID that doesn't exist in the cache + initialized := cache.init(id1, controlMsgType) + require.True(t, initialized, "expected record to be initialized") + require.True(t, cache.has(id1), "expected record to exist") + + // test initializing a record for an ID that already exists in the cache + initialized = cache.init(id1, controlMsgType) + require.False(t, initialized, "expected record not to be initialized") + require.True(t, cache.has(id1), "expected record to exist") + + // test initializing a record for another ID + initialized = cache.init(id2, controlMsgType) + require.True(t, initialized, "expected record to be initialized") + require.True(t, cache.has(id2), "expected record to exist") +} + +// TestCache_ConcurrentInit tests the concurrent initialization of records. +// The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different ids. +// 2. Ensuring that all records are correctly initialized. +func TestCache_ConcurrentInit(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + ids := unittest.IdentifierListFixture(10) + + var wg sync.WaitGroup + wg.Add(len(ids)) + + for _, id := range ids { + go func(id flow.Identifier) { + defer wg.Done() + cache.init(id, controlMsgType) + }(id) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that all records are correctly initialized + for _, id := range ids { + require.True(t, cache.has(id)) + } +} + +// TestCache_ConcurrentSameRecordInit tests the concurrent initialization of the same record. +// The test covers the following scenarios: +// 1. Multiple goroutines attempting to initialize the same record concurrently. +// 2. Only one goroutine successfully initializes the record, and others receive false on initialization. +// 3. The record is correctly initialized in the cache and can be retrieved using the Get method. +func TestCache_ConcurrentSameRecordInit(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + id := unittest.IdentifierFixture() + const concurrentAttempts = 10 + + var wg sync.WaitGroup + wg.Add(concurrentAttempts) + + successGauge := atomic.Int32{} + + for i := 0; i < concurrentAttempts; i++ { + go func() { + defer wg.Done() + initSuccess := cache.init(id, controlMsgType) + if initSuccess { + successGauge.Inc() + } + }() + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that only one goroutine successfully initialized the record + require.Equal(t, int32(1), successGauge.Load()) + + // ensure that the record is correctly initialized in the cache + require.True(t, cache.has(id)) +} + +// TestCache_Remove tests the remove method of the RecordCache. +// The test covers the following scenarios: +// 1. Initializing the cache with multiple records. +// 2. Removing a record and checking if it is removed correctly. +// 3. Ensuring the other records are still in the cache after removal. +// 4. Attempting to remove a non-existent ID. +func TestCache_Remove(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + // initialize spam records for a few ids + id1 := unittest.IdentifierFixture() + id2 := unittest.IdentifierFixture() + id3 := unittest.IdentifierFixture() + + require.True(t, cache.init(id1, controlMsgType)) + require.True(t, cache.init(id2, controlMsgType)) + require.True(t, cache.init(id3, controlMsgType)) + + numOfIds := uint(3) + require.Equal(t, numOfIds, cache.size(), fmt.Sprintf("expected size of the cache to be %d", numOfIds)) + // remove id1 and check if the record is removed + require.True(t, cache.remove(id1)) + require.NotContains(t, id1, cache.ids()) + + // check if the other ids are still in the cache + require.True(t, cache.has(id2)) + require.True(t, cache.has(id3)) + + // attempt to remove a non-existent ID + id4 := unittest.IdentifierFixture() + require.False(t, cache.remove(id4)) +} + +// TestCache_ConcurrentRemove tests the concurrent removal of records for different ids. +// The test covers the following scenarios: +// 1. Multiple goroutines removing records for different ids concurrently. +// 2. The records are correctly removed from the cache. +func TestCache_ConcurrentRemove(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + ids := unittest.IdentifierListFixture(10) + for _, id := range ids { + cache.init(id, controlMsgType) + } + + var wg sync.WaitGroup + wg.Add(len(ids)) + + for _, id := range ids { + go func(id flow.Identifier) { + defer wg.Done() + require.True(t, cache.remove(id)) + require.NotContains(t, id, cache.ids()) + }(id) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + require.Equal(t, uint(0), cache.size()) +} + +// TestRecordCache_ConcurrentInitAndRemove tests the concurrent initialization and removal of records for different +// ids. The test covers the following scenarios: +// 1. Multiple goroutines initializing records for different ids concurrently. +// 2. Multiple goroutines removing records for different ids concurrently. +// 3. The initialized records are correctly added to the cache. +// 4. The removed records are correctly removed from the cache. +func TestRecordCache_ConcurrentInitAndRemove(t *testing.T) { + cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) + controlMsgType := p2p.CtrlMsgIHave + ids := unittest.IdentifierListFixture(20) + idsToAdd := ids[:10] + idsToRemove := ids[10:] + + for _, id := range idsToRemove { + cache.init(id, controlMsgType) + } + + var wg sync.WaitGroup + wg.Add(len(ids)) + + // initialize spam records concurrently + for _, id := range idsToAdd { + go func(id flow.Identifier) { + defer wg.Done() + cache.init(id, controlMsgType) + }(id) + } + + // remove spam records concurrently + for _, id := range idsToRemove { + go func(id flow.Identifier) { + defer wg.Done() + require.True(t, cache.remove(id)) + require.NotContains(t, id, cache.ids()) + }(id) + } + + unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") + + // ensure that the initialized records are correctly added to the cache + // and removed records are correctly removed from the cache + require.ElementsMatch(t, idsToAdd, cache.ids()) +} + +// cacheFixture returns a new *RecordCache. +func cacheFixture(t *testing.T, sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *rpcSentCache { + config := &RPCSentCacheConfig{ + sizeLimit: sizeLimit, + logger: logger, + collector: collector, + } + r, err := newRPCSentCache(config) + require.NoError(t, err) + // expect cache to be empty + require.Equalf(t, uint(0), r.size(), "cache size must be 0") + require.NotNil(t, r) + return r +} diff --git a/network/p2p/tracer/internal/rpc_send_entity.go b/network/p2p/tracer/internal/rpc_send_entity.go new file mode 100644 index 00000000000..493e4808440 --- /dev/null +++ b/network/p2p/tracer/internal/rpc_send_entity.go @@ -0,0 +1,37 @@ +package internal + +import ( + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/network/p2p" +) + +// rpcSentEntity struct representing an RPC control message sent from local node. +// This struct implements the flow.Entity interface and uses messageID field deduplication. +type rpcSentEntity struct { + // messageID the messageID of the rpc control message. + messageID flow.Identifier + // controlMsgType the control message type. + controlMsgType p2p.ControlMessageType +} + +var _ flow.Entity = (*rpcSentEntity)(nil) + +// ID returns the node ID of the sender, which is used as the unique identifier of the entity for maintenance and +// deduplication purposes in the cache. +func (r rpcSentEntity) ID() flow.Identifier { + return r.messageID +} + +// Checksum returns the node ID of the sender, it does not have any purpose in the cache. +// It is implemented to satisfy the flow.Entity interface. +func (r rpcSentEntity) Checksum() flow.Identifier { + return r.messageID +} + +// newRPCSentEntity returns a new rpcSentEntity. +func newRPCSentEntity(id flow.Identifier, controlMessageType p2p.ControlMessageType) rpcSentEntity { + return rpcSentEntity{ + messageID: id, + controlMsgType: controlMessageType, + } +} diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go new file mode 100644 index 00000000000..047976cd424 --- /dev/null +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -0,0 +1,64 @@ +package internal + +import ( + "fmt" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/network/p2p" +) + +// RPCSentTracker tracks RPC messages that are sent. +type RPCSentTracker struct { + cache *rpcSentCache +} + +// NewRPCSentTracker returns a new *NewRPCSentTracker. +func NewRPCSentTracker(logger zerolog.Logger, sizeLimit uint32, collector module.HeroCacheMetrics) (*RPCSentTracker, error) { + config := &RPCSentCacheConfig{ + sizeLimit: sizeLimit, + logger: logger, + collector: collector, + } + cache, err := newRPCSentCache(config) + if err != nil { + return nil, fmt.Errorf("failed to create new rpc sent cahe: %w", err) + } + return &RPCSentTracker{cache: cache}, nil +} + +// OnIHaveRPCSent caches a unique entity message ID for each message ID included in each rpc iHave control message. +// Args: +// - *pubsub.RPC: the rpc sent. +func (t *RPCSentTracker) OnIHaveRPCSent(rpc *pubsub.RPC) { + controlMsgType := p2p.CtrlMsgIHave + for _, iHave := range rpc.GetControl().GetIhave() { + topicID := iHave.GetTopicID() + for _, messageID := range iHave.GetMessageIDs() { + entityMsgID := iHaveRPCSentEntityID(topicID, messageID) + t.cache.init(entityMsgID, controlMsgType) + } + } +} + +// WasIHaveRPCSent checks if an iHave control message with the provided message ID was sent. +// Args: +// - string: the topic ID of the iHave RPC. +// - string: the message ID of the iHave RPC. +// Returns: +// - bool: true if the iHave rpc with the provided message ID was sent. +func (t *RPCSentTracker) WasIHaveRPCSent(topicID, messageID string) bool { + entityMsgID := iHaveRPCSentEntityID(topicID, messageID) + return t.cache.has(entityMsgID) +} + +// iHaveRPCSentEntityID appends the topicId and messageId and returns the flow.Identifier hash. +// Each iHave RPC control message contains a single topicId and multiple messageIds, to ensure we +// produce a unique id for each message we append the messageId to the topicId. +func iHaveRPCSentEntityID(topicId, messageId string) flow.Identifier { + b := []byte(fmt.Sprintf("%s%s", topicId, messageId)) + return flow.HashToID(b) +} diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go new file mode 100644 index 00000000000..f0a0dda5195 --- /dev/null +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -0,0 +1,71 @@ +package internal + +import ( + "testing" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestNewRPCSentTracker ensures *RPCSenTracker is created as expected. +func TestNewRPCSentTracker(t *testing.T) { + tracker := mockTracker(t) + require.NotNil(t, tracker) +} + +// TestRPCSentTracker_IHave ensures *RPCSentTracker tracks sent iHave control messages as expected. +func TestRPCSentTracker_IHave(t *testing.T) { + tracker := mockTracker(t) + require.NotNil(t, tracker) + + t.Run("WasIHaveRPCSent should return false for iHave message Id that has not been tracked", func(t *testing.T) { + require.False(t, tracker.WasIHaveRPCSent("topic_id", "message_id")) + }) + + t.Run("WasIHaveRPCSent should return true for iHave message after it is tracked with OnIHaveRPCSent", func(t *testing.T) { + topicID := channels.PushBlocks.String() + messageID := unittest.IdentifierFixture().String() + iHaves := []*pb.ControlIHave{{ + TopicID: &topicID, + MessageIDs: []string{messageID}, + }} + rpc := rpcFixture(withIhaves(iHaves)) + tracker.OnIHaveRPCSent(rpc) + require.True(t, tracker.WasIHaveRPCSent(topicID, messageID)) + }) +} + +func mockTracker(t *testing.T) *RPCSentTracker { + logger := zerolog.Nop() + sizeLimit := uint32(100) + collector := metrics.NewNoopCollector() + tracker, err := NewRPCSentTracker(logger, sizeLimit, collector) + require.NoError(t, err) + return tracker +} + +type rpcFixtureOpt func(*pubsub.RPC) + +func withIhaves(iHave []*pb.ControlIHave) rpcFixtureOpt { + return func(rpc *pubsub.RPC) { + rpc.Control.Ihave = iHave + } +} + +func rpcFixture(opts ...rpcFixtureOpt) *pubsub.RPC { + rpc := &pubsub.RPC{ + RPC: pb.RPC{ + Control: &pb.ControlMessage{}, + }, + } + for _, opt := range opts { + opt(rpc) + } + return rpc +} From 2850cacaf64fbc7c6133d9202565bb14817c323e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 6 Jul 2023 01:33:07 -0400 Subject: [PATCH 085/169] add rpc sent tracker to gossipsub mesh tracer - track iHave messages on each RPC sent --- .../node_builder/access_node_builder.go | 17 +++++--- cmd/observer/node_builder/observer_builder.go | 17 +++++--- config/default-config.yml | 2 + follower/follower_builder.go | 17 +++++--- module/metrics/herocache.go | 9 ++++ module/metrics/labels.go | 1 + network/internal/p2pfixtures/fixtures.go | 16 +++++--- network/netconf/flags.go | 10 +++-- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 14 ++++++- network/p2p/p2pconf/gossipsub.go | 2 + network/p2p/tracer/gossipSubMeshTracer.go | 41 ++++++++++++++----- .../p2p/tracer/gossipSubMeshTracer_test.go | 12 +++++- network/p2p/tracer/internal/cache.go | 6 +-- .../p2p/tracer/internal/rpc_send_entity.go | 6 +-- .../p2p/tracer/internal/rpc_sent_tracker.go | 10 ++--- 15 files changed, 132 insertions(+), 48 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index bf7a52047b4..e4c21e47ca6 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1192,11 +1192,18 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri return nil, fmt.Errorf("could not create connection manager: %w", err) } - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - networkMetrics, - builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: networkMetrics, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) + } libp2pNode, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index be518249714..cbaa51cdc62 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -703,11 +703,18 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr pis = append(pis, pi) } - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - builder.Metrics.Network, - builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: builder.Metrics.Network, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) + } node, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/config/default-config.yml b/config/default-config.yml index 371fc4c385c..7788aa6a91d 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -63,6 +63,8 @@ network-config: # The default interval at which the gossipsub score tracer logs the peer scores. This is used for debugging and forensics purposes. # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. gossipsub-score-tracer-interval: 1m + # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node. + gossipsub-rpc-sent-tracker-cache-size: 10000 # Peer scoring is the default value for enabling peer scoring gossipsub-peer-scoring-enabled: true # Gossipsub rpc inspectors configs diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 36486907d1c..95c7cb60108 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -605,11 +605,18 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr pis = append(pis, pi) } - meshTracer := tracer.NewGossipSubMeshTracer( - builder.Logger, - builder.Metrics.Network, - builder.IdentityProvider, - builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: builder.Logger, + Metrics: builder.Metrics.Network, + IDProvider: builder.IdentityProvider, + LoggerInterval: builder.FlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), + RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) + } node, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/module/metrics/herocache.go b/module/metrics/herocache.go index 54e287bdb1b..f3a88341c87 100644 --- a/module/metrics/herocache.go +++ b/module/metrics/herocache.go @@ -146,6 +146,15 @@ func GossipSubRPCInspectorQueueMetricFactory(f HeroCacheMetricsFactory, networkT return f(namespaceNetwork, r) } +func GossipSubRPCSentTrackerMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { + // we don't use the public prefix for the metrics here for sake of backward compatibility of metric name. + r := ResourceNetworkingRPCSentTrackerCache + if networkType == network.PublicNetwork { + r = PrependPublicPrefix(r) + } + return f(namespaceNetwork, r) +} + func RpcInspectorNotificationQueueMetricFactory(f HeroCacheMetricsFactory, networkType network.NetworkingType) module.HeroCacheMetrics { r := ResourceNetworkingRpcInspectorNotificationQueue if networkType == network.PublicNetwork { diff --git a/module/metrics/labels.go b/module/metrics/labels.go index 353e1b3ca25..d57dd418b56 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -92,6 +92,7 @@ const ( ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" ResourceNetworkingDisallowListCache = "disallow_list_cache" + ResourceNetworkingRPCSentTrackerCache = "rpc_sent_tracker_cache" ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index 40229337dfa..2d2c18983ab 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -102,11 +102,17 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif idProvider := id.NewFixedIdentityProvider(nodeIds) defaultFlowConfig, err := config.DefaultConfig() require.NoError(t, err) - meshTracer := tracer.NewGossipSubMeshTracer( - logger, - metrics.NewNoopCollector(), - idProvider, - defaultFlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval) + + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: metrics.NewNoopCollector(), + IDProvider: idProvider, + LoggerInterval: defaultFlowConfig.NetworkConfig.GossipSubConfig.LocalMeshLogInterval, + RpcSentTrackerCacheCollector: metrics.NewNoopCollector(), + RpcSentTrackerCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + require.NoError(t, err) builder := p2pbuilder.NewNodeBuilder( logger, diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 2b76042e6c8..504d9ee177a 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -36,9 +36,10 @@ const ( gracePeriod = "libp2p-grace-period" silencePeriod = "libp2p-silence-period" // gossipsub - peerScoring = "gossipsub-peer-scoring-enabled" - localMeshLogInterval = "gossipsub-local-mesh-logging-interval" - scoreTracerInterval = "gossipsub-score-tracer-interval" + peerScoring = "gossipsub-peer-scoring-enabled" + localMeshLogInterval = "gossipsub-local-mesh-logging-interval" + rpcSentTrackerCacheSize = "gossipsub-rpc-sent-tracker-cache-size" + scoreTracerInterval = "gossipsub-score-tracer-interval" // gossipsub validation inspector gossipSubRPCInspectorNotificationCacheSize = "gossipsub-rpc-inspector-notification-cache-size" validationInspectorNumberOfWorkers = "gossipsub-rpc-validation-inspector-workers" @@ -65,7 +66,7 @@ func AllFlagNames() []string { return []string{ networkingConnectionPruning, preferredUnicastsProtocols, receivedMessageCacheSize, peerUpdateInterval, unicastMessageTimeout, unicastCreateStreamRetryDelay, dnsCacheTTL, disallowListNotificationCacheSize, dryRun, lockoutDuration, messageRateLimit, bandwidthRateLimit, bandwidthBurstLimit, memoryLimitRatio, - fileDescriptorsRatio, peerBaseLimitConnsInbound, highWatermark, lowWatermark, gracePeriod, silencePeriod, peerScoring, localMeshLogInterval, scoreTracerInterval, + fileDescriptorsRatio, peerBaseLimitConnsInbound, highWatermark, lowWatermark, gracePeriod, silencePeriod, peerScoring, localMeshLogInterval, rpcSentTrackerCacheSize, scoreTracerInterval, gossipSubRPCInspectorNotificationCacheSize, validationInspectorNumberOfWorkers, validationInspectorInspectMessageQueueCacheSize, validationInspectorClusterPrefixedTopicsReceivedCacheSize, validationInspectorClusterPrefixedTopicsReceivedCacheDecay, validationInspectorClusterPrefixHardThreshold, ihaveSyncSampleSizePercentage, ihaveAsyncSampleSizePercentage, ihaveMaxSampleSize, metricsInspectorNumberOfWorkers, metricsInspectorCacheSize, alspDisabled, alspSpamRecordCacheSize, alspSpamRecordQueueSize, alspHearBeatInterval, @@ -106,6 +107,7 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { flags.Bool(peerScoring, config.GossipSubConfig.PeerScoring, "enabling peer scoring on pubsub network") flags.Duration(localMeshLogInterval, config.GossipSubConfig.LocalMeshLogInterval, "logging interval for local mesh in gossipsub") flags.Duration(scoreTracerInterval, config.GossipSubConfig.ScoreTracerInterval, "logging interval for peer score tracer in gossipsub, set to 0 to disable") + flags.Uint32(rpcSentTrackerCacheSize, config.GossipSubConfig.RPCSentTrackerCacheSize, "cache size of the rpc sent tracker used by the gossipsub mesh tracer.") // gossipsub RPC control message validation limits used for validation configuration and rate limiting flags.Int(validationInspectorNumberOfWorkers, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.NumberOfWorkers, "number of gossupsub RPC control message validation inspector component workers") flags.Uint32(validationInspectorInspectMessageQueueCacheSize, config.GossipSubConfig.GossipSubRPCInspectorsConfig.GossipSubRPCValidationInspectorConfigs.CacheSize, "cache size for gossipsub RPC validation inspector events worker pool queue.") diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index c0a6412297c..da8768402ed 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -26,6 +26,7 @@ import ( "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" flownet "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/netconf" "github.com/onflow/flow-go/network/p2p" @@ -494,7 +495,18 @@ func DefaultNodeBuilder( builder.EnableGossipSubPeerScoring(nil) } - meshTracer := tracer.NewGossipSubMeshTracer(logger, metricsCfg.Metrics, idProvider, gossipCfg.LocalMeshLogInterval) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: metricsCfg.Metrics, + IDProvider: idProvider, + LoggerInterval: gossipCfg.LocalMeshLogInterval, + RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(metricsCfg.HeroCacheFactory, flownet.PrivateNetwork), + RpcSentTrackerCacheSize: gossipCfg.RPCSentTrackerCacheSize, + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + if err != nil { + return nil, fmt.Errorf("could not create gossipsub mesh tracer: %w", err) + } builder.SetGossipSubTracer(meshTracer) builder.SetGossipSubScoreTracerInterval(gossipCfg.ScoreTracerInterval) diff --git a/network/p2p/p2pconf/gossipsub.go b/network/p2p/p2pconf/gossipsub.go index f9155129efd..d297f5cba8b 100644 --- a/network/p2p/p2pconf/gossipsub.go +++ b/network/p2p/p2pconf/gossipsub.go @@ -21,6 +21,8 @@ type GossipSubConfig struct { LocalMeshLogInterval time.Duration `mapstructure:"gossipsub-local-mesh-logging-interval"` // ScoreTracerInterval is the interval at which the score tracer logs the peer scores. ScoreTracerInterval time.Duration `mapstructure:"gossipsub-score-tracer-interval"` + // RPCSentTrackerCacheSize cache size of the rpc sent tracker used by the gossipsub mesh tracer. + RPCSentTrackerCacheSize uint32 `mapstructure:"gossipsub-rpc-sent-tracker-cache-size"` // PeerScoring is whether to enable GossipSub peer scoring. PeerScoring bool `mapstructure:"gossipsub-peer-scoring-enabled"` } diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 7cd4dd2b692..48054427b1d 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/module/component" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/tracer/internal" "github.com/onflow/flow-go/utils/logging" ) @@ -43,23 +44,34 @@ type GossipSubMeshTracer struct { idProvider module.IdentityProvider loggerInterval time.Duration metrics module.GossipSubLocalMeshMetrics + rpcSentTracker *internal.RPCSentTracker } var _ p2p.PubSubTracer = (*GossipSubMeshTracer)(nil) -func NewGossipSubMeshTracer( - logger zerolog.Logger, - metrics module.GossipSubLocalMeshMetrics, - idProvider module.IdentityProvider, - loggerInterval time.Duration) *GossipSubMeshTracer { +type GossipSubMeshTracerConfig struct { + Logger zerolog.Logger + Metrics module.GossipSubLocalMeshMetrics + IDProvider module.IdentityProvider + LoggerInterval time.Duration + RpcSentTrackerCacheCollector module.HeroCacheMetrics + RpcSentTrackerCacheSize uint32 +} + +func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTracer, error) { + rpcSentTracker, err := internal.NewRPCSentTracker(config.Logger, config.RpcSentTrackerCacheSize, config.RpcSentTrackerCacheCollector) + if err != nil { + return nil, err + } g := &GossipSubMeshTracer{ RawTracer: NewGossipSubNoopTracer(), topicMeshMap: make(map[string]map[peer.ID]struct{}), - idProvider: idProvider, - metrics: metrics, - logger: logger.With().Str("component", "gossip_sub_topology_tracer").Logger(), - loggerInterval: loggerInterval, + idProvider: config.IDProvider, + metrics: config.Metrics, + logger: config.Logger.With().Str("component", "gossip_sub_topology_tracer").Logger(), + loggerInterval: config.LoggerInterval, + rpcSentTracker: rpcSentTracker, } g.Component = component.NewComponentManagerBuilder(). @@ -69,7 +81,7 @@ func NewGossipSubMeshTracer( }). Build() - return g + return g, nil } // GetMeshPeers returns the local mesh peers for the given topic. @@ -139,6 +151,15 @@ func (t *GossipSubMeshTracer) Prune(p peer.ID, topic string) { lg.Info().Hex("flow_id", logging.ID(id.NodeID)).Str("role", id.Role.String()).Msg("pruned peer") } +// SendRPC is called when a RPC is sent. Currently, the GossipSubMeshTracer tracks iHave RPC messages that have been sent. +// This function can be updated to track other control messages in the future as required. +func (t *GossipSubMeshTracer) SendRPC(rpc *pubsub.RPC, _ peer.ID) { + switch { + case len(rpc.GetControl().GetIhave()) > 0: + t.rpcSentTracker.OnIHaveRPCSent(rpc.GetControl().GetIhave()) + } +} + // logLoop logs the mesh peers of the local node for each topic at a regular interval. func (t *GossipSubMeshTracer) logLoop(ctx irrecoverable.SignalerContext) { ticker := time.NewTicker(t.loggerInterval) diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index fc14b280282..4b469beb2e7 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/metrics" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" @@ -61,7 +62,16 @@ func TestGossipSubMeshTracer(t *testing.T) { // we only need one node with a meshTracer to test the meshTracer. // meshTracer logs at 1 second intervals for sake of testing. collector := mockmodule.NewGossipSubLocalMeshMetrics(t) - meshTracer := tracer.NewGossipSubMeshTracer(logger, collector, idProvider, 1*time.Second) + meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ + Logger: logger, + Metrics: collector, + IDProvider: idProvider, + LoggerInterval: 1 * time.Second, + RpcSentTrackerCacheCollector: metrics.NewNoopCollector(), + RpcSentTrackerCacheSize: uint32(100), + } + meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) + require.NoError(t, err) tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index 597f25c1534..34f34c4fa01 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -8,11 +8,9 @@ import ( herocache "github.com/onflow/flow-go/module/mempool/herocache/backdata" "github.com/onflow/flow-go/module/mempool/herocache/backdata/heropool" "github.com/onflow/flow-go/module/mempool/stdmap" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) -type entityFactory func(id flow.Identifier, controlMsgType p2p.ControlMessageType) rpcSentEntity - type RPCSentCacheConfig struct { sizeLimit uint32 logger zerolog.Logger @@ -52,7 +50,7 @@ func newRPCSentCache(config *RPCSentCacheConfig) (*rpcSentCache, error) { // - bool: true if the record is initialized, false otherwise (i.e.: the record already exists). // Note that if init is called multiple times for the same messageID, the record is initialized only once, and the // subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). -func (r *rpcSentCache) init(messageID flow.Identifier, controlMsgType p2p.ControlMessageType) bool { +func (r *rpcSentCache) init(messageID flow.Identifier, controlMsgType p2pmsg.ControlMessageType) bool { return r.c.Add(newRPCSentEntity(messageID, controlMsgType)) } diff --git a/network/p2p/tracer/internal/rpc_send_entity.go b/network/p2p/tracer/internal/rpc_send_entity.go index 493e4808440..b3f56ce0b55 100644 --- a/network/p2p/tracer/internal/rpc_send_entity.go +++ b/network/p2p/tracer/internal/rpc_send_entity.go @@ -2,7 +2,7 @@ package internal import ( "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // rpcSentEntity struct representing an RPC control message sent from local node. @@ -11,7 +11,7 @@ type rpcSentEntity struct { // messageID the messageID of the rpc control message. messageID flow.Identifier // controlMsgType the control message type. - controlMsgType p2p.ControlMessageType + controlMsgType p2pmsg.ControlMessageType } var _ flow.Entity = (*rpcSentEntity)(nil) @@ -29,7 +29,7 @@ func (r rpcSentEntity) Checksum() flow.Identifier { } // newRPCSentEntity returns a new rpcSentEntity. -func newRPCSentEntity(id flow.Identifier, controlMessageType p2p.ControlMessageType) rpcSentEntity { +func newRPCSentEntity(id flow.Identifier, controlMessageType p2pmsg.ControlMessageType) rpcSentEntity { return rpcSentEntity{ messageID: id, controlMsgType: controlMessageType, diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index 047976cd424..f580c443f41 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -3,12 +3,12 @@ package internal import ( "fmt" - pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/rs/zerolog" + pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) // RPCSentTracker tracks RPC messages that are sent. @@ -33,9 +33,9 @@ func NewRPCSentTracker(logger zerolog.Logger, sizeLimit uint32, collector module // OnIHaveRPCSent caches a unique entity message ID for each message ID included in each rpc iHave control message. // Args: // - *pubsub.RPC: the rpc sent. -func (t *RPCSentTracker) OnIHaveRPCSent(rpc *pubsub.RPC) { - controlMsgType := p2p.CtrlMsgIHave - for _, iHave := range rpc.GetControl().GetIhave() { +func (t *RPCSentTracker) OnIHaveRPCSent(iHaves []*pb.ControlIHave) { + controlMsgType := p2pmsg.CtrlMsgIHave + for _, iHave := range iHaves { topicID := iHave.GetTopicID() for _, messageID := range iHave.GetMessageIDs() { entityMsgID := iHaveRPCSentEntityID(topicID, messageID) From 86884ea0a915b4927d6a15341f93c5863d07d184 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 12:57:18 +0300 Subject: [PATCH 086/169] Refactored according to comments --- .../node_builder/access_node_builder.go | 11 +--- cmd/observer/node_builder/observer_builder.go | 54 +++++++++---------- engine/access/rest_api_test.go | 10 ++-- engine/access/rpc/engine.go | 2 + engine/access/rpc/rate_limit_test.go | 11 ++-- engine/access/secure_grpcr_test.go | 29 +++++++--- engine/access/state_stream/engine.go | 1 + module/component/component.go | 12 +++++ module/grpcserver/server.go | 4 +- module/grpcserver/server_builder.go | 7 ++- 10 files changed, 81 insertions(+), 60 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 175bb1e74d0..c6d2b3d258d 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -991,27 +991,20 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("creating grpc servers", func(node *cmd.NodeConfig) error { - var err error - builder.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, builder.rpcConf.SecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits, grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() - if err != nil { - return err - } - builder.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, builder.rpcConf.UnsecureGRPCListenAddr, builder.rpcConf.MaxMsgSize, builder.rpcMetricsEnabled, builder.apiRatelimits, builder.apiBurstlimits).Build() - if err != nil { - return err - } return nil }). diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 6ab82e4d776..c97e85ed998 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -164,6 +164,10 @@ type ObserverServiceBuilder struct { // Public network peerID peer.ID + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer } // deriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters. @@ -641,9 +645,7 @@ func (builder *ObserverServiceBuilder) Initialize() error { builder.enqueueConnectWithStakedAN() - if err := builder.enqueueRPCServer(); err != nil { - return err - } + builder.enqueueRPCServer() if builder.BaseConfig.MetricsEnabled { builder.EnqueueMetricsServerInit() @@ -843,28 +845,25 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { }) } -func (builder *ObserverServiceBuilder) enqueueRPCServer() error { - secureGrpcServer, err := grpcserver.NewGrpcServerBuilder(builder.Logger, - builder.rpcConf.SecureGRPCListenAddr, - builder.rpcConf.MaxMsgSize, - builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits, - grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() - if err != nil { - return err - } +func (builder *ObserverServiceBuilder) enqueueRPCServer() { + builder.Module("creating grpc servers", func(node *cmd.NodeConfig) error { + builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.SecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits, + grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() - unsecureGrpcServer, err := grpcserver.NewGrpcServerBuilder(builder.Logger, - builder.rpcConf.UnsecureGRPCListenAddr, - builder.rpcConf.MaxMsgSize, - builder.rpcMetricsEnabled, - builder.apiRatelimits, - builder.apiBurstlimits).Build() - if err != nil { - return err - } + builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.rpcConf.UnsecureGRPCListenAddr, + builder.rpcConf.MaxMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits).Build() + return nil + }) builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { engineBuilder, err := rpc.NewBuilder( node.Logger, @@ -885,8 +884,8 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() error { false, builder.rpcMetricsEnabled, builder.Me, - secureGrpcServer, - unsecureGrpcServer, + builder.secureGrpcServer, + builder.unsecureGrpcServer, ) if err != nil { return nil, err @@ -924,14 +923,13 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() error { // build secure grpc server builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - return secureGrpcServer, nil + return builder.secureGrpcServer, nil }) // build unsecure grpc server builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - return unsecureGrpcServer, nil + return builder.unsecureGrpcServer, nil }) - return nil } // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update diff --git a/engine/access/rest_api_test.go b/engine/access/rest_api_test.go index 2c6dc95ebcd..8cab886605c 100644 --- a/engine/access/rest_api_test.go +++ b/engine/access/rest_api_test.go @@ -135,22 +135,20 @@ func (suite *RestAPITestSuite) SetupTest() { // set the transport credentials for the server to use config.TransportCredentials = credentials.NewTLS(tlsConfig) - suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil, grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() - assert.NoError(suite.T(), err) - suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil).Build() - assert.NoError(suite.T(), err) rpcEngBuilder, err := rpc.NewBuilder( suite.log, @@ -180,6 +178,8 @@ func (suite *RestAPITestSuite) SetupTest() { suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.rpcEng.Start(suite.ctx) + suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) @@ -187,7 +187,7 @@ func (suite *RestAPITestSuite) SetupTest() { unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) - suite.rpcEng.Start(suite.ctx) + // wait for the engine to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) } diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 0013b04618e..6e328e66055 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -178,6 +178,8 @@ func NewBuilder(log zerolog.Logger, eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). + //AddWorker(component.WaitForComponentReady(secureGrpcServer)). + //AddWorker(component.WaitForComponentReady(unsecureGrpcServer)). AddWorker(eng.serveGRPCWebProxyWorker). AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). diff --git a/engine/access/rpc/rate_limit_test.go b/engine/access/rpc/rate_limit_test.go index da92dd2c42a..3fce4aeded0 100644 --- a/engine/access/rpc/rate_limit_test.go +++ b/engine/access/rpc/rate_limit_test.go @@ -128,22 +128,20 @@ func (suite *RateLimitTestSuite) SetupTest() { "Ping": suite.rateLimit, } - suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, apiRateLimt, apiBurstLimt, grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() - require.NoError(suite.T(), err) - suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, apiRateLimt, apiBurstLimt).Build() - require.NoError(suite.T(), err) block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) @@ -155,6 +153,8 @@ func (suite *RateLimitTestSuite) SetupTest() { require.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.rpcEng.Start(suite.ctx) + suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) @@ -162,8 +162,7 @@ func (suite *RateLimitTestSuite) SetupTest() { unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) - suite.rpcEng.Start(suite.ctx) - + // wait for the engine to startup unittest.RequireCloseBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second, "engine not ready at startup") // create the access api client diff --git a/engine/access/secure_grpcr_test.go b/engine/access/secure_grpcr_test.go index d7f3325d45d..82fbfa0bd5a 100644 --- a/engine/access/secure_grpcr_test.go +++ b/engine/access/secure_grpcr_test.go @@ -2,19 +2,20 @@ package access import ( "context" - "io" "os" "testing" "time" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow-go/crypto" accessmock "github.com/onflow/flow-go/engine/access/mock" @@ -111,22 +112,20 @@ func (suite *SecureGRPCTestSuite) SetupTest() { // save the public key to use later in tests later suite.publicKey = networkingKey.PublicKey() - suite.secureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.SecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil, grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() - assert.NoError(suite.T(), err) - suite.unsecureGrpcServer, err = grpcserver.NewGrpcServerBuilder(suite.log, + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, config.UnsecureGRPCListenAddr, grpcutils.DefaultMaxMsgSize, false, nil, nil).Build() - assert.NoError(suite.T(), err) block := unittest.BlockHeaderFixture() suite.snapshot.On("Head").Return(block, nil) @@ -158,6 +157,8 @@ func (suite *SecureGRPCTestSuite) SetupTest() { assert.NoError(suite.T(), err) suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + suite.rpcEng.Start(suite.ctx) + suite.secureGrpcServer.Start(suite.ctx) suite.unsecureGrpcServer.Start(suite.ctx) @@ -165,8 +166,7 @@ func (suite *SecureGRPCTestSuite) SetupTest() { unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) - suite.rpcEng.Start(suite.ctx) - // wait for the server to startup + // wait for the engine to startup unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) } @@ -205,6 +205,19 @@ func (suite *SecureGRPCTestSuite) TestAPICallUsingSecureGRPC() { _, err := client.Ping(ctx, req) assert.Error(suite.T(), err) }) + + suite.Run("happy path - connection fails, unsecure client can not get info from secure server connection", func() { + conn, err := grpc.Dial( + suite.secureGrpcServer.GRPCAddress().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(suite.T(), err) + + client := accessproto.NewAccessAPIClient(conn) + closer := io.Closer(conn) + defer closer.Close() + + _, err = client.Ping(ctx, req) + assert.Error(suite.T(), err) + }) } // secureGRPCClient creates a secure GRPC client using the given public key diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index 2fec00781e5..83667f81ea4 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -114,6 +114,7 @@ func NewEng( } e.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(component.WaitForComponentReady(server)). Build() access.RegisterExecutionDataAPIServer(server.Server, e.handler) diff --git a/module/component/component.go b/module/component/component.go index 34f8f61cf14..38d21781053 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -122,6 +122,18 @@ func RunComponent(ctx context.Context, componentFactory ComponentFactory, handle } } +// WaitForComponentReady is used for worker if it needs to wait until another component is ready +func WaitForComponentReady(component Component) ComponentWorker { + return func(ctx irrecoverable.SignalerContext, ready ReadyFunc) { + select { + case <-ctx.Done(): + case <-component.Ready(): + ready() + } + <-component.Done() + } +} + // ReadyFunc is called within a ComponentWorker function to indicate that the worker is ready // ComponentManager's Ready channel is closed when all workers are ready. type ReadyFunc func() diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index 1fa71c3007b..b68471d7981 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -29,7 +29,7 @@ type GrpcServer struct { func NewGrpcServer(log zerolog.Logger, grpcListenAddr string, grpcServer *grpc.Server, -) (*GrpcServer, error) { +) *GrpcServer { server := &GrpcServer{ log: log, Server: grpcServer, @@ -39,7 +39,7 @@ func NewGrpcServer(log zerolog.Logger, AddWorker(server.serveGRPCWorker). AddWorker(server.shutdownWorker). Build() - return server, nil + return server } // serveGRPCWorker is a worker routine which starts the gRPC server. diff --git a/module/grpcserver/server_builder.go b/module/grpcserver/server_builder.go index 3f71fdbe4c6..0db2a0a98d7 100644 --- a/module/grpcserver/server_builder.go +++ b/module/grpcserver/server_builder.go @@ -42,7 +42,6 @@ func NewGrpcServerBuilder(log zerolog.Logger, log = log.With().Str("component", "grpc_server").Logger() grpcServerBuilder := &GrpcServerBuilder{ - log: log, gRPCListenAddr: gRPCListenAddr, } @@ -74,14 +73,18 @@ func NewGrpcServerBuilder(log zerolog.Logger, grpcOpts = append(grpcOpts, chainedInterceptors) if grpcServerBuilder.transportCredentials != nil { + log = log.With().Str("endpoint", "secure").Logger() // create a secure server by using the secure grpc credentials that are passed in as part of config grpcOpts = append(grpcOpts, grpc.Creds(grpcServerBuilder.transportCredentials)) + } else { + log = log.With().Str("endpoint", "unsecure").Logger() } + grpcServerBuilder.log = log grpcServerBuilder.server = grpc.NewServer(grpcOpts...) return grpcServerBuilder } -func (b *GrpcServerBuilder) Build() (*GrpcServer, error) { +func (b *GrpcServerBuilder) Build() *GrpcServer { return NewGrpcServer(b.log, b.gRPCListenAddr, b.server) } From a879d4ebc1c46375cee5fe5a6e50595fcb9a01af Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 18:32:45 +0300 Subject: [PATCH 087/169] Updated module ordering --- .../node_builder/access_node_builder.go | 17 ++++++++------ engine/access/rpc/engine.go | 22 +++++++++++++++++++ module/component/component.go | 15 ++++++++----- module/grpcserver/server.go | 11 +++++++--- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index c6d2b3d258d..0f5e600df4a 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -896,6 +896,14 @@ func (builder *FlowAccessNodeBuilder) enqueueRelayNetwork() { } func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { + builder.Module("access metrics", func(node *cmd.NodeConfig) error { + builder.AccessMetrics = metrics.NewAccessCollector( + metrics.WithTransactionMetrics(builder.TransactionMetrics), + metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), + ) + return nil + }) + builder. BuildConsensusFollower(). Module("collection node client", func(node *cmd.NodeConfig) error { @@ -969,13 +977,6 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ) return nil }). - Module("access metrics", func(node *cmd.NodeConfig) error { - builder.AccessMetrics = metrics.NewAccessCollector( - metrics.WithTransactionMetrics(builder.TransactionMetrics), - metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), - ) - return nil - }). Module("ping metrics", func(node *cmd.NodeConfig) error { builder.PingMetrics = metrics.NewPingCollector() return nil @@ -1009,6 +1010,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + builder.Logger.Info().Msg("____RPC engine") engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, @@ -1120,6 +1122,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { } builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + builder.Logger.Info().Msg("____secure grpc server in access node builder") return builder.secureGrpcServer, nil }) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 6e328e66055..3f5a3fa339f 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -178,6 +178,28 @@ func NewBuilder(log zerolog.Logger, eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + log.Info().Msg("================> Start secure add worker") + select { + case <-ctx.Done(): + case <-secureGrpcServer.Ready(): + log.Info().Msg("================> secure grpc component is ready") + ready() + } + log.Info().Msg("================> secure grpc component done") + <-secureGrpcServer.Done() + }). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + log.Info().Msg("================> Start unsecure add worker") + select { + case <-ctx.Done(): + case <-unsecureGrpcServer.Ready(): + log.Info().Msg("================> unsecure grpc component is ready") + ready() + } + log.Info().Msg("================> unsecure grpc component done") + <-unsecureGrpcServer.Done() + }). //AddWorker(component.WaitForComponentReady(secureGrpcServer)). //AddWorker(component.WaitForComponentReady(unsecureGrpcServer)). AddWorker(eng.serveGRPCWebProxyWorker). diff --git a/module/component/component.go b/module/component/component.go index 38d21781053..2bf636dc3d7 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -3,6 +3,7 @@ package component import ( "context" "fmt" + "github.com/rs/zerolog" "sync" "go.uber.org/atomic" @@ -160,20 +161,21 @@ type ComponentManagerBuilder interface { Build() *ComponentManager } -type componentManagerBuilderImpl struct { +type ComponentManagerBuilderImpl struct { + Log zerolog.Logger workers []ComponentWorker } // NewComponentManagerBuilder returns a new ComponentManagerBuilder func NewComponentManagerBuilder() ComponentManagerBuilder { - return &componentManagerBuilderImpl{} + return &ComponentManagerBuilderImpl{} } // AddWorker adds a ComponentWorker closure to the ComponentManagerBuilder // All worker functions will be run in parallel when the ComponentManager is started. // Note: AddWorker is not concurrency-safe, and should only be called on an individual builder // within a single goroutine. -func (c *componentManagerBuilderImpl) AddWorker(worker ComponentWorker) ComponentManagerBuilder { +func (c *ComponentManagerBuilderImpl) AddWorker(worker ComponentWorker) ComponentManagerBuilder { c.workers = append(c.workers, worker) return c } @@ -182,8 +184,9 @@ func (c *componentManagerBuilderImpl) AddWorker(worker ComponentWorker) Componen // Build may be called multiple times to create multiple individual ComponentManagers. This will // result in the worker routines being called multiple times. If this is unsafe, do not call it // more than once! -func (c *componentManagerBuilderImpl) Build() *ComponentManager { - return &ComponentManager{ +func (c *ComponentManagerBuilderImpl) Build() *ComponentManager { + c.Log.Info().Msg("_______ComponentManagerBuilderImpl Build()") + cm := &ComponentManager{ started: atomic.NewBool(false), ready: make(chan struct{}), done: make(chan struct{}), @@ -191,6 +194,8 @@ func (c *componentManagerBuilderImpl) Build() *ComponentManager { shutdownSignal: make(chan struct{}), workers: c.workers, } + c.Log.Info().Msg(fmt.Sprintf("_______ComponentManagerBuilderImpl %d", len(c.workers))) + return cm } var _ Component = (*ComponentManager)(nil) diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index b68471d7981..ebfd67bd312 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -30,22 +30,27 @@ func NewGrpcServer(log zerolog.Logger, grpcListenAddr string, grpcServer *grpc.Server, ) *GrpcServer { + log.Info().Msg("================> NewGrpcServer") server := &GrpcServer{ log: log, Server: grpcServer, grpcListenAddr: grpcListenAddr, } - server.Component = component.NewComponentManagerBuilder(). + cm := component.ComponentManagerBuilderImpl{Log: log} + server.Component = cm. AddWorker(server.serveGRPCWorker). AddWorker(server.shutdownWorker). Build() + log.Info().Msg("================> End NewGrpcServer") return server } // serveGRPCWorker is a worker routine which starts the gRPC server. // The ready callback is called after the server address is bound and set. func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - g.log.Info().Str("grpc_address", g.grpcListenAddr).Msg("starting grpc server on address") + g.log = g.log.With().Str("grpc_address", g.grpcListenAddr).Logger() + g.log.Info().Msg("================> serveGRPCWorker") + g.log.Info().Msg("starting grpc server on address") l, err := net.Listen("tcp", g.grpcListenAddr) if err != nil { @@ -59,7 +64,7 @@ func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready co g.addrLock.Lock() g.grpcAddress = l.Addr() g.addrLock.Unlock() - g.log.Debug().Str("grpc_address", g.grpcAddress.String()).Msg("listening on port") + g.log.Debug().Msg("listening on port") ready() err = g.Server.Serve(l) // blocking call From 9f04ea151633e61e9f1edc5e29eabe61e8ac52a9 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 19:28:07 +0300 Subject: [PATCH 088/169] Removed unnecessary logs --- .../node_builder/access_node_builder.go | 17 +++++------- engine/access/rpc/engine.go | 26 ++----------------- module/component/component.go | 15 ++++------- module/grpcserver/server.go | 6 +---- 4 files changed, 15 insertions(+), 49 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 0f5e600df4a..c6d2b3d258d 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -896,14 +896,6 @@ func (builder *FlowAccessNodeBuilder) enqueueRelayNetwork() { } func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { - builder.Module("access metrics", func(node *cmd.NodeConfig) error { - builder.AccessMetrics = metrics.NewAccessCollector( - metrics.WithTransactionMetrics(builder.TransactionMetrics), - metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), - ) - return nil - }) - builder. BuildConsensusFollower(). Module("collection node client", func(node *cmd.NodeConfig) error { @@ -977,6 +969,13 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ) return nil }). + Module("access metrics", func(node *cmd.NodeConfig) error { + builder.AccessMetrics = metrics.NewAccessCollector( + metrics.WithTransactionMetrics(builder.TransactionMetrics), + metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), + ) + return nil + }). Module("ping metrics", func(node *cmd.NodeConfig) error { builder.PingMetrics = metrics.NewPingCollector() return nil @@ -1010,7 +1009,6 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - builder.Logger.Info().Msg("____RPC engine") engineBuilder, err := rpc.NewBuilder( node.Logger, node.State, @@ -1122,7 +1120,6 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { } builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - builder.Logger.Info().Msg("____secure grpc server in access node builder") return builder.secureGrpcServer, nil }) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 3f5a3fa339f..1f2bdffacf4 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -178,30 +178,8 @@ func NewBuilder(log zerolog.Logger, eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - log.Info().Msg("================> Start secure add worker") - select { - case <-ctx.Done(): - case <-secureGrpcServer.Ready(): - log.Info().Msg("================> secure grpc component is ready") - ready() - } - log.Info().Msg("================> secure grpc component done") - <-secureGrpcServer.Done() - }). - AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { - log.Info().Msg("================> Start unsecure add worker") - select { - case <-ctx.Done(): - case <-unsecureGrpcServer.Ready(): - log.Info().Msg("================> unsecure grpc component is ready") - ready() - } - log.Info().Msg("================> unsecure grpc component done") - <-unsecureGrpcServer.Done() - }). - //AddWorker(component.WaitForComponentReady(secureGrpcServer)). - //AddWorker(component.WaitForComponentReady(unsecureGrpcServer)). + AddWorker(component.WaitForComponentReady(secureGrpcServer)). + AddWorker(component.WaitForComponentReady(unsecureGrpcServer)). AddWorker(eng.serveGRPCWebProxyWorker). AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). diff --git a/module/component/component.go b/module/component/component.go index 2bf636dc3d7..38d21781053 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -3,7 +3,6 @@ package component import ( "context" "fmt" - "github.com/rs/zerolog" "sync" "go.uber.org/atomic" @@ -161,21 +160,20 @@ type ComponentManagerBuilder interface { Build() *ComponentManager } -type ComponentManagerBuilderImpl struct { - Log zerolog.Logger +type componentManagerBuilderImpl struct { workers []ComponentWorker } // NewComponentManagerBuilder returns a new ComponentManagerBuilder func NewComponentManagerBuilder() ComponentManagerBuilder { - return &ComponentManagerBuilderImpl{} + return &componentManagerBuilderImpl{} } // AddWorker adds a ComponentWorker closure to the ComponentManagerBuilder // All worker functions will be run in parallel when the ComponentManager is started. // Note: AddWorker is not concurrency-safe, and should only be called on an individual builder // within a single goroutine. -func (c *ComponentManagerBuilderImpl) AddWorker(worker ComponentWorker) ComponentManagerBuilder { +func (c *componentManagerBuilderImpl) AddWorker(worker ComponentWorker) ComponentManagerBuilder { c.workers = append(c.workers, worker) return c } @@ -184,9 +182,8 @@ func (c *ComponentManagerBuilderImpl) AddWorker(worker ComponentWorker) Componen // Build may be called multiple times to create multiple individual ComponentManagers. This will // result in the worker routines being called multiple times. If this is unsafe, do not call it // more than once! -func (c *ComponentManagerBuilderImpl) Build() *ComponentManager { - c.Log.Info().Msg("_______ComponentManagerBuilderImpl Build()") - cm := &ComponentManager{ +func (c *componentManagerBuilderImpl) Build() *ComponentManager { + return &ComponentManager{ started: atomic.NewBool(false), ready: make(chan struct{}), done: make(chan struct{}), @@ -194,8 +191,6 @@ func (c *ComponentManagerBuilderImpl) Build() *ComponentManager { shutdownSignal: make(chan struct{}), workers: c.workers, } - c.Log.Info().Msg(fmt.Sprintf("_______ComponentManagerBuilderImpl %d", len(c.workers))) - return cm } var _ Component = (*ComponentManager)(nil) diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index ebfd67bd312..425c37a042a 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -30,18 +30,15 @@ func NewGrpcServer(log zerolog.Logger, grpcListenAddr string, grpcServer *grpc.Server, ) *GrpcServer { - log.Info().Msg("================> NewGrpcServer") server := &GrpcServer{ log: log, Server: grpcServer, grpcListenAddr: grpcListenAddr, } - cm := component.ComponentManagerBuilderImpl{Log: log} - server.Component = cm. + server.Component = component.NewComponentManagerBuilder(). AddWorker(server.serveGRPCWorker). AddWorker(server.shutdownWorker). Build() - log.Info().Msg("================> End NewGrpcServer") return server } @@ -49,7 +46,6 @@ func NewGrpcServer(log zerolog.Logger, // The ready callback is called after the server address is bound and set. func (g *GrpcServer) serveGRPCWorker(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { g.log = g.log.With().Str("grpc_address", g.grpcListenAddr).Logger() - g.log.Info().Msg("================> serveGRPCWorker") g.log.Info().Msg("starting grpc server on address") l, err := net.Listen("tcp", g.grpcListenAddr) From d72f862a95c85cfdb252ff8ec065668effc8ee51 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 19:34:52 +0300 Subject: [PATCH 089/169] Added integration test accorfing to comment --- .../integration_unsecure_grpc_server_test.go | 298 ++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 engine/access/integration_unsecure_grpc_server_test.go diff --git a/engine/access/integration_unsecure_grpc_server_test.go b/engine/access/integration_unsecure_grpc_server_test.go new file mode 100644 index 00000000000..e5794828028 --- /dev/null +++ b/engine/access/integration_unsecure_grpc_server_test.go @@ -0,0 +1,298 @@ +package access + +import ( + "context" + "io" + "os" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/onflow/flow-go/engine" + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc" + "github.com/onflow/flow-go/engine/access/state_stream" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/blobs" + "github.com/onflow/flow-go/module/executiondatasync/execution_data" + "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" + "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mempool/herocache" + "github.com/onflow/flow-go/module/metrics" + module "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network" + protocol "github.com/onflow/flow-go/state/protocol/mock" + "github.com/onflow/flow-go/storage" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/grpcutils" + "github.com/onflow/flow-go/utils/unittest" + + accessproto "github.com/onflow/flow/protobuf/go/flow/access" + executiondataproto "github.com/onflow/flow/protobuf/go/flow/executiondata" +) + +// SameGRPCPortTestSuite verifies both continue to work when configured +// on the AccessAPI and ExecutionDataAPI on the same port +type SameGRPCPortTestSuite struct { + suite.Suite + state *protocol.State + snapshot *protocol.Snapshot + epochQuery *protocol.EpochQuery + log zerolog.Logger + net *network.Network + request *module.Requester + collClient *accessmock.AccessAPIClient + execClient *accessmock.ExecutionAPIClient + me *module.Local + chainID flow.ChainID + metrics *metrics.NoopCollector + rpcEng *rpc.Engine + stateStreamEng *state_stream.Engine + + // storage + blocks *storagemock.Blocks + headers *storagemock.Headers + collections *storagemock.Collections + transactions *storagemock.Transactions + receipts *storagemock.ExecutionReceipts + seals *storagemock.Seals + results *storagemock.ExecutionResults + + ctx irrecoverable.SignalerContext + cancel context.CancelFunc + + // grpc servers + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer + + bs blobs.Blobstore + eds execution_data.ExecutionDataStore + broadcaster *engine.Broadcaster + execDataCache *cache.ExecutionDataCache + execDataHeroCache *herocache.BlockExecutionData + + blockMap map[uint64]*flow.Block +} + +func (suite *SameGRPCPortTestSuite) SetupTest() { + suite.log = zerolog.New(os.Stdout) + suite.net = new(network.Network) + suite.state = new(protocol.State) + suite.snapshot = new(protocol.Snapshot) + + suite.epochQuery = new(protocol.EpochQuery) + suite.state.On("Sealed").Return(suite.snapshot, nil).Maybe() + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Epochs").Return(suite.epochQuery).Maybe() + suite.blocks = new(storagemock.Blocks) + suite.headers = new(storagemock.Headers) + suite.transactions = new(storagemock.Transactions) + suite.collections = new(storagemock.Collections) + suite.receipts = new(storagemock.ExecutionReceipts) + suite.results = new(storagemock.ExecutionResults) + suite.seals = new(storagemock.Seals) + + suite.collClient = new(accessmock.AccessAPIClient) + suite.execClient = new(accessmock.ExecutionAPIClient) + + suite.request = new(module.Requester) + suite.request.On("EntityByID", mock.Anything, mock.Anything) + + suite.me = new(module.Local) + suite.eds = execution_data.NewExecutionDataStore(suite.bs, execution_data.DefaultSerializer) + + suite.broadcaster = engine.NewBroadcaster() + + suite.execDataHeroCache = herocache.NewBlockExecutionData(state_stream.DefaultCacheSize, suite.log, metrics.NewNoopCollector()) + suite.execDataCache = cache.NewExecutionDataCache(suite.eds, suite.headers, suite.seals, suite.results, suite.execDataHeroCache) + + accessIdentity := unittest.IdentityFixture(unittest.WithRole(flow.RoleAccess)) + suite.me. + On("NodeID"). + Return(accessIdentity.NodeID) + + suite.chainID = flow.Testnet + suite.metrics = metrics.NewNoopCollector() + + config := rpc.Config{ + UnsecureGRPCListenAddr: unittest.DefaultAddress, + SecureGRPCListenAddr: unittest.DefaultAddress, + HTTPListenAddr: unittest.DefaultAddress, + } + + blockCount := 5 + suite.blockMap = make(map[uint64]*flow.Block, blockCount) + // generate blockCount consecutive blocks with associated seal, result and execution data + rootBlock := unittest.BlockFixture() + parent := rootBlock.Header + suite.blockMap[rootBlock.Header.Height] = &rootBlock + + for i := 0; i < blockCount; i++ { + block := unittest.BlockWithParentFixture(parent) + suite.blockMap[block.Header.Height] = block + } + + // generate a server certificate that will be served by the GRPC server + networkingKey := unittest.NetworkingPrivKeyFixture() + x509Certificate, err := grpcutils.X509Certificate(networkingKey) + assert.NoError(suite.T(), err) + tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) + // set the transport credentials for the server to use + config.TransportCredentials = credentials.NewTLS(tlsConfig) + + suite.secureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.SecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil, + grpcserver.WithTransportCredentials(config.TransportCredentials)).Build() + + suite.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(suite.log, + config.UnsecureGRPCListenAddr, + grpcutils.DefaultMaxMsgSize, + false, + nil, + nil).Build() + + block := unittest.BlockHeaderFixture() + suite.snapshot.On("Head").Return(block, nil) + + // create rpc engine builder + rpcEngBuilder, err := rpc.NewBuilder( + suite.log, + suite.state, + config, + suite.collClient, + nil, + suite.blocks, + suite.headers, + suite.collections, + suite.transactions, + nil, + nil, + suite.chainID, + suite.metrics, + 0, + 0, + false, + false, + suite.me, + suite.secureGrpcServer, + suite.unsecureGrpcServer, + ) + assert.NoError(suite.T(), err) + suite.rpcEng, err = rpcEngBuilder.WithLegacy().Build() + assert.NoError(suite.T(), err) + suite.ctx, suite.cancel = irrecoverable.NewMockSignalerContextWithCancel(suite.T(), context.Background()) + + suite.headers.On("BlockIDByHeight", mock.AnythingOfType("uint64")).Return( + func(height uint64) flow.Identifier { + if block, ok := suite.blockMap[height]; ok { + return block.Header.ID() + } + return flow.ZeroID + }, + func(height uint64) error { + if _, ok := suite.blockMap[height]; ok { + return nil + } + return storage.ErrNotFound + }, + ).Maybe() + + conf := state_stream.Config{ + ClientSendTimeout: state_stream.DefaultSendTimeout, + ClientSendBufferSize: state_stream.DefaultSendBufferSize, + } + + // create state stream engine + suite.stateStreamEng, err = state_stream.NewEng( + suite.log, + conf, + nil, + suite.execDataCache, + suite.state, + suite.headers, + suite.seals, + suite.results, + suite.chainID, + rootBlock.Header.Height, + rootBlock.Header.Height, + suite.unsecureGrpcServer, + ) + assert.NoError(suite.T(), err) + + suite.rpcEng.Start(suite.ctx) + suite.stateStreamEng.Start(suite.ctx) + + suite.secureGrpcServer.Start(suite.ctx) + suite.unsecureGrpcServer.Start(suite.ctx) + + // wait for the servers to startup + unittest.AssertClosesBefore(suite.T(), suite.secureGrpcServer.Ready(), 2*time.Second) + unittest.AssertClosesBefore(suite.T(), suite.unsecureGrpcServer.Ready(), 2*time.Second) + + // wait for the rpc engine to startup + unittest.AssertClosesBefore(suite.T(), suite.rpcEng.Ready(), 2*time.Second) + // wait for the state stream engine to startup + unittest.AssertClosesBefore(suite.T(), suite.stateStreamEng.Ready(), 2*time.Second) +} + +func (suite *SameGRPCPortTestSuite) TestEnginesOnTheSameGrpcPort() { + ctx := context.Background() + + conn, err := grpc.Dial( + suite.unsecureGrpcServer.GRPCAddress().String(), + grpc.WithTransportCredentials(insecure.NewCredentials())) + assert.NoError(suite.T(), err) + closer := io.Closer(conn) + + suite.Run("happy path - grpc access api client can connect successfully", func() { + req := &accessproto.GetNetworkParametersRequest{} + + // expect 2 upstream calls + suite.execClient.On("GetNetworkParameters", mock.Anything, mock.Anything).Return(nil, nil).Twice() + suite.collClient.On("GetNetworkParameters", mock.Anything, mock.Anything).Return(nil, nil).Twice() + + client := suite.unsecureAccessAPIClient(conn) + + _, err := client.GetNetworkParameters(ctx, req) + assert.NoError(suite.T(), err, "failed to get network") + }) + + suite.Run("happy path - grpc execution data api client can connect successfully", func() { + req := &executiondataproto.SubscribeEventsRequest{} + + client := suite.unsecureExecutionDataAPIClient(conn) + + _, err := client.SubscribeEvents(ctx, req) + assert.NoError(suite.T(), err, "failed to subscribe events") + }) + defer closer.Close() +} + +func TestSameGRPCTestSuite(t *testing.T) { + suite.Run(t, new(SameGRPCPortTestSuite)) +} + +// unsecureGRPCClient creates an unsecure GRPC client +func (suite *SameGRPCPortTestSuite) unsecureAccessAPIClient(conn *grpc.ClientConn) accessproto.AccessAPIClient { + client := accessproto.NewAccessAPIClient(conn) + return client +} + +// unsecureExecutionDataAPIClient creates an unsecure ExecutionDataAPI client +func (suite *SameGRPCPortTestSuite) unsecureExecutionDataAPIClient(conn *grpc.ClientConn) executiondataproto.ExecutionDataAPIClient { + client := executiondataproto.NewExecutionDataAPIClient(conn) + return client +} From 3aa370534ab1da600ccbe8c4f8bf4ae9b380021e Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 20:07:11 +0300 Subject: [PATCH 090/169] Updated comment --- engine/access/integration_unsecure_grpc_server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/access/integration_unsecure_grpc_server_test.go b/engine/access/integration_unsecure_grpc_server_test.go index e5794828028..78725d73f7e 100644 --- a/engine/access/integration_unsecure_grpc_server_test.go +++ b/engine/access/integration_unsecure_grpc_server_test.go @@ -40,8 +40,8 @@ import ( executiondataproto "github.com/onflow/flow/protobuf/go/flow/executiondata" ) -// SameGRPCPortTestSuite verifies both continue to work when configured -// on the AccessAPI and ExecutionDataAPI on the same port +// SameGRPCPortTestSuite verifies both AccessAPI and ExecutionDataAPI client continue to work when configured +// on the same port type SameGRPCPortTestSuite struct { suite.Suite state *protocol.State From b56ebee57e8fe79bcac4181b41a1ab70863de118 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 6 Jul 2023 21:40:15 +0300 Subject: [PATCH 091/169] Added workers to engines --- engine/access/rpc/engine.go | 10 ++++++++-- engine/access/state_stream/engine.go | 6 +++++- module/component/component.go | 12 ------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/engine/access/rpc/engine.go b/engine/access/rpc/engine.go index 1f2bdffacf4..bfbeb07aa53 100644 --- a/engine/access/rpc/engine.go +++ b/engine/access/rpc/engine.go @@ -178,8 +178,14 @@ func NewBuilder(log zerolog.Logger, eng.backendNotifierActor = backendNotifierActor eng.Component = component.NewComponentManagerBuilder(). - AddWorker(component.WaitForComponentReady(secureGrpcServer)). - AddWorker(component.WaitForComponentReady(unsecureGrpcServer)). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-secureGrpcServer.Done() + }). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-unsecureGrpcServer.Done() + }). AddWorker(eng.serveGRPCWebProxyWorker). AddWorker(eng.serveREST). AddWorker(finalizedCacheWorker). diff --git a/engine/access/state_stream/engine.go b/engine/access/state_stream/engine.go index 83667f81ea4..cb3a3e73813 100644 --- a/engine/access/state_stream/engine.go +++ b/engine/access/state_stream/engine.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" "github.com/onflow/flow-go/module/grpcserver" + "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/storage" "github.com/onflow/flow-go/utils/logging" @@ -114,7 +115,10 @@ func NewEng( } e.ComponentManager = component.NewComponentManagerBuilder(). - AddWorker(component.WaitForComponentReady(server)). + AddWorker(func(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + <-server.Done() + }). Build() access.RegisterExecutionDataAPIServer(server.Server, e.handler) diff --git a/module/component/component.go b/module/component/component.go index 38d21781053..34f8f61cf14 100644 --- a/module/component/component.go +++ b/module/component/component.go @@ -122,18 +122,6 @@ func RunComponent(ctx context.Context, componentFactory ComponentFactory, handle } } -// WaitForComponentReady is used for worker if it needs to wait until another component is ready -func WaitForComponentReady(component Component) ComponentWorker { - return func(ctx irrecoverable.SignalerContext, ready ReadyFunc) { - select { - case <-ctx.Done(): - case <-component.Ready(): - ready() - } - <-component.Done() - } -} - // ReadyFunc is called within a ComponentWorker function to indicate that the worker is ready // ComponentManager's Ready channel is closed when all workers are ready. type ReadyFunc func() From 923de3dfcaba27ebfe37cccf7c9aa3b16579ae70 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 6 Jul 2023 14:48:29 -0400 Subject: [PATCH 092/169] Update module/metrics/network.go Co-authored-by: Peter Argue <89119817+peterargue@users.noreply.github.com> --- module/metrics/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/metrics/network.go b/module/metrics/network.go index a1a92c28274..311dbba9f15 100644 --- a/module/metrics/network.go +++ b/module/metrics/network.go @@ -251,7 +251,7 @@ func NewNetworkCollector(logger zerolog.Logger, opts ...NetworkCollectorOpt) *Ne Namespace: namespaceNetwork, Subsystem: subsystemSecurity, Name: nc.prefix + "slashing_violation_reports_skipped_count", - Help: "number of slashing violations consumer violations that were not reported for misbehavior when the identity of the sender not known", + Help: "number of slashing violations consumer violations that were not reported for misbehavior because the identity of the sender not known", }, ) From b68dcecdaca86c2c2c9be273b9623832a2f53668 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 6 Jul 2023 14:51:30 -0400 Subject: [PATCH 093/169] add comment about fatal log when creating misbehavior report --- network/slashing/consumer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index 765cdd657b1..b7f09bb12b7 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -82,9 +82,10 @@ func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation } report, err := alsp.NewMisbehaviorReport(violation.Identity.NodeID, misbehavior) if err != nil { + // failing to create the misbehavior report is unlikely. If an error is encountered while + // creating the misbehavior report it indicates a bug and processing can not proceed. c.log.Fatal(). Err(err). - Bool(logging.KeySuspicious, true). Str("peerID", violation.PeerID). Msg("failed to create misbehavior report") From c5b2ff05d41722f1484fa367fa4495ee34afb4c6 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 6 Jul 2023 14:52:40 -0400 Subject: [PATCH 094/169] remove unused field --- cmd/node_builder.go | 3 --- cmd/scaffold.go | 1 - 2 files changed, 4 deletions(-) diff --git a/cmd/node_builder.go b/cmd/node_builder.go index 94a553dbf00..1b944db832b 100644 --- a/cmd/node_builder.go +++ b/cmd/node_builder.go @@ -224,9 +224,6 @@ type NodeConfig struct { // UnicastRateLimiterDistributor notifies consumers when a peer's unicast message is rate limited. UnicastRateLimiterDistributor p2p.UnicastRateLimiterDistributor - // MisbehaviorReportConsumer consumers used to disseminate misbehavior reports to the ALSP misbehavior report manager. - MisbehaviorReportConsumer network.MisbehaviorReportConsumer - // GossipSubRpcInspectorSuite rpc inspector suite. GossipSubRpcInspectorSuite p2p.GossipSubInspectorSuite } diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 0269cde565f..dddbb68918c 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -499,7 +499,6 @@ func (fnb *FlowNodeBuilder) InitFlowNetworkWithConduitFactory( } fnb.Network = net - fnb.MisbehaviorReportConsumer = net // register middleware's ReadyDoneAware interface so other components can depend on it for startup if fnb.middlewareDependable != nil { From 4b138db07d8eff6108c0a955de3599ec2e88a714 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Thu, 6 Jul 2023 15:59:31 -0400 Subject: [PATCH 095/169] merge master --- network/alsp/manager/manager_test.go | 51 ++++++-------------------- network/internal/testutils/testUtil.go | 2 +- network/test/blob_service_test.go | 3 +- network/test/echoengine_test.go | 3 +- network/test/epochtransition_test.go | 3 +- network/test/meshengine_test.go | 11 +++--- 6 files changed, 24 insertions(+), 49 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index f9de16c3e44..9b067e1ede8 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -54,18 +54,9 @@ func TestNetworkPassesReportedMisbehavior(t *testing.T) { misbehaviorReportManger.On("Ready").Return(readyDoneChan).Once() misbehaviorReportManger.On("Done").Return(readyDoneChan).Once() ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) - mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t)) - - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(misbehaviorReportManger))) - sms := testutils.GenerateSubscriptionManagers(t, mws) - - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0]) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0]) net, err := p2p.NewNetwork(networkCfg, p2p.WithAlspManager(misbehaviorReportManger)) require.NoError(t, err) @@ -119,15 +110,9 @@ func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { return cache }), } - - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) - sms := testutils.GenerateSubscriptionManagers(t, mws) - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) net, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) @@ -219,15 +204,10 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) }), } - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 3, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) - sms := testutils.GenerateSubscriptionManagers(t, mws) - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) - + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 3, + p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) victimNetwork, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) @@ -298,16 +278,9 @@ func TestMisbehaviorReportMetrics(t *testing.T) { alspMetrics := mockmodule.NewAlspMetrics(t) cfg.AlspMetrics = alspMetrics - ids, nodes, mws, _, _ := testutils.GenerateIDsAndMiddlewares( - t, - 1, - unittest.Logger(), - unittest.NetworkCodec(), - unittest.NetworkSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), unittest.NewMisbehaviorReportConsumerFixture(nil))) - sms := testutils.GenerateSubscriptionManagers(t, mws) - - networkCfg := testutils.NetworkConfigFixture(t, unittest.Logger(), *ids[0], ids, mws[0], sms[0], p2p.WithAlspConfig(cfg)) - + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 1) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) net, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) diff --git a/network/internal/testutils/testUtil.go b/network/internal/testutils/testUtil.go index f387f4c4ac8..95707ee9e3c 100644 --- a/network/internal/testutils/testUtil.go +++ b/network/internal/testutils/testUtil.go @@ -197,8 +197,8 @@ func MiddlewareFixtures(t *testing.T, identities flow.IdentityList, libP2PNodes cfg.FlowId = identities[i].NodeID idProviders[i] = unittest.NewUpdatableIDProvider(identities) cfg.IdTranslator = translator.NewIdentityProviderIDTranslator(idProviders[i]) - mws[i].SetSlashingViolationsConsumer(consumer) mws[i] = middleware.NewMiddleware(cfg, opts...) + mws[i].SetSlashingViolationsConsumer(consumer) } return mws, idProviders } diff --git a/network/test/blob_service_test.go b/network/test/blob_service_test.go index 40c052111d7..c0979244ad8 100644 --- a/network/test/blob_service_test.go +++ b/network/test/blob_service_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/suite" "go.uber.org/atomic" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p/connection" "github.com/onflow/flow-go/network/p2p/dht" p2pconfig "github.com/onflow/flow-go/network/p2p/p2pbuilder/config" @@ -89,7 +90,7 @@ func (suite *BlobServiceTestSuite) SetupTest() { ConnectionPruning: true, ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), }, nil)) - mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) suite.networks = testutils.NetworksFixture(suite.T(), ids, mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.networks, 100*time.Millisecond) diff --git a/network/test/echoengine_test.go b/network/test/echoengine_test.go index eb170cbf266..55732b64d17 100644 --- a/network/test/echoengine_test.go +++ b/network/test/echoengine_test.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/utils/unittest" ) @@ -54,7 +55,7 @@ func (suite *EchoEngineTestSuite) SetupTest() { // both nodes should be of the same role to get connected on epidemic dissemination var nodes []p2p.LibP2PNode suite.ids, nodes, _ = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) - suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) } diff --git a/network/test/epochtransition_test.go b/network/test/epochtransition_test.go index e471b1d8f48..4e1eeabf717 100644 --- a/network/test/epochtransition_test.go +++ b/network/test/epochtransition_test.go @@ -24,6 +24,7 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/mocknetwork" mockprotocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -181,7 +182,7 @@ func (suite *MutableIdentityTableSuite) addNodes(count int) { // create the ids, middlewares and networks ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) - mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + mws, _ := testutils.MiddlewareFixtures(suite.T(), ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) nets := testutils.NetworksFixture(suite.T(), ids, mws) suite.cancels = append(suite.cancels, cancel) diff --git a/network/test/meshengine_test.go b/network/test/meshengine_test.go index 612d7679796..55a95994d45 100644 --- a/network/test/meshengine_test.go +++ b/network/test/meshengine_test.go @@ -11,8 +11,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/p2p" - "github.com/ipfs/go-log" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/rs/zerolog" @@ -20,9 +18,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/onflow/flow-go/network/p2p/middleware" - "github.com/onflow/flow-go/network/p2p/p2pnode" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/model/libp2p/message" @@ -31,6 +26,10 @@ import ( "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/internal/testutils" + "github.com/onflow/flow-go/network/mocknetwork" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/middleware" + "github.com/onflow/flow-go/network/p2p/p2pnode" "github.com/onflow/flow-go/utils/unittest" ) @@ -74,7 +73,7 @@ func (suite *MeshEngineTestSuite) SetupTest() { var nodes []p2p.LibP2PNode suite.ids, nodes, obs = testutils.LibP2PNodeForMiddlewareFixture(suite.T(), count) - suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T())) + suite.mws, _ = testutils.MiddlewareFixtures(suite.T(), suite.ids, nodes, testutils.MiddlewareConfigFixture(suite.T()), mocknetwork.NewViolationsConsumer(suite.T())) suite.nets = testutils.NetworksFixture(suite.T(), suite.ids, suite.mws) testutils.StartNodesAndNetworks(signalerCtx, suite.T(), nodes, suite.nets, 100*time.Millisecond) From 4e7b6146137c83bcc3d8d63f7edfc15ac11d33bb Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 01:45:35 -0600 Subject: [PATCH 096/169] fix verification tests by building QC's source of randoms ahead of building blocks --- .../computation/computer/computer_test.go | 18 +++--- .../execution_verification_test.go | 2 +- .../computation/manager_benchmark_test.go | 2 +- engine/execution/computation/manager_test.go | 4 +- engine/execution/computation/programs_test.go | 4 +- engine/execution/testutil/fixtures.go | 11 ++-- .../assigner/blockconsumer/consumer_test.go | 3 +- engine/verification/utils/unittest/fixture.go | 29 ++++++---- engine/verification/utils/unittest/helper.go | 6 +- utils/unittest/fixtures.go | 57 ++++++++++++++++++- 10 files changed, 101 insertions(+), 35 deletions(-) diff --git a/engine/execution/computation/computer/computer_test.go b/engine/execution/computation/computer/computer_test.go index c332c01b4ed..4428f06465f 100644 --- a/engine/execution/computation/computer/computer_test.go +++ b/engine/execution/computation/computer/computer_test.go @@ -172,7 +172,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -307,7 +307,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -405,7 +405,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -465,7 +465,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -682,7 +682,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -793,7 +793,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -906,7 +906,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -951,7 +951,7 @@ func TestBlockExecutor_ExecuteBlock(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -1271,7 +1271,7 @@ func Test_ExecutingSystemCollection(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) diff --git a/engine/execution/computation/execution_verification_test.go b/engine/execution/computation/execution_verification_test.go index e3efda55410..9b5f53641c1 100644 --- a/engine/execution/computation/execution_verification_test.go +++ b/engine/execution/computation/execution_verification_test.go @@ -781,7 +781,7 @@ func executeBlockAndVerifyWithParameters(t *testing.T, me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testVerifyMaxConcurrency) require.NoError(t, err) diff --git a/engine/execution/computation/manager_benchmark_test.go b/engine/execution/computation/manager_benchmark_test.go index 5026ba57f14..d5d55a50691 100644 --- a/engine/execution/computation/manager_benchmark_test.go +++ b/engine/execution/computation/manager_benchmark_test.go @@ -201,7 +201,7 @@ func benchmarkComputeBlock( me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), maxConcurrency) require.NoError(b, err) diff --git a/engine/execution/computation/manager_test.go b/engine/execution/computation/manager_test.go index 31f641ebd11..3dbfcb4e527 100644 --- a/engine/execution/computation/manager_test.go +++ b/engine/execution/computation/manager_test.go @@ -143,7 +143,7 @@ func TestComputeBlockWithStorage(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -836,7 +836,7 @@ func Test_EventEncodingFailsOnlyTxAndCarriesOn(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) diff --git a/engine/execution/computation/programs_test.go b/engine/execution/computation/programs_test.go index 092a4a6e8ea..8fe46e8ea6e 100644 --- a/engine/execution/computation/programs_test.go +++ b/engine/execution/computation/programs_test.go @@ -138,7 +138,7 @@ func TestPrograms_TestContractUpdates(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) @@ -250,7 +250,7 @@ func TestPrograms_TestBlockForks(t *testing.T) { me, prov, nil, - testutil.ProtocolStateFixture(), + testutil.ProtocolStateWithSourceFixture(nil), testMaxConcurrency) require.NoError(t, err) diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 96d47e93b79..4bddd7d1e94 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -630,11 +630,14 @@ func ComputationResultFixture(t *testing.T) *execution.ComputationResult { } } -// ProtocolStateFixture returns a protocol state that can be used -// by BlockComputer tests -func ProtocolStateFixture() protocol.State { +// ProtocolStateWithSourceFixture returns a protocol state that supports RandomSource() +// If input is nil, a random source fixture is generated. +func ProtocolStateWithSourceFixture(source []byte) protocol.State { + if source == nil { + source = unittest.SignatureFixture() + } snapshot := pmock.Snapshot{} - snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) + snapshot.On("RandomSource").Return(source, nil) state := pmock.State{} state.On("AtBlockID", mock.Anything).Return(&snapshot) return &state diff --git a/engine/verification/assigner/blockconsumer/consumer_test.go b/engine/verification/assigner/blockconsumer/consumer_test.go index e049ba7c662..67ea6773194 100644 --- a/engine/verification/assigner/blockconsumer/consumer_test.go +++ b/engine/verification/assigner/blockconsumer/consumer_test.go @@ -149,7 +149,8 @@ func withConsumer( root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, s.State, vertestutils.WithClusterCommittee(clusterCommittee)) + sources := unittest.RandomSourcesFixture(110) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) // makes sure that we generated a block chain of requested length. require.Len(t, blocks, blockCount) diff --git a/engine/verification/utils/unittest/fixture.go b/engine/verification/utils/unittest/fixture.go index 5b1a922098b..57c9916e62d 100644 --- a/engine/verification/utils/unittest/fixture.go +++ b/engine/verification/utils/unittest/fixture.go @@ -25,7 +25,6 @@ import ( "github.com/onflow/flow-go/module/epochs" "github.com/onflow/flow-go/module/signature" "github.com/onflow/flow-go/state/cluster" - "github.com/onflow/flow-go/state/protocol" envMock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/model/flow" @@ -195,7 +194,7 @@ func ExecutionResultFixture(t *testing.T, chain flow.Chain, refBlkHeader *flow.Header, clusterCommittee flow.IdentityList, - state protocol.ParticipantState, + source []byte, ) (*flow.ExecutionResult, *ExecutionReceiptData) { // setups up the first collection of block consists of three transactions @@ -301,7 +300,7 @@ func ExecutionResultFixture(t *testing.T, me, prov, nil, - state, + testutil.ProtocolStateWithSourceFixture(source), testMaxConcurrency) require.NoError(t, err) @@ -377,7 +376,7 @@ func ExecutionResultFixture(t *testing.T, func CompleteExecutionReceiptChainFixture(t *testing.T, root *flow.Header, count int, - state protocol.ParticipantState, + sources [][]byte, opts ...CompleteExecutionReceiptBuilderOpt, ) []*CompleteExecutionReceipt { completeERs := make([]*CompleteExecutionReceipt, 0, count) @@ -401,11 +400,14 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, require.GreaterOrEqual(t, len(builder.executorIDs), builder.executorCount, "number of executors in the tests should be greater than or equal to the number of receipts per block") + var sourcesIndex = 0 for i := 0; i < count; i++ { // Generates two blocks as parent <- R <- C where R is a reference block containing guarantees, // and C is a container block containing execution receipt for R. - receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder, state) - containerBlock := ContainerBlockFixture(head, receipts) + receipts, allData, head := ExecutionReceiptsFromParentBlockFixture(t, parent, builder, sources[sourcesIndex:]) + sourcesIndex += builder.resultsCount + containerBlock := ContainerBlockFixture(head, receipts, sources[sourcesIndex]) + sourcesIndex++ completeERs = append(completeERs, &CompleteExecutionReceipt{ ContainerBlock: containerBlock, Receipts: receipts, @@ -426,7 +428,7 @@ func CompleteExecutionReceiptChainFixture(t *testing.T, func ExecutionReceiptsFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder, - state protocol.ParticipantState) ( + sources [][]byte) ( []*flow.ExecutionReceipt, []*ExecutionReceiptData, *flow.Header) { @@ -434,7 +436,7 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, allReceipts := make([]*flow.ExecutionReceipt, 0, builder.resultsCount*builder.executorCount) for i := 0; i < builder.resultsCount; i++ { - result, data := ExecutionResultFromParentBlockFixture(t, parent, builder, state) + result, data := ExecutionResultFromParentBlockFixture(t, parent, builder, sources[i:]) // makes several copies of the same result for cp := 0; cp < builder.executorCount; cp++ { @@ -455,16 +457,19 @@ func ExecutionReceiptsFromParentBlockFixture(t *testing.T, func ExecutionResultFromParentBlockFixture(t *testing.T, parent *flow.Header, builder *CompleteExecutionReceiptBuilder, - state protocol.ParticipantState, + sources [][]byte, ) (*flow.ExecutionResult, *ExecutionReceiptData) { - refBlkHeader := unittest.BlockHeaderWithParentFixture(parent) - return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee, state) + // create the block header including a QC with source a index `i` + refBlkHeader := unittest.BlockHeaderWithParentWithSoRFixture(parent, sources[0]) + // execute the block with the source a index `i+1` (which will be included later in the child block) + return ExecutionResultFixture(t, builder.chunksCount, builder.chain, refBlkHeader, builder.clusterCommittee, sources[1]) } // ContainerBlockFixture builds and returns a block that contains input execution receipts. -func ContainerBlockFixture(parent *flow.Header, receipts []*flow.ExecutionReceipt) *flow.Block { +func ContainerBlockFixture(parent *flow.Header, receipts []*flow.ExecutionReceipt, source []byte) *flow.Block { // container block is the block that contains the execution receipt of reference block containerBlock := unittest.BlockWithParentFixture(parent) + containerBlock.Header.ParentVoterSigData = unittest.QCSigDataWithSoRFixture(source) containerBlock.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipts...))) return containerBlock diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index 6636e24fbc3..fee800b34e9 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -494,7 +494,11 @@ func withConsumers(t *testing.T, builder.clusterCommittee = participants.Filter(filter.HasRole(flow.RoleCollection)) }) - completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, s.State, ops...) + // random sources for all blocks: + // - root block (block[0]) is executed with sources[0] (included in QC of child block[1]) + // - block[i] is executed with sources[i] (included in QC of child block[i+1]) + sources := unittest.RandomSourcesFixture(30) + completeERs := CompleteExecutionReceiptChainFixture(t, root, blockCount, sources, ops...) blocks := ExtendStateWithFinalizedBlocks(t, completeERs, s.State) // chunk assignment diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 884bdd5abf7..03f4b0c58ec 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -470,6 +470,38 @@ func BlockHeaderWithParentFixture(parent *flow.Header) *flow.Header { } } +func BlockHeaderWithParentWithSoRFixture(parent *flow.Header, source []byte) *flow.Header { + height := parent.Height + 1 + view := parent.View + 1 + uint64(rand.Intn(10)) // Intn returns [0, n) + var lastViewTC *flow.TimeoutCertificate + if view != parent.View+1 { + newestQC := QuorumCertificateFixture(func(qc *flow.QuorumCertificate) { + qc.View = parent.View + }) + lastViewTC = &flow.TimeoutCertificate{ + View: view - 1, + NewestQCViews: []uint64{newestQC.View}, + NewestQC: newestQC, + SignerIndices: SignerIndicesFixture(4), + SigData: SignatureFixture(), + } + } + return &flow.Header{ + ChainID: parent.ChainID, + ParentID: parent.ID(), + Height: height, + PayloadHash: IdentifierFixture(), + Timestamp: time.Now().UTC(), + View: view, + ParentView: parent.View, + ParentVoterIndices: SignerIndicesFixture(4), + ParentVoterSigData: QCSigDataWithSoRFixture(source), + ProposerID: IdentifierFixture(), + ProposerSigData: SignatureFixture(), + LastViewTC: lastViewTC, + } +} + func ClusterPayloadFixture(n int) *cluster.Payload { transactions := make([]*flow.TransactionBody, n) for i := 0; i < n; i++ { @@ -1270,8 +1302,7 @@ func ChunkStatusListFixture( return statuses } -func QCSigDataFixture() []byte { - packer := hotstuff.SigDataPacker{} +func qcSignatureDataFixture() hotstuff.SignatureData { sigType := RandomBytes(5) for i := range sigType { sigType[i] = sigType[i] % 2 @@ -1282,6 +1313,20 @@ func QCSigDataFixture() []byte { AggregatedRandomBeaconSig: SignatureFixture(), ReconstructedRandomBeaconSig: SignatureFixture(), } + return sigData +} + +func QCSigDataFixture() []byte { + packer := hotstuff.SigDataPacker{} + sigData := qcSignatureDataFixture() + encoded, _ := packer.Encode(&sigData) + return encoded +} + +func QCSigDataWithSoRFixture(sor []byte) []byte { + packer := hotstuff.SigDataPacker{} + sigData := qcSignatureDataFixture() + sigData.ReconstructedRandomBeaconSig = sor encoded, _ := packer.Encode(&sigData) return encoded } @@ -1300,6 +1345,14 @@ func SignaturesFixture(n int) []crypto.Signature { return sigs } +func RandomSourcesFixture(n int) [][]byte { + var sigs [][]byte + for i := 0; i < n; i++ { + sigs = append(sigs, SignatureFixture()) + } + return sigs +} + func TransactionFixture(n ...func(t *flow.Transaction)) flow.Transaction { tx := flow.Transaction{TransactionBody: TransactionBodyFixture()} if len(n) > 0 { From 3ae7c26402238dd9a80e99b87627729c6b2d1cc3 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 7 Jul 2023 13:07:00 +0300 Subject: [PATCH 097/169] Configured state streaming on a separate port --- .../node_builder/access_node_builder.go | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index c6d2b3d258d..5c4b20d913b 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -243,8 +243,9 @@ type FlowAccessNodeBuilder struct { StateStreamEng *state_stream.Engine // grpc servers - secureGrpcServer *grpcserver.GrpcServer - unsecureGrpcServer *grpcserver.GrpcServer + secureGrpcServer *grpcserver.GrpcServer + unsecureGrpcServer *grpcserver.GrpcServer + stateStreamGrpcServer *grpcserver.GrpcServer } func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { @@ -617,7 +618,7 @@ func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessN node.RootChainID, builder.executionDataConfig.InitialBlockHeight, highestAvailableHeight, - builder.unsecureGrpcServer, + builder.stateStreamGrpcServer, ) if err != nil { return nil, fmt.Errorf("could not create state stream engine: %w", err) @@ -1006,6 +1007,17 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.apiRatelimits, builder.apiBurstlimits).Build() + if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { + builder.stateStreamGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, + builder.stateStreamConf.ListenAddr, + builder.stateStreamConf.MaxExecutionDataMsgSize, + builder.rpcMetricsEnabled, + builder.apiRatelimits, + builder.apiBurstlimits).Build() + } else { + builder.stateStreamGrpcServer = builder.unsecureGrpcServer + } + return nil }). Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { @@ -1127,6 +1139,12 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return builder.unsecureGrpcServer, nil }) + if builder.stateStreamGrpcServer != builder.unsecureGrpcServer { + builder.Component("state stream unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + return builder.stateStreamGrpcServer, nil + }) + } + builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { ping, err := pingeng.New( node.Logger, From 1e758003cf88215662f02e05502453542405befc Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 7 Jul 2023 12:32:45 -0400 Subject: [PATCH 098/169] use MakeIDFromFingerPrint --- network/p2p/tracer/internal/cache_test.go | 14 +++++++------- network/p2p/tracer/internal/rpc_sent_tracker.go | 3 +-- .../p2p/tracer/internal/rpc_sent_tracker_test.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go index 2d4017c54e9..fc043ee116f 100644 --- a/network/p2p/tracer/internal/cache_test.go +++ b/network/p2p/tracer/internal/cache_test.go @@ -13,7 +13,7 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" - "github.com/onflow/flow-go/network/p2p" + p2pmsg "github.com/onflow/flow-go/network/p2p/message" "github.com/onflow/flow-go/utils/unittest" ) @@ -22,7 +22,7 @@ import ( // and false when an existing record is initialized. func TestCache_Init(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave id1 := unittest.IdentifierFixture() id2 := unittest.IdentifierFixture() @@ -48,7 +48,7 @@ func TestCache_Init(t *testing.T) { // 2. Ensuring that all records are correctly initialized. func TestCache_ConcurrentInit(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave ids := unittest.IdentifierListFixture(10) var wg sync.WaitGroup @@ -76,7 +76,7 @@ func TestCache_ConcurrentInit(t *testing.T) { // 3. The record is correctly initialized in the cache and can be retrieved using the Get method. func TestCache_ConcurrentSameRecordInit(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave id := unittest.IdentifierFixture() const concurrentAttempts = 10 @@ -112,7 +112,7 @@ func TestCache_ConcurrentSameRecordInit(t *testing.T) { // 4. Attempting to remove a non-existent ID. func TestCache_Remove(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave // initialize spam records for a few ids id1 := unittest.IdentifierFixture() id2 := unittest.IdentifierFixture() @@ -143,7 +143,7 @@ func TestCache_Remove(t *testing.T) { // 2. The records are correctly removed from the cache. func TestCache_ConcurrentRemove(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave ids := unittest.IdentifierListFixture(10) for _, id := range ids { cache.init(id, controlMsgType) @@ -173,7 +173,7 @@ func TestCache_ConcurrentRemove(t *testing.T) { // 4. The removed records are correctly removed from the cache. func TestRecordCache_ConcurrentInitAndRemove(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2p.CtrlMsgIHave + controlMsgType := p2pmsg.CtrlMsgIHave ids := unittest.IdentifierListFixture(20) idsToAdd := ids[:10] idsToRemove := ids[10:] diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index f580c443f41..e2db526d549 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -59,6 +59,5 @@ func (t *RPCSentTracker) WasIHaveRPCSent(topicID, messageID string) bool { // Each iHave RPC control message contains a single topicId and multiple messageIds, to ensure we // produce a unique id for each message we append the messageId to the topicId. func iHaveRPCSentEntityID(topicId, messageId string) flow.Identifier { - b := []byte(fmt.Sprintf("%s%s", topicId, messageId)) - return flow.HashToID(b) + return flow.MakeIDFromFingerPrint([]byte(fmt.Sprintf("%s%s", topicId, messageId))) } diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go index f0a0dda5195..09d54ea6f04 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker_test.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -36,7 +36,7 @@ func TestRPCSentTracker_IHave(t *testing.T) { MessageIDs: []string{messageID}, }} rpc := rpcFixture(withIhaves(iHaves)) - tracker.OnIHaveRPCSent(rpc) + tracker.OnIHaveRPCSent(rpc.GetControl().GetIhave()) require.True(t, tracker.WasIHaveRPCSent(topicID, messageID)) }) } From c7119f9d482369d0bd39517ad775e247294ff9c3 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 7 Jul 2023 12:40:59 -0400 Subject: [PATCH 099/169] update WasIHaveRPCSent test ensure false positive not returned --- network/p2p/tracer/internal/rpc_sent_tracker_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go index 09d54ea6f04..5ebfc2da1d1 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker_test.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -30,14 +30,19 @@ func TestRPCSentTracker_IHave(t *testing.T) { t.Run("WasIHaveRPCSent should return true for iHave message after it is tracked with OnIHaveRPCSent", func(t *testing.T) { topicID := channels.PushBlocks.String() - messageID := unittest.IdentifierFixture().String() + messageID1 := unittest.IdentifierFixture().String() iHaves := []*pb.ControlIHave{{ TopicID: &topicID, - MessageIDs: []string{messageID}, + MessageIDs: []string{messageID1}, }} rpc := rpcFixture(withIhaves(iHaves)) tracker.OnIHaveRPCSent(rpc.GetControl().GetIhave()) - require.True(t, tracker.WasIHaveRPCSent(topicID, messageID)) + require.True(t, tracker.WasIHaveRPCSent(topicID, messageID1)) + + // manipulate last byte of message ID ensure false positive not returned + messageID2 := []byte(messageID1) + messageID2[len(messageID2)-1] = 'X' + require.False(t, tracker.WasIHaveRPCSent(topicID, string(messageID2))) }) } From 33681ece86ea7f40ba696640283f30c7ab03434c Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Fri, 7 Jul 2023 12:42:08 -0400 Subject: [PATCH 100/169] Update rpc_sent_tracker.go --- network/p2p/tracer/internal/rpc_sent_tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index e2db526d549..66791217c57 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -3,9 +3,9 @@ package internal import ( "fmt" + pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/rs/zerolog" - pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" p2pmsg "github.com/onflow/flow-go/network/p2p/message" From eef05fdd29ec36536a08f7c72fe6b6ed073724fc Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 10:51:40 -0600 Subject: [PATCH 101/169] fix test bug: missing parameter --- module/jobqueue/finalized_block_reader_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/jobqueue/finalized_block_reader_test.go b/module/jobqueue/finalized_block_reader_test.go index 679a63e6f2f..8349828d272 100644 --- a/module/jobqueue/finalized_block_reader_test.go +++ b/module/jobqueue/finalized_block_reader_test.go @@ -65,7 +65,8 @@ func withReader( root, err := s.State.Params().FinalizedRoot() require.NoError(t, err) clusterCommittee := participants.Filter(filter.HasRole(flow.RoleCollection)) - results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, vertestutils.WithClusterCommittee(clusterCommittee)) + sources := unittest.RandomSourcesFixture(10) + results := vertestutils.CompleteExecutionReceiptChainFixture(t, root, blockCount/2, sources, vertestutils.WithClusterCommittee(clusterCommittee)) blocks := vertestutils.ExtendStateWithFinalizedBlocks(t, results, s.State) withBlockReader(reader, blocks) From 86464d70cb86db9f99d8dd1d52887cb2a54ee018 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 12:33:54 -0600 Subject: [PATCH 102/169] clean up and rename unsafeRandomGenerator to RandomGenerator --- engine/execution/testutil/fixtures.go | 20 +++++++++---- fvm/environment/facade_env.go | 13 ++++----- ...andom_generator.go => random_generator.go} | 14 +++++----- ...andom_generator.go => random_generator.go} | 28 +++++++++---------- ...rator_test.go => random_generator_test.go} | 5 ++-- fvm/fvm_blockcontext_test.go | 6 ++-- module/chunks/chunk_assigner_test.go | 4 +-- state/protocol/badger/snapshot.go | 2 +- 8 files changed, 48 insertions(+), 44 deletions(-) rename fvm/environment/mock/{unsafe_random_generator.go => random_generator.go} (52%) rename fvm/environment/{unsafe_random_generator.go => random_generator.go} (86%) rename fvm/environment/{unsafe_random_generator_test.go => random_generator_test.go} (93%) diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 4bddd7d1e94..21717f8da61 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -26,7 +26,7 @@ import ( "github.com/onflow/flow-go/module/epochs" "github.com/onflow/flow-go/module/executiondatasync/execution_data" "github.com/onflow/flow-go/state/protocol" - pmock "github.com/onflow/flow-go/state/protocol/mock" + protocolMock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -630,15 +630,25 @@ func ComputationResultFixture(t *testing.T) *execution.ComputationResult { } } -// ProtocolStateWithSourceFixture returns a protocol state that supports RandomSource() +// ProtocolSnapshotWithSourceFixture returns a protocol state snapshot mock that only +// supports RandomSource(). // If input is nil, a random source fixture is generated. -func ProtocolStateWithSourceFixture(source []byte) protocol.State { +func ProtocolSnapshotWithSourceFixture(source []byte) protocol.Snapshot { if source == nil { source = unittest.SignatureFixture() } - snapshot := pmock.Snapshot{} + snapshot := protocolMock.Snapshot{} snapshot.On("RandomSource").Return(source, nil) - state := pmock.State{} + return &snapshot +} + +// ProtocolStateWithSourceFixture returns a protocol state mock that only +// supports AtBlockID to return a snapshot mock. +// The snapshot mock only supports RandomSource(). +// If input is nil, a random source fixture is generated. +func ProtocolStateWithSourceFixture(source []byte) protocol.State { + snapshot := ProtocolSnapshotWithSourceFixture(source) + state := protocolMock.State{} state.On("AtBlockID", mock.Anything).Return(&snapshot) return &state } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index a25a71a69f6..04ff8e5a5ea 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -24,7 +24,7 @@ type facadeEnvironment struct { *ProgramLogger EventEmitter - UnsafeRandomGenerator + RandomGenerator CryptoLibrary BlockInfo @@ -170,11 +170,8 @@ func NewScriptEnv( params, txnState, NewCancellableMeter(ctx, txnState)) - - env.UnsafeRandomGenerator = NewDummyRandomGenerator() - + env.RandomGenerator = NewDummyRandomGenerator() env.addParseRestrictedChecks() - return env } @@ -229,7 +226,7 @@ func NewTransactionEnvironment( txnState, env) - env.UnsafeRandomGenerator = NewUnsafeRandomGenerator( + env.RandomGenerator = NewUnsafeRandomGenerator( tracer, params.Snapshot, params.TxId, @@ -275,9 +272,9 @@ func (env *facadeEnvironment) addParseRestrictedChecks() { env.TransactionInfo = NewParseRestrictedTransactionInfo( env.txnState, env.TransactionInfo) - env.UnsafeRandomGenerator = NewParseRestrictedUnsafeRandomGenerator( + env.RandomGenerator = NewParseRestrictedUnsafeRandomGenerator( env.txnState, - env.UnsafeRandomGenerator) + env.RandomGenerator) env.UUIDGenerator = NewParseRestrictedUUIDGenerator( env.txnState, env.UUIDGenerator) diff --git a/fvm/environment/mock/unsafe_random_generator.go b/fvm/environment/mock/random_generator.go similarity index 52% rename from fvm/environment/mock/unsafe_random_generator.go rename to fvm/environment/mock/random_generator.go index c92560981dd..0d0f1cf00e4 100644 --- a/fvm/environment/mock/unsafe_random_generator.go +++ b/fvm/environment/mock/random_generator.go @@ -4,13 +4,13 @@ package mock import mock "github.com/stretchr/testify/mock" -// UnsafeRandomGenerator is an autogenerated mock type for the UnsafeRandomGenerator type -type UnsafeRandomGenerator struct { +// RandomGenerator is an autogenerated mock type for the RandomGenerator type +type RandomGenerator struct { mock.Mock } // UnsafeRandom provides a mock function with given fields: -func (_m *UnsafeRandomGenerator) UnsafeRandom() (uint64, error) { +func (_m *RandomGenerator) UnsafeRandom() (uint64, error) { ret := _m.Called() var r0 uint64 @@ -33,14 +33,14 @@ func (_m *UnsafeRandomGenerator) UnsafeRandom() (uint64, error) { return r0, r1 } -type mockConstructorTestingTNewUnsafeRandomGenerator interface { +type mockConstructorTestingTNewRandomGenerator interface { mock.TestingT Cleanup(func()) } -// NewUnsafeRandomGenerator creates a new instance of UnsafeRandomGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewUnsafeRandomGenerator(t mockConstructorTestingTNewUnsafeRandomGenerator) *UnsafeRandomGenerator { - mock := &UnsafeRandomGenerator{} +// NewRandomGenerator creates a new instance of RandomGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewRandomGenerator(t mockConstructorTestingTNewRandomGenerator) *RandomGenerator { + mock := &RandomGenerator{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/fvm/environment/unsafe_random_generator.go b/fvm/environment/random_generator.go similarity index 86% rename from fvm/environment/unsafe_random_generator.go rename to fvm/environment/random_generator.go index 404cecaea44..435e4644b53 100644 --- a/fvm/environment/unsafe_random_generator.go +++ b/fvm/environment/random_generator.go @@ -15,14 +15,14 @@ import ( "github.com/onflow/flow-go/state/protocol/prg" ) -type UnsafeRandomGenerator interface { +type RandomGenerator interface { // UnsafeRandom returns a random uint64 UnsafeRandom() (uint64, error) } -var _ UnsafeRandomGenerator = (*unsafeRandomGenerator)(nil) +var _ RandomGenerator = (*unsafeRandomGenerator)(nil) -// unsafeRandomGenerator implements UnsafeRandomGenerator and is used +// unsafeRandomGenerator implements RandomGenerator and is used // for the transactions execution environment type unsafeRandomGenerator struct { tracer tracing.TracerSpan @@ -37,13 +37,13 @@ type unsafeRandomGenerator struct { type ParseRestrictedUnsafeRandomGenerator struct { txnState state.NestedTransactionPreparer - impl UnsafeRandomGenerator + impl RandomGenerator } func NewParseRestrictedUnsafeRandomGenerator( txnState state.NestedTransactionPreparer, - impl UnsafeRandomGenerator, -) UnsafeRandomGenerator { + impl RandomGenerator, +) RandomGenerator { return ParseRestrictedUnsafeRandomGenerator{ txnState: txnState, impl: impl, @@ -64,7 +64,7 @@ func NewUnsafeRandomGenerator( tracer tracing.TracerSpan, stateSnapshot protocol.Snapshot, txId flow.Identifier, -) UnsafeRandomGenerator { +) RandomGenerator { gen := &unsafeRandomGenerator{ tracer: tracer, stateSnapshot: stateSnapshot, @@ -78,10 +78,10 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( random.Rand, error, ) { - // Use the protocol state source of randomness [SoR] for the current block + // Use the protocol state source of randomness [SoR] for the current block's // execution source, err := gen.stateSnapshot.RandomSource() - // expected errors of RandomSource() are : + // expected errors of RandomSource() are: // - storage.ErrNotFound if the QC is unknown. // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown // at this stage, snapshot reference block should be known and the QC should also be known, @@ -105,7 +105,7 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( } // maybeCreateRandomGenerator seeds the pseudo-random number generator using the -// block header ID and transaction index as an entropy source. The seed +// block SoR as an entropy source, customized with the transaction hash. The seed // function is currently called for each transaction, the PRG is used to // provide all the randoms the transaction needs through UnsafeRandom. // @@ -124,7 +124,7 @@ func (gen *unsafeRandomGenerator) maybeCreateRandomGenerator() error { // using a crypto-secure one). This is not thread safe, due to the gen.prg // instance currently used. Its also not thread safe because each thread needs // to be deterministically seeded with a different seed. This is Ok because a -// single transaction has a single UnsafeRandomGenerator and is run in a single +// single transaction has a single RandomGenerator and is run in a single // thread. func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { defer gen.tracer.StartExtensiveTracingChildSpan( @@ -141,13 +141,13 @@ func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { return binary.LittleEndian.Uint64(buf), nil } -var _ UnsafeRandomGenerator = (*dummyRandomGenerator)(nil) +var _ RandomGenerator = (*dummyRandomGenerator)(nil) -// dummyRandomGenerator implements UnsafeRandomGenerator and is used +// dummyRandomGenerator implements RandomGenerator and is used // for the scripts execution environment type dummyRandomGenerator struct{} -func NewDummyRandomGenerator() UnsafeRandomGenerator { +func NewDummyRandomGenerator() RandomGenerator { return &dummyRandomGenerator{} } diff --git a/fvm/environment/unsafe_random_generator_test.go b/fvm/environment/random_generator_test.go similarity index 93% rename from fvm/environment/unsafe_random_generator_test.go rename to fvm/environment/random_generator_test.go index 83089129d07..2754a5bb931 100644 --- a/fvm/environment/unsafe_random_generator_test.go +++ b/fvm/environment/random_generator_test.go @@ -8,17 +8,16 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto/random" + "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" - pmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) func TestUnsafeRandomGenerator(t *testing.T) { // protocol snapshot mock - snapshot := &pmock.Snapshot{} - snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) + snapshot := testutil.ProtocolSnapshotWithSourceFixture(nil) getRandoms := func(txId flow.Identifier, N int) []uint64 { // seed the RG with the same block header diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index 2dac41abc48..51e3c8f3801 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -25,7 +25,6 @@ import ( errors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/model/flow" - pmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/utils/unittest" ) @@ -1672,13 +1671,12 @@ func TestBlockContext_UnsafeRandom(t *testing.T) { chain, vm := createChainAndVm(flow.Mainnet) header := &flow.Header{Height: 42} - snapshot := pmock.Snapshot{} - snapshot.On("RandomSource").Return(unittest.RandomBytes(48), nil) + snapshot := testutil.ProtocolSnapshotWithSourceFixture(nil) ctx := fvm.NewContext( fvm.WithChain(chain), fvm.WithBlockHeader(header), - fvm.WithProtocolSnapshot(&snapshot), + fvm.WithProtocolSnapshot(snapshot), fvm.WithCadenceLogging(true), ) diff --git a/module/chunks/chunk_assigner_test.go b/module/chunks/chunk_assigner_test.go index 21586afad89..339275752a3 100644 --- a/module/chunks/chunk_assigner_test.go +++ b/module/chunks/chunk_assigner_test.go @@ -10,7 +10,7 @@ import ( chmodels "github.com/onflow/flow-go/model/chunks" "github.com/onflow/flow-go/model/flow" - protocolMock "github.com/onflow/flow-go/state/protocol/mock" + protocol "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/state/protocol/prg" "github.com/onflow/flow-go/utils/unittest" ) @@ -21,7 +21,7 @@ type PublicAssignmentTestSuite struct { } // Setup test with n verification nodes -func (a *PublicAssignmentTestSuite) SetupTest(n int) (*flow.Header, *protocolMock.Snapshot, *protocolMock.State) { +func (a *PublicAssignmentTestSuite) SetupTest(n int) (*flow.Header, *protocol.Snapshot, *protocol.State) { nodes := make([]flow.Role, 0) for i := 1; i < n; i++ { nodes = append(nodes, flow.RoleVerification) diff --git a/state/protocol/badger/snapshot.go b/state/protocol/badger/snapshot.go index 33522480301..1a121e81748 100644 --- a/state/protocol/badger/snapshot.go +++ b/state/protocol/badger/snapshot.go @@ -377,7 +377,7 @@ func (s *Snapshot) descendants(blockID flow.Identifier) ([]flow.Identifier, erro return descendantIDs, nil } -// RandomSource returns the seed for the current block snapshot. +// RandomSource returns the seed for the current block's snapshot. // Expected error returns: // * storage.ErrNotFound is returned if the QC is unknown. func (s *Snapshot) RandomSource() ([]byte, error) { From e70640e2567d99ea0aee85211a651d0cd3d0795d Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 14:00:33 -0600 Subject: [PATCH 103/169] more renaming of unsafeRandom to random --- fvm/environment/facade_env.go | 4 +-- fvm/environment/mock/random_generator.go | 4 +-- fvm/environment/random_generator.go | 37 ++++++++++++------------ fvm/environment/random_generator_test.go | 10 +++---- fvm/fvm_blockcontext_test.go | 2 +- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 04ff8e5a5ea..7c73dca0854 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -226,7 +226,7 @@ func NewTransactionEnvironment( txnState, env) - env.RandomGenerator = NewUnsafeRandomGenerator( + env.RandomGenerator = NewRandomGenerator( tracer, params.Snapshot, params.TxId, @@ -272,7 +272,7 @@ func (env *facadeEnvironment) addParseRestrictedChecks() { env.TransactionInfo = NewParseRestrictedTransactionInfo( env.txnState, env.TransactionInfo) - env.RandomGenerator = NewParseRestrictedUnsafeRandomGenerator( + env.RandomGenerator = NewParseRestrictedRandomGenerator( env.txnState, env.RandomGenerator) env.UUIDGenerator = NewParseRestrictedUUIDGenerator( diff --git a/fvm/environment/mock/random_generator.go b/fvm/environment/mock/random_generator.go index 0d0f1cf00e4..de838f89979 100644 --- a/fvm/environment/mock/random_generator.go +++ b/fvm/environment/mock/random_generator.go @@ -9,8 +9,8 @@ type RandomGenerator struct { mock.Mock } -// UnsafeRandom provides a mock function with given fields: -func (_m *RandomGenerator) UnsafeRandom() (uint64, error) { +// Random provides a mock function with given fields: +func (_m *RandomGenerator) Random() (uint64, error) { ret := _m.Called() var r0 uint64 diff --git a/fvm/environment/random_generator.go b/fvm/environment/random_generator.go index 435e4644b53..5492a4aeed4 100644 --- a/fvm/environment/random_generator.go +++ b/fvm/environment/random_generator.go @@ -17,14 +17,15 @@ import ( type RandomGenerator interface { // UnsafeRandom returns a random uint64 + // Todo: rename to Random() once Cadence interface is updated UnsafeRandom() (uint64, error) } -var _ RandomGenerator = (*unsafeRandomGenerator)(nil) +var _ RandomGenerator = (*randomGenerator)(nil) -// unsafeRandomGenerator implements RandomGenerator and is used +// randomGenerator implements RandomGenerator and is used // for the transactions execution environment -type unsafeRandomGenerator struct { +type randomGenerator struct { tracer tracing.TracerSpan stateSnapshot protocol.Snapshot @@ -35,37 +36,37 @@ type unsafeRandomGenerator struct { createErr error } -type ParseRestrictedUnsafeRandomGenerator struct { +type ParseRestrictedRandomGenerator struct { txnState state.NestedTransactionPreparer impl RandomGenerator } -func NewParseRestrictedUnsafeRandomGenerator( +func NewParseRestrictedRandomGenerator( txnState state.NestedTransactionPreparer, impl RandomGenerator, ) RandomGenerator { - return ParseRestrictedUnsafeRandomGenerator{ + return ParseRestrictedRandomGenerator{ txnState: txnState, impl: impl, } } -func (gen ParseRestrictedUnsafeRandomGenerator) UnsafeRandom() ( +func (gen ParseRestrictedRandomGenerator) UnsafeRandom() ( uint64, error, ) { return parseRestrict1Ret( gen.txnState, - trace.FVMEnvUnsafeRandom, + trace.FVMEnvRandom, gen.impl.UnsafeRandom) } -func NewUnsafeRandomGenerator( +func NewRandomGenerator( tracer tracing.TracerSpan, stateSnapshot protocol.Snapshot, txId flow.Identifier, ) RandomGenerator { - gen := &unsafeRandomGenerator{ + gen := &randomGenerator{ tracer: tracer, stateSnapshot: stateSnapshot, txId: txId, @@ -74,7 +75,7 @@ func NewUnsafeRandomGenerator( return gen } -func (gen *unsafeRandomGenerator) createRandomGenerator() ( +func (gen *randomGenerator) createRandomGenerator() ( random.Rand, error, ) { @@ -107,12 +108,12 @@ func (gen *unsafeRandomGenerator) createRandomGenerator() ( // maybeCreateRandomGenerator seeds the pseudo-random number generator using the // block SoR as an entropy source, customized with the transaction hash. The seed // function is currently called for each transaction, the PRG is used to -// provide all the randoms the transaction needs through UnsafeRandom. +// provide all the randoms the transaction needs through Random. // // This allows lazy seeding of the random number generator, since not a lot of // transactions/scripts use it and the time it takes to seed it is not // negligible. -func (gen *unsafeRandomGenerator) maybeCreateRandomGenerator() error { +func (gen *randomGenerator) maybeCreateRandomGenerator() error { gen.createOnce.Do(func() { gen.prg, gen.createErr = gen.createRandomGenerator() }) @@ -120,15 +121,15 @@ func (gen *unsafeRandomGenerator) maybeCreateRandomGenerator() error { return gen.createErr } -// UnsafeRandom returns a random uint64 using the underlying PRG (currently +// Random returns a random uint64 using the underlying PRG (currently // using a crypto-secure one). This is not thread safe, due to the gen.prg // instance currently used. Its also not thread safe because each thread needs // to be deterministically seeded with a different seed. This is Ok because a // single transaction has a single RandomGenerator and is run in a single // thread. -func (gen *unsafeRandomGenerator) UnsafeRandom() (uint64, error) { +func (gen *randomGenerator) UnsafeRandom() (uint64, error) { defer gen.tracer.StartExtensiveTracingChildSpan( - trace.FVMEnvUnsafeRandom).End() + trace.FVMEnvRandom).End() // The internal seeding is only done once. err := gen.maybeCreateRandomGenerator() @@ -151,8 +152,8 @@ func NewDummyRandomGenerator() RandomGenerator { return &dummyRandomGenerator{} } -// UnsafeRandom() returns an error because executing scripts +// Random() returns an error because executing scripts // does not support randomness APIs. func (gen *dummyRandomGenerator) UnsafeRandom() (uint64, error) { - return 0, errors.NewOperationNotSupportedError("UnsafeRandom") + return 0, errors.NewOperationNotSupportedError("Random") } diff --git a/fvm/environment/random_generator_test.go b/fvm/environment/random_generator_test.go index 2754a5bb931..60f413b7a36 100644 --- a/fvm/environment/random_generator_test.go +++ b/fvm/environment/random_generator_test.go @@ -15,13 +15,13 @@ import ( "github.com/onflow/flow-go/utils/unittest" ) -func TestUnsafeRandomGenerator(t *testing.T) { +func TestRandomGenerator(t *testing.T) { // protocol snapshot mock snapshot := testutil.ProtocolSnapshotWithSourceFixture(nil) getRandoms := func(txId flow.Identifier, N int) []uint64 { // seed the RG with the same block header - urg := environment.NewUnsafeRandomGenerator( + urg := environment.NewRandomGenerator( tracing.NewTracerSpan(), snapshot, txId) @@ -39,7 +39,7 @@ func TestUnsafeRandomGenerator(t *testing.T) { t.Run("randomness test", func(t *testing.T) { for i := 0; i < 10; i++ { txId := unittest.TransactionFixture().ID() - urg := environment.NewUnsafeRandomGenerator( + urg := environment.NewRandomGenerator( tracing.NewTracerSpan(), snapshot, txId) @@ -52,8 +52,8 @@ func TestUnsafeRandomGenerator(t *testing.T) { } }) - // tests that unsafeRandom is PRG based and hence has deterministic outputs. - t.Run("PRG-based UnsafeRandom", func(t *testing.T) { + // tests that has deterministic outputs. + t.Run("PRG-based Random", func(t *testing.T) { for i := 0; i < 10; i++ { txId := unittest.TransactionFixture().ID() N := 100 diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index 51e3c8f3801..d3ef4cb4100 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -1664,7 +1664,7 @@ func TestBlockContext_GetAccount(t *testing.T) { }) } -func TestBlockContext_UnsafeRandom(t *testing.T) { +func TestBlockContext_Random(t *testing.T) { t.Parallel() From f9993990005c0aa5f1456f6e17bf9fb7679021c8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 14:12:48 -0600 Subject: [PATCH 104/169] refactor environment/randomGenerator to use simpler do.once pattern --- fvm/environment/random_generator.go | 49 ++++++++++------------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/fvm/environment/random_generator.go b/fvm/environment/random_generator.go index 5492a4aeed4..030d7381c52 100644 --- a/fvm/environment/random_generator.go +++ b/fvm/environment/random_generator.go @@ -3,7 +3,6 @@ package environment import ( "encoding/binary" "fmt" - "sync" "github.com/onflow/flow-go/crypto/random" "github.com/onflow/flow-go/fvm/errors" @@ -31,9 +30,8 @@ type randomGenerator struct { stateSnapshot protocol.Snapshot txId flow.Identifier - prg random.Rand - createOnce sync.Once - createErr error + prg random.Rand + isPRGCreated bool } type ParseRestrictedRandomGenerator struct { @@ -70,15 +68,13 @@ func NewRandomGenerator( tracer: tracer, stateSnapshot: stateSnapshot, txId: txId, + isPRGCreated: false, // PRG is not created } return gen } -func (gen *randomGenerator) createRandomGenerator() ( - random.Rand, - error, -) { +func (gen *randomGenerator) createPRG() (random.Rand, error) { // Use the protocol state source of randomness [SoR] for the current block's // execution source, err := gen.stateSnapshot.RandomSource() @@ -105,36 +101,23 @@ func (gen *randomGenerator) createRandomGenerator() ( return csprg, nil } -// maybeCreateRandomGenerator seeds the pseudo-random number generator using the -// block SoR as an entropy source, customized with the transaction hash. The seed -// function is currently called for each transaction, the PRG is used to -// provide all the randoms the transaction needs through Random. -// -// This allows lazy seeding of the random number generator, since not a lot of -// transactions/scripts use it and the time it takes to seed it is not -// negligible. -func (gen *randomGenerator) maybeCreateRandomGenerator() error { - gen.createOnce.Do(func() { - gen.prg, gen.createErr = gen.createRandomGenerator() - }) - - return gen.createErr -} - -// Random returns a random uint64 using the underlying PRG (currently -// using a crypto-secure one). This is not thread safe, due to the gen.prg -// instance currently used. Its also not thread safe because each thread needs -// to be deterministically seeded with a different seed. This is Ok because a +// UnsafeRandom returns a random uint64 using the underlying PRG (currently +// using a crypto-secure one). This function is not thread safe, due to the gen.prg +// instance currently used. This is fine because a // single transaction has a single RandomGenerator and is run in a single // thread. func (gen *randomGenerator) UnsafeRandom() (uint64, error) { defer gen.tracer.StartExtensiveTracingChildSpan( trace.FVMEnvRandom).End() - // The internal seeding is only done once. - err := gen.maybeCreateRandomGenerator() - if err != nil { - return 0, err + // PRG creation is only done once. + if !gen.isPRGCreated { + newPRG, err := gen.createPRG() + if err != nil { + return 0, err + } + gen.prg = newPRG + gen.isPRGCreated = true } buf := make([]byte, 8) @@ -152,7 +135,7 @@ func NewDummyRandomGenerator() RandomGenerator { return &dummyRandomGenerator{} } -// Random() returns an error because executing scripts +// UnsafeRandom() returns an error because executing scripts // does not support randomness APIs. func (gen *dummyRandomGenerator) UnsafeRandom() (uint64, error) { return 0, errors.NewOperationNotSupportedError("Random") From 63e2b6d9d6df6edcaa5a24832d5dc2d179f6a901 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 17:36:44 -0600 Subject: [PATCH 105/169] rename VFMEnvRandom constant --- module/trace/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/trace/constants.go b/module/trace/constants.go index 76db4374abb..5dada818038 100644 --- a/module/trace/constants.go +++ b/module/trace/constants.go @@ -178,7 +178,7 @@ const ( FVMEnvBLSAggregatePublicKeys SpanName = "fvm.env.blsAggregatePublicKeys" FVMEnvGetCurrentBlockHeight SpanName = "fvm.env.getCurrentBlockHeight" FVMEnvGetBlockAtHeight SpanName = "fvm.env.getBlockAtHeight" - FVMEnvUnsafeRandom SpanName = "fvm.env.unsafeRandom" + FVMEnvRandom SpanName = "fvm.env.unsafeRandom" FVMEnvCreateAccount SpanName = "fvm.env.createAccount" FVMEnvAddAccountKey SpanName = "fvm.env.addAccountKey" FVMEnvAddEncodedAccountKey SpanName = "fvm.env.addEncodedAccountKey" From 54a68032043aaa597ec8bdf43cc25c549d9609d8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 17:43:14 -0600 Subject: [PATCH 106/169] add tmate for remote debugging --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5772ef5dcb3..fc57f51ee31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,8 @@ jobs: with: go-version: ${{ env.GO_VERSION }} cache: true + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 - name: Run tidy run: make tidy - name: code sanity check From 3a3b8ba3e810047cbb22fcb69d65e08ddfad91b5 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 21:07:05 -0600 Subject: [PATCH 107/169] fix merging bug --- network/p2p/connection/connector_factory.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/network/p2p/connection/connector_factory.go b/network/p2p/connection/connector_factory.go index 0960f026e7b..4547d1cb301 100644 --- a/network/p2p/connection/connector_factory.go +++ b/network/p2p/connection/connector_factory.go @@ -38,18 +38,12 @@ const ( // DefaultLibp2pBackoffConnectorFactory is a factory function to create a new BackoffConnector. It uses the default // values for the backoff connector. // (https://github.com/libp2p/go-libp2p-pubsub/blob/master/discovery.go#L34) -<<<<<<< HEAD -func DefaultLibp2pBackoffConnectorFactory(host host.Host) func() (*discoveryBackoff.BackoffConnector, error) { - return func() (*discoveryBackoff.BackoffConnector, error) { +func DefaultLibp2pBackoffConnectorFactory() p2p.ConnectorFactory { + return func(host host.Host) (p2p.Connector, error) { rngSrc, err := newSource() if err != nil { return nil, fmt.Errorf("failed to generate a random source: %w", err) } -======= -func DefaultLibp2pBackoffConnectorFactory() p2p.ConnectorFactory { - return func(host host.Host) (p2p.Connector, error) { - rngSrc := rand.NewSource(rand.Int63()) ->>>>>>> master cacheSize := 100 dialTimeout := time.Minute * 2 From c7ed3c3fc9f36569461470452cf7715eb3832045 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Jul 2023 21:11:22 -0600 Subject: [PATCH 108/169] add random generator mock --- fvm/environment/mock/random_generator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fvm/environment/mock/random_generator.go b/fvm/environment/mock/random_generator.go index de838f89979..0d0f1cf00e4 100644 --- a/fvm/environment/mock/random_generator.go +++ b/fvm/environment/mock/random_generator.go @@ -9,8 +9,8 @@ type RandomGenerator struct { mock.Mock } -// Random provides a mock function with given fields: -func (_m *RandomGenerator) Random() (uint64, error) { +// UnsafeRandom provides a mock function with given fields: +func (_m *RandomGenerator) UnsafeRandom() (uint64, error) { ret := _m.Called() var r0 uint64 From 8b2a7a5eb63b3953af38f671becafa703dc0567c Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Sun, 9 Jul 2023 11:51:56 -0700 Subject: [PATCH 109/169] [CI] Rename without_netgo tag to without-netgo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 09a2b1b9456..28ca0a96294 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ ifeq (${IMAGE_TAG},) IMAGE_TAG := ${SHORT_COMMIT} endif -IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without_netgo +IMAGE_TAG_NO_NETGO := $(IMAGE_TAG)-without-netgo # Name of the cover profile COVER_PROFILE := coverage.txt From 92674a70750f1b5801eff2064c31a790e024da19 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 10 Jul 2023 12:48:29 +0300 Subject: [PATCH 110/169] Updated README file, updated logging accoring to comments --- engine/access/rest/README.md | 11 ++++--- .../rest/apiproxy/rest_proxy_handler.go | 33 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/engine/access/rest/README.md b/engine/access/rest/README.md index 68378fa19b1..d94af68c238 100644 --- a/engine/access/rest/README.md +++ b/engine/access/rest/README.md @@ -12,7 +12,8 @@ available on our [docs site](https://docs.onflow.org/http-api/). - `request`: Implementation of API requests that provide validation for input data and build request models. - `routes`: The common HTTP handlers for all the requests, tests for each request. - `apiproxy`: Implementation of proxy backend handler which includes the local backend and forwards the methods which -can't be handled locally to an upstream using gRPC API. +can't be handled locally to an upstream using gRPC API. This is used by observers that don't have all data in their +local db. ## Request lifecycle @@ -51,7 +52,7 @@ generator models.LinkGenerator, ) (interface{}, error) ``` -That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. Also needs -to override the method if it should be proxied to the backend handler `RestProxyHandler` for request forwarding. Adding -a new API endpoint also requires for a new request builder to be implemented and added in request package. Make sure to -not forget about adding tests for each of the API handler. +That handler implementation needs to be added to the `router.go` with corresponding API endpoint and method. If the data +is not available on observers, an override the method is needed in the backend handler `RestProxyHandler` for request +forwarding. Adding a new API endpoint also requires for a new request builder to be implemented and added in request +package. Make sure to not forget about adding tests for each of the API handler. diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index 9fb8cc64ada..9dd1ea58e82 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -89,8 +89,9 @@ func (r *RestProxyHandler) GetCollectionByID(ctx context.Context, id flow.Identi } collectionResponse, err := upstream.GetCollectionByID(ctx, getCollectionByIDRequest) + r.log("upstream", "GetCollectionByID", err) + if err != nil { - r.log("upstream", "GetCollectionByID", err) return nil, err } @@ -131,8 +132,9 @@ func (r *RestProxyHandler) GetTransaction(ctx context.Context, id flow.Identifie Id: id[:], } transactionResponse, err := upstream.GetTransaction(ctx, getTransactionRequest) + r.log("upstream", "GetTransaction", err) + if err != nil { - r.log("upstream", "GetTransaction", err) return nil, err } @@ -159,8 +161,9 @@ func (r *RestProxyHandler) GetTransactionResult(ctx context.Context, id flow.Ide } transactionResultResponse, err := upstream.GetTransactionResult(ctx, getTransactionResultRequest) + r.log("upstream", "GetTransactionResult", err) + if err != nil { - r.log("upstream", "GetTransactionResult", err) return nil, err } @@ -180,8 +183,9 @@ func (r *RestProxyHandler) GetAccountAtBlockHeight(ctx context.Context, address } accountResponse, err := upstream.GetAccountAtBlockHeight(ctx, getAccountAtBlockHeightRequest) + r.log("upstream", "GetAccountAtBlockHeight", err) + if err != nil { - r.log("upstream", "GetAccountAtBlockHeight", err) return nil, models.NewNotFoundError("not found account at block height", err) } @@ -200,8 +204,9 @@ func (r *RestProxyHandler) ExecuteScriptAtLatestBlock(ctx context.Context, scrip Arguments: arguments, } executeScriptAtLatestBlockResponse, err := upstream.ExecuteScriptAtLatestBlock(ctx, executeScriptAtLatestBlockRequest) + r.log("upstream", "ExecuteScriptAtLatestBlock", err) + if err != nil { - r.log("upstream", "ExecuteScriptAtLatestBlock", err) return nil, err } @@ -221,8 +226,9 @@ func (r *RestProxyHandler) ExecuteScriptAtBlockHeight(ctx context.Context, block Arguments: arguments, } executeScriptAtBlockHeightResponse, err := upstream.ExecuteScriptAtBlockHeight(ctx, executeScriptAtBlockHeightRequest) + r.log("upstream", "ExecuteScriptAtBlockHeight", err) + if err != nil { - r.log("upstream", "ExecuteScriptAtBlockHeight", err) return nil, err } @@ -242,8 +248,9 @@ func (r *RestProxyHandler) ExecuteScriptAtBlockID(ctx context.Context, blockID f Arguments: arguments, } executeScriptAtBlockIDResponse, err := upstream.ExecuteScriptAtBlockID(ctx, executeScriptAtBlockIDRequest) + r.log("upstream", "ExecuteScriptAtBlockID", err) + if err != nil { - r.log("upstream", "ExecuteScriptAtBlockID", err) return nil, err } @@ -263,8 +270,9 @@ func (r *RestProxyHandler) GetEventsForHeightRange(ctx context.Context, eventTyp EndHeight: endHeight, } eventsResponse, err := upstream.GetEventsForHeightRange(ctx, getEventsForHeightRangeRequest) + r.log("upstream", "GetEventsForHeightRange", err) + if err != nil { - r.log("upstream", "GetEventsForHeightRange", err) return nil, err } @@ -285,8 +293,9 @@ func (r *RestProxyHandler) GetEventsForBlockIDs(ctx context.Context, eventType s BlockIds: blockIds, } eventsResponse, err := upstream.GetEventsForBlockIDs(ctx, getEventsForBlockIDsRequest) + r.log("upstream", "GetEventsForBlockIDs", err) + if err != nil { - r.log("upstream", "GetEventsForBlockIDs", err) return nil, err } @@ -304,8 +313,9 @@ func (r *RestProxyHandler) GetExecutionResultForBlockID(ctx context.Context, blo BlockId: blockID[:], } executionResultForBlockIDResponse, err := upstream.GetExecutionResultForBlockID(ctx, getExecutionResultForBlockID) + r.log("upstream", "GetExecutionResultForBlockID", err) + if err != nil { - r.log("upstream", "GetExecutionResultForBlockID", err) return nil, err } @@ -324,8 +334,9 @@ func (r *RestProxyHandler) GetExecutionResultByID(ctx context.Context, id flow.I } executionResultByIDResponse, err := upstream.GetExecutionResultByID(ctx, executionResultByIDRequest) + r.log("upstream", "GetExecutionResultByID", err) + if err != nil { - r.log("upstream", "GetExecutionResultByID", err) return nil, err } From 9256de44d33a6c2edcd048867e369e9e9fbe5d72 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Mon, 10 Jul 2023 15:20:00 +0300 Subject: [PATCH 111/169] Added tests for convert functions, moved 2 to right file --- access/handler.go | 6 ++--- engine/common/rpc/convert/blocks.go | 23 ------------------- engine/common/rpc/convert/collections_test.go | 20 +++++++++++++--- engine/common/rpc/convert/events.go | 21 +++++++++++++++++ engine/common/rpc/convert/events_test.go | 22 ++++++++++++++++++ 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/access/handler.go b/access/handler.go index 2417d4037ac..e5b4a18374c 100644 --- a/access/handler.go +++ b/access/handler.go @@ -516,7 +516,7 @@ func (h *Handler) GetEventsForHeightRange( return nil, err } - resultEvents, err := blockEventsToMessages(results) + resultEvents, err := BlockEventsToMessages(results) if err != nil { return nil, err } @@ -548,7 +548,7 @@ func (h *Handler) GetEventsForBlockIDs( return nil, err } - resultEvents, err := blockEventsToMessages(results) + resultEvents, err := BlockEventsToMessages(results) if err != nil { return nil, err } @@ -680,7 +680,7 @@ func executionResultToMessages(er *flow.ExecutionResult, metadata *entities.Meta }, nil } -func blockEventsToMessages(blocks []flow.BlockEvents) ([]*access.EventsResponse_Result, error) { +func BlockEventsToMessages(blocks []flow.BlockEvents) ([]*access.EventsResponse_Result, error) { results := make([]*access.EventsResponse_Result, len(blocks)) for i, block := range blocks { diff --git a/engine/common/rpc/convert/blocks.go b/engine/common/rpc/convert/blocks.go index a10cd6a7854..2e7f5689515 100644 --- a/engine/common/rpc/convert/blocks.go +++ b/engine/common/rpc/convert/blocks.go @@ -7,7 +7,6 @@ import ( "github.com/onflow/flow-go/model/flow" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" ) @@ -154,25 +153,3 @@ func PayloadFromMessage(m *entities.Block) (*flow.Payload, error) { Results: results, }, nil } - -// MessagesToBlockEvents converts a protobuf EventsResponse_Result messages to a flow.BlockEvents. -func MessagesToBlockEvents(blocksEvents []*accessproto.EventsResponse_Result) []flow.BlockEvents { - evs := make([]flow.BlockEvents, len(blocksEvents)) - for _, ev := range blocksEvents { - var blockEvent flow.BlockEvents - MessageToBlockEvents(ev) - evs = append(evs, blockEvent) - } - - return evs -} - -// MessageToBlockEvents converts a protobuf EventsResponse_Result message to a slice of flow.BlockEvents. -func MessageToBlockEvents(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { - return flow.BlockEvents{ - BlockHeight: blockEvents.BlockHeight, - BlockID: MessageToIdentifier(blockEvents.BlockId), - BlockTimestamp: blockEvents.BlockTimestamp.AsTime(), - Events: MessagesToEvents(blockEvents.Events), - } -} diff --git a/engine/common/rpc/convert/collections_test.go b/engine/common/rpc/convert/collections_test.go index 75ab6f25adc..2e14a6dc225 100644 --- a/engine/common/rpc/convert/collections_test.go +++ b/engine/common/rpc/convert/collections_test.go @@ -9,6 +9,8 @@ import ( "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" + + "github.com/onflow/flow/protobuf/go/flow/entities" ) // TestConvertCollection tests that converting a collection to a protobuf message results in the correct @@ -32,10 +34,12 @@ func TestConvertCollection(t *testing.T) { } }) - t.Run("convert light collection to message", func(t *testing.T) { - lightCollection := flow.LightCollection{Transactions: txIDs} + var msg *entities.Collection + lightCollection := flow.LightCollection{Transactions: txIDs} - msg, err := convert.LightCollectionToMessage(&lightCollection) + t.Run("convert light collection to message", func(t *testing.T) { + var err error + msg, err = convert.LightCollectionToMessage(&lightCollection) require.NoError(t, err) assert.Len(t, msg.TransactionIds, len(txIDs)) @@ -43,6 +47,16 @@ func TestConvertCollection(t *testing.T) { assert.Equal(t, txID[:], msg.TransactionIds[i]) } }) + + t.Run("convert message to light collection", func(t *testing.T) { + lightColl, err := convert.MessageToLightCollection(msg) + require.NoError(t, err) + + assert.Equal(t, len(txIDs), len(lightColl.Transactions)) + for _, txID := range lightColl.Transactions { + assert.Equal(t, txID[:], txID[:]) + } + }) } // TestConvertCollectionGuarantee tests that converting a collection guarantee to and from a protobuf diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index d3bd469cd48..e80631ae4a3 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" @@ -172,3 +173,23 @@ func CcfEventToJsonEvent(e flow.Event) (*flow.Event, error) { Payload: convertedPayload, }, nil } + +// MessagesToBlockEvents converts a protobuf EventsResponse_Result messages to a slice of flow.BlockEvents. +func MessagesToBlockEvents(blocksEvents []*accessproto.EventsResponse_Result) []flow.BlockEvents { + evs := make([]flow.BlockEvents, len(blocksEvents)) + for i, ev := range blocksEvents { + evs[i] = MessageToBlockEvents(ev) + } + + return evs +} + +// MessageToBlockEvents converts a protobuf EventsResponse_Result message to a flow.BlockEvents. +func MessageToBlockEvents(blockEvents *accessproto.EventsResponse_Result) flow.BlockEvents { + return flow.BlockEvents{ + BlockHeight: blockEvents.BlockHeight, + BlockID: MessageToIdentifier(blockEvents.BlockId), + BlockTimestamp: blockEvents.BlockTimestamp.AsTime(), + Events: MessagesToEvents(blockEvents.Events), + } +} diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go index 2cf010fa011..6483dac72ee 100644 --- a/engine/common/rpc/convert/events_test.go +++ b/engine/common/rpc/convert/events_test.go @@ -11,6 +11,7 @@ import ( jsoncdc "github.com/onflow/cadence/encoding/json" execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -193,3 +194,24 @@ func TestConvertServiceEventList(t *testing.T) { assert.Equal(t, serviceEvents, converted) } + +// TestConvertMessagesToBlockEvents tests that converting a protobuf EventsResponse_Result message to and from block events in the same +// block +func TestConvertMessagesToBlockEvents(t *testing.T) { + t.Parallel() + + count := 2 + blockEvents := make([]flow.BlockEvents, count) + for i := 0; i < count; i++ { + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(i))) + blockEvents[i] = unittest.BlockEventsFixture(header, 2) + } + + msg, err := access.BlockEventsToMessages(blockEvents) + require.NoError(t, err) + + converted := convert.MessagesToBlockEvents(msg) + require.NoError(t, err) + + assert.Equal(t, blockEvents, converted) +} From 8bd1ae8d93020b1271547e8c98594cfa12eadaae Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 10 Jul 2023 13:15:45 -0400 Subject: [PATCH 112/169] wip --- network/alsp/manager/manager.go | 5 +- network/alsp/manager/manager_test.go | 124 ++++++++++++++++++++++++++- network/slashing/consumer.go | 1 - 3 files changed, 123 insertions(+), 7 deletions(-) diff --git a/network/alsp/manager/manager.go b/network/alsp/manager/manager.go index 941407e2637..87dcd89cd11 100644 --- a/network/alsp/manager/manager.go +++ b/network/alsp/manager/manager.go @@ -332,8 +332,9 @@ func (m *MisbehaviorReportManager) onHeartbeat() error { FlowIds: flow.IdentifierList{id}, Cause: network.DisallowListedCauseAlsp, // sets the ALSP disallow listing cause on node }) + fmt.Println("DISALLOW-LISTED", record.OriginId, record.Penalty) } - + fmt.Println(record.OriginId, record.Penalty) // each time we decay the penalty by the decay speed, the penalty is a negative number, and the decay speed // is a positive number. So the penalty is getting closer to zero. // We use math.Min() to make sure the penalty is never positive. @@ -376,6 +377,7 @@ func (m *MisbehaviorReportManager) onHeartbeat() error { Float64("updated_penalty", penalty). Msg("spam record decayed") } + fmt.Println() return nil } @@ -425,7 +427,6 @@ func (m *MisbehaviorReportManager) processMisbehaviorReport(report internal.Repo // we should crash the node in this case to prevent further misbehavior reports from being lost and fix the bug. return fmt.Errorf("failed to apply penalty to the spam record: %w", err) } - lg.Debug().Float64("updated_penalty", updatedPenalty).Msg("misbehavior report handled") return nil } diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 9b067e1ede8..81678f9a4a4 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -2,6 +2,8 @@ package alspmgr_test import ( "context" + "fmt" + "github.com/onflow/flow-go/network/slashing" "math" "math/rand" "sync" @@ -183,7 +185,7 @@ func TestHandleReportedMisbehavior_Cache_Integration(t *testing.T) { // TestHandleReportedMisbehavior_And_DisallowListing_Integration implements an end-to-end integration test for the // handling of reported misbehavior and disallow listing. // -// The test sets up 3 nodes, one victim, one honest, and one (alledged) spammer. +// The test sets up 3 nodes, one victim, one honest, and one (alleged) spammer. // Initially, the test ensures that all nodes are connected to each other. // Then, test imitates that victim node reports the spammer node for spamming. // The test generates enough spam reports to trigger the disallow-listing of the victim node. @@ -196,11 +198,11 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute // of the network, we need to configure the ALSP manager via the network configuration, and let the network create // the ALSP manager. - var victimSpamRecordCacheCache alsp.SpamRecordCache + var victimSpamRecordCache alsp.SpamRecordCache cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { - victimSpamRecordCacheCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) - return victimSpamRecordCacheCache + victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return victimSpamRecordCache }), } @@ -266,6 +268,120 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[spammerIndex]}) } +// TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration implements an end-to-end integration test for the +// handling of reported misbehavior from the slashing.ViolationsConsumer. +// +// The test sets up one victim, one honest, and one (alleged) spammer for each of the current slashing violations. +// Initially, the test ensures that all nodes are connected to each other. +// Then, test imitates the slashing violations consumer on the victim node reporting misbehavior's for each slashing violation. +// The test generates enough slashing violations to trigger the connection to each of the spamming nodes to be eventually pruned. +// The test ensures that the victim node is disconnected from all spammer nodes. +// The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and +// the pruned spammer nodes are established. +func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { + cfg := managerCfgFixture(t) + + // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute + // of the network, we need to configure the ALSP manager via the network configuration, and let the network create + // the ALSP manager. + var victimSpamRecordCache alsp.SpamRecordCache + cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ + alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { + victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) + return victimSpamRecordCache + }), + } + + slashingMisbehaviors := []network.Misbehavior{ + alsp.InvalidMessage, alsp.SenderEjected, alsp.UnauthorizedUnicastOnChannel, + alsp.UnauthorizedPublishOnChannel, alsp.UnknownMsgType, + } + + // create 1 victim node, 1 honest node and a node for each slashing violation + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, len(slashingMisbehaviors)+2, + p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) + victimNetwork, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) + + // create slashing violations consumer with victim node network providing the network.MisbehaviorReportConsumer interface + violationsConsumer := slashing.NewSlashingViolationsConsumer(unittest.Logger(), metrics.NewNoopCollector(), victimNetwork) + mws[0].SetSlashingViolationsConsumer(violationsConsumer) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + testutils.StartNodesAndNetworks(signalerCtx, t, nodes, []network.Network{victimNetwork}, 100*time.Millisecond) + defer testutils.StopComponents[p2p.LibP2PNode](t, nodes, 100*time.Millisecond) + defer cancel() + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + // initially victim and spammer should be able to connect to each other. + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // each slashing violation func is mapped to a violation with the identity of one of the misbehaving nodes + // index of the victim node in the nodes slice. + victimIndex := 0 + honestNodeIndex := 1 + invalidMessageIndex := 2 + senderEjectedIndex := 3 + unauthorizedUnicastOnChannelIndex := 4 + unauthorizedPublishOnChannelIndex := 5 + unknownMsgTypeIndex := 6 + slashingViolationTestCases := []struct { + violationsConsumerFunc func(violation *network.Violation) + violation *network.Violation + }{ + //{violationsConsumer.OnUnAuthorizedSenderError, &network.Violation{Identity: ids[invalidMessageIndex]}}, + //{violationsConsumer.OnSenderEjectedError, &network.Violation{Identity: ids[senderEjectedIndex]}}, + {violationsConsumer.OnUnauthorizedUnicastOnChannel, &network.Violation{Identity: ids[unauthorizedUnicastOnChannelIndex]}}, + {violationsConsumer.OnUnauthorizedPublishOnChannel, &network.Violation{Identity: ids[unauthorizedPublishOnChannelIndex]}}, + {violationsConsumer.OnUnknownMsgTypeError, &network.Violation{Identity: ids[unknownMsgTypeIndex]}}, + } + + violationsWg := sync.WaitGroup{} + violationCount := 120 + for _, testCase := range slashingViolationTestCases { + for i := 0; i < violationCount; i++ { + violationsWg.Add(1) + go func() { + defer violationsWg.Done() + testCase.violationsConsumerFunc(testCase.violation) + }() + } + } + unittest.RequireReturnsBefore(t, violationsWg.Wait, 100*time.Millisecond, "slashing violations not reported in time") + + time.Sleep(10 * time.Second) + + forEachMisbehavingNode := func(f func(i int)) { + //for misbehavingNodeIndex := 2; misbehavingNodeIndex <= len(nodes)-1; misbehavingNodeIndex++ { + // f(misbehavingNodeIndex) + //} + f(invalidMessageIndex) + f(senderEjectedIndex) + } + + // ensures connections to all misbehaving nodes are pruned + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + fmt.Println(misbehavingNodeIndex) + p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}, 100*time.Millisecond, 2*time.Second) + }) + + // despite being disconnected from the victim node misbehaving nodes and the honest node are still connected. + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[misbehavingNodeIndex]}, 1*time.Millisecond, 100*time.Millisecond) + }) + + // despite pruning misbehaving nodes, ensure that (victim and honest) are still connected. + p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) + + // while misbehaving nodes are pruned, they cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the misbehaving nodes until each node's peer score decays. + forEachMisbehavingNode(func(misbehavingNodeIndex int) { + p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}) + }) +} + // TestMisbehaviorReportMetrics tests the recording of misbehavior report metrics. // It checks that when a misbehavior report is received by the ALSP manager, the metrics are recorded. // It fails the test if the metrics are not recorded or if they are recorded incorrectly. diff --git a/network/slashing/consumer.go b/network/slashing/consumer.go index b7f09bb12b7..3ba8d656c21 100644 --- a/network/slashing/consumer.go +++ b/network/slashing/consumer.go @@ -88,7 +88,6 @@ func (c *Consumer) reportMisbehavior(misbehavior network.Misbehavior, violation Err(err). Str("peerID", violation.PeerID). Msg("failed to create misbehavior report") - } c.misbehaviorReportConsumer.ReportMisbehaviorOnChannel(violation.Channel, report) } From 1eed112554b0aa3f64127f3cfbd1c71715960eb2 Mon Sep 17 00:00:00 2001 From: Yahya Hassanzadeh Date: Mon, 10 Jul 2023 10:29:49 -0700 Subject: [PATCH 113/169] applies the fix --- network/alsp/manager/manager.go | 1 + network/alsp/manager/manager_test.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/network/alsp/manager/manager.go b/network/alsp/manager/manager.go index 87dcd89cd11..573d8d624ad 100644 --- a/network/alsp/manager/manager.go +++ b/network/alsp/manager/manager.go @@ -230,6 +230,7 @@ func (m *MisbehaviorReportManager) HandleMisbehaviorReport(channel channels.Chan Hex("misbehaving_id", logging.ID(report.OriginId())). Str("reason", report.Reason().String()). Float64("penalty", report.Penalty()).Logger() + lg.Trace().Msg("received misbehavior report") m.metrics.OnMisbehaviorReported(channel.String(), report.Reason().String()) nonce := [internal.NonceSize]byte{} diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 81678f9a4a4..e053e4c8be3 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -3,13 +3,14 @@ package alspmgr_test import ( "context" "fmt" - "github.com/onflow/flow-go/network/slashing" "math" "math/rand" "sync" "testing" "time" + "github.com/onflow/flow-go/network/slashing" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -343,6 +344,7 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t violationCount := 120 for _, testCase := range slashingViolationTestCases { for i := 0; i < violationCount; i++ { + testCase := testCase violationsWg.Add(1) go func() { defer violationsWg.Done() From c7675452368cddbfd467c956b42efae7ccff1e89 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 10 Jul 2023 13:38:02 -0400 Subject: [PATCH 114/169] update test --- network/alsp/manager/manager.go | 2 -- network/alsp/manager/manager_test.go | 18 ++++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/network/alsp/manager/manager.go b/network/alsp/manager/manager.go index 573d8d624ad..a49adee9146 100644 --- a/network/alsp/manager/manager.go +++ b/network/alsp/manager/manager.go @@ -333,9 +333,7 @@ func (m *MisbehaviorReportManager) onHeartbeat() error { FlowIds: flow.IdentifierList{id}, Cause: network.DisallowListedCauseAlsp, // sets the ALSP disallow listing cause on node }) - fmt.Println("DISALLOW-LISTED", record.OriginId, record.Penalty) } - fmt.Println(record.OriginId, record.Penalty) // each time we decay the penalty by the decay speed, the penalty is a negative number, and the decay speed // is a positive number. So the penalty is getting closer to zero. // We use math.Min() to make sure the penalty is never positive. diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index e053e4c8be3..c4793480a86 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -2,7 +2,6 @@ package alspmgr_test import ( "context" - "fmt" "math" "math/rand" "sync" @@ -317,7 +316,7 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t defer cancel() p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) - // initially victim and spammer should be able to connect to each other. + // initially victim and misbehaving nodes should be able to connect to each other. p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) // each slashing violation func is mapped to a violation with the identity of one of the misbehaving nodes @@ -333,8 +332,8 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t violationsConsumerFunc func(violation *network.Violation) violation *network.Violation }{ - //{violationsConsumer.OnUnAuthorizedSenderError, &network.Violation{Identity: ids[invalidMessageIndex]}}, - //{violationsConsumer.OnSenderEjectedError, &network.Violation{Identity: ids[senderEjectedIndex]}}, + {violationsConsumer.OnUnAuthorizedSenderError, &network.Violation{Identity: ids[invalidMessageIndex]}}, + {violationsConsumer.OnSenderEjectedError, &network.Violation{Identity: ids[senderEjectedIndex]}}, {violationsConsumer.OnUnauthorizedUnicastOnChannel, &network.Violation{Identity: ids[unauthorizedUnicastOnChannelIndex]}}, {violationsConsumer.OnUnauthorizedPublishOnChannel, &network.Violation{Identity: ids[unauthorizedPublishOnChannelIndex]}}, {violationsConsumer.OnUnknownMsgTypeError, &network.Violation{Identity: ids[unknownMsgTypeIndex]}}, @@ -354,19 +353,14 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t } unittest.RequireReturnsBefore(t, violationsWg.Wait, 100*time.Millisecond, "slashing violations not reported in time") - time.Sleep(10 * time.Second) - forEachMisbehavingNode := func(f func(i int)) { - //for misbehavingNodeIndex := 2; misbehavingNodeIndex <= len(nodes)-1; misbehavingNodeIndex++ { - // f(misbehavingNodeIndex) - //} - f(invalidMessageIndex) - f(senderEjectedIndex) + for misbehavingNodeIndex := 2; misbehavingNodeIndex <= len(nodes)-1; misbehavingNodeIndex++ { + f(misbehavingNodeIndex) + } } // ensures connections to all misbehaving nodes are pruned forEachMisbehavingNode(func(misbehavingNodeIndex int) { - fmt.Println(misbehavingNodeIndex) p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}, 100*time.Millisecond, 2*time.Second) }) From 1ed374cac7cb5a517da0a7372b45a4dfaa71399e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 10 Jul 2023 13:38:38 -0400 Subject: [PATCH 115/169] Update manager.go --- network/alsp/manager/manager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/alsp/manager/manager.go b/network/alsp/manager/manager.go index a49adee9146..a1c3e25bf03 100644 --- a/network/alsp/manager/manager.go +++ b/network/alsp/manager/manager.go @@ -376,7 +376,6 @@ func (m *MisbehaviorReportManager) onHeartbeat() error { Float64("updated_penalty", penalty). Msg("spam record decayed") } - fmt.Println() return nil } From cce68b5bbedb20a758acbb46dcf056ff912a2144 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Mon, 10 Jul 2023 13:45:59 -0400 Subject: [PATCH 116/169] Update manager_test.go --- network/alsp/manager/manager_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index c4793480a86..8563053b48d 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -359,20 +359,20 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t } } - // ensures connections to all misbehaving nodes are pruned + // ensures all misbehaving nodes are disconnected from the victim node forEachMisbehavingNode(func(misbehavingNodeIndex int) { p2ptest.RequireEventuallyNotConnected(t, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}, 100*time.Millisecond, 2*time.Second) }) - // despite being disconnected from the victim node misbehaving nodes and the honest node are still connected. + // despite being disconnected from the victim node, misbehaving nodes and the honest node are still connected. forEachMisbehavingNode(func(misbehavingNodeIndex int) { p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[misbehavingNodeIndex]}, 1*time.Millisecond, 100*time.Millisecond) }) - // despite pruning misbehaving nodes, ensure that (victim and honest) are still connected. + // despite disconnecting misbehaving nodes, ensure that (victim and honest) are still connected. p2ptest.RequireConnectedEventually(t, []p2p.LibP2PNode{nodes[honestNodeIndex], nodes[victimIndex]}, 1*time.Millisecond, 100*time.Millisecond) - // while misbehaving nodes are pruned, they cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the misbehaving nodes until each node's peer score decays. + // while misbehaving nodes are disconnected, they cannot connect to the victim node. Also, the victim node cannot directly dial and connect to the misbehaving nodes until each node's peer score decays. forEachMisbehavingNode(func(misbehavingNodeIndex int) { p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, []p2p.LibP2PNode{nodes[victimIndex]}, []p2p.LibP2PNode{nodes[misbehavingNodeIndex]}) }) From c467235817e73b05aef2974535424dd477c13873 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 11 Jul 2023 15:34:40 +0300 Subject: [PATCH 117/169] Updated unittest according to comment --- .../access/rest/routes/transactions_test.go | 36 +--------------- integration/tests/access/observer_test.go | 43 ++++++++++++++++--- utils/unittest/fixtures.go | 33 ++++++++++++++ 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/engine/access/rest/routes/transactions_test.go b/engine/access/rest/routes/transactions_test.go index 6adde49b245..a23e2053cb7 100644 --- a/engine/access/rest/routes/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -69,38 +69,6 @@ func createTransactionReq(body interface{}) *http.Request { return req } -func validCreateBody(tx flow.TransactionBody) map[string]interface{} { - tx.Arguments = [][]uint8{} // fix how fixture creates nil values - auth := make([]string, len(tx.Authorizers)) - for i, a := range tx.Authorizers { - auth[i] = a.String() - } - - return map[string]interface{}{ - "script": util.ToBase64(tx.Script), - "arguments": tx.Arguments, - "reference_block_id": tx.ReferenceBlockID.String(), - "gas_limit": fmt.Sprintf("%d", tx.GasLimit), - "payer": tx.Payer.String(), - "proposal_key": map[string]interface{}{ - "address": tx.ProposalKey.Address.String(), - "key_index": fmt.Sprintf("%d", tx.ProposalKey.KeyIndex), - "sequence_number": fmt.Sprintf("%d", tx.ProposalKey.SequenceNumber), - }, - "authorizers": auth, - "payload_signatures": []map[string]interface{}{{ - "address": tx.PayloadSignatures[0].Address.String(), - "key_index": fmt.Sprintf("%d", tx.PayloadSignatures[0].KeyIndex), - "signature": util.ToBase64(tx.PayloadSignatures[0].Signature), - }}, - "envelope_signatures": []map[string]interface{}{{ - "address": tx.EnvelopeSignatures[0].Address.String(), - "key_index": fmt.Sprintf("%d", tx.EnvelopeSignatures[0].KeyIndex), - "signature": util.ToBase64(tx.EnvelopeSignatures[0].Signature), - }}, - } -} - func TestGetTransactions(t *testing.T) { t.Run("get by ID without results", func(t *testing.T) { backend := &mock.API{} @@ -380,7 +348,7 @@ func TestCreateTransaction(t *testing.T) { tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} tx.Arguments = [][]uint8{} - req := createTransactionReq(validCreateBody(tx)) + req := createTransactionReq(unittest.ValidCreateBody(tx)) backend.Mock. On("SendTransaction", mocks.Anything, &tx). @@ -447,7 +415,7 @@ func TestCreateTransaction(t *testing.T) { for _, test := range tests { tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} - testTx := validCreateBody(tx) + testTx := unittest.ValidCreateBody(tx) testTx[test.inputField] = test.inputValue req := createTransactionReq(testTx) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 164b131be38..57a1553fc3b 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -1,8 +1,11 @@ package access import ( + "bytes" "context" + "encoding/json" "fmt" + "io" "net/http" "strings" "testing" @@ -173,7 +176,7 @@ func (s *ObserverSuite) TestObserverRPC() { // TestObserverRest runs the following tests: // 1. CompareRPCs: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN // 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rests handled by the upstream -// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries +// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all others queries func (s *ObserverSuite) TestObserverRest() { t := s.T() @@ -186,7 +189,14 @@ func (s *ObserverSuite) TestObserverRest() { case http.MethodGet: return httpClient.Get(url) case http.MethodPost: - return httpClient.Post(url, "application/json", strings.NewReader("{}")) + var body io.Reader + if strings.Contains(url, "/transactions") { + body = createTx(s.net) + } else { + body = strings.NewReader("{}") + } + + return httpClient.Post(url, "application/json", body) } panic("not supported") } @@ -210,6 +220,7 @@ func (s *ObserverSuite) TestObserverRest() { assert.NoError(t, accessErr) assert.NoError(t, observerErr) assert.Equal(t, accessResp.Status, observerResp.Status) + assert.Equal(t, accessResp.StatusCode, observerResp.StatusCode) }) } }) @@ -219,7 +230,7 @@ func (s *ObserverSuite) TestObserverRest() { require.NoError(t, err) t.Run("HandledByUpstream", func(t *testing.T) { - // verify that we receive StatusInternalServerError, StatusServiceUnavailable, StatusBadRequest errors from all rests handled upstream + // verify that we receive StatusInternalServerError, StatusServiceUnavailable errors from all rests handled upstream for _, endpoint := range s.getRestEndpoints() { if _, local := s.localRest[endpoint.name]; local { continue @@ -229,8 +240,7 @@ func (s *ObserverSuite) TestObserverRest() { require.NoError(t, observerErr) assert.Contains(t, [...]int{ http.StatusInternalServerError, - http.StatusServiceUnavailable, - http.StatusBadRequest}, observerResp.StatusCode) + http.StatusServiceUnavailable}, observerResp.StatusCode) }) } }) @@ -393,7 +403,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { block := unittest.BlockFixture() executionResult := unittest.ExecutionResultFixture() collection := unittest.CollectionFixture(2) - blockEvents := unittest.BlockEventsFixture(unittest.BlockHeaderFixture(unittest.WithHeaderHeight(uint64(2))), 2) + eventType := "A.0123456789abcdef.flow.event" return []RestEndpointTest{ { @@ -454,7 +464,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { { name: "getEvents", method: http.MethodGet, - path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", blockEvents.Events[0].Type, 1, 3), + path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", eventType, 1, 3), }, { name: "getNetworkParameters", @@ -468,3 +478,22 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { }, } } + +func createTx(net *testnet.FlowNetwork) *bytes.Buffer { + flowAddr := flow.Localnet.Chain().ServiceAddress() + signature := unittest.TransactionSignatureFixture() + signature.Address = flowAddr + + tx := flow.NewTransactionBody(). + AddAuthorizer(flowAddr). + SetPayer(flowAddr). + SetScript(unittest.NoopTxScript()). + SetReferenceBlockID(net.Root().ID()). + SetProposalKey(flowAddr, 1, 0) + tx.PayloadSignatures = []flow.TransactionSignature{signature} + tx.EnvelopeSignatures = []flow.TransactionSignature{signature} + + jsonBody, _ := json.Marshal(unittest.ValidCreateBody(*tx)) + + return bytes.NewBuffer(jsonBody) +} diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index f5d454d01b0..6746875b3b7 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -22,6 +22,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/crypto/hash" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/bitutils" @@ -2461,3 +2462,35 @@ func ChunkExecutionDataFixture(t *testing.T, minSize int, opts ...func(*executio size *= 2 } } + +func ValidCreateBody(tx flow.TransactionBody) map[string]interface{} { + tx.Arguments = [][]uint8{} // fix how fixture creates nil values + auth := make([]string, len(tx.Authorizers)) + for i, a := range tx.Authorizers { + auth[i] = a.String() + } + + return map[string]interface{}{ + "script": util.ToBase64(tx.Script), + "arguments": tx.Arguments, + "reference_block_id": tx.ReferenceBlockID.String(), + "gas_limit": fmt.Sprintf("%d", tx.GasLimit), + "payer": tx.Payer.String(), + "proposal_key": map[string]interface{}{ + "address": tx.ProposalKey.Address.String(), + "key_index": fmt.Sprintf("%d", tx.ProposalKey.KeyIndex), + "sequence_number": fmt.Sprintf("%d", tx.ProposalKey.SequenceNumber), + }, + "authorizers": auth, + "payload_signatures": []map[string]interface{}{{ + "address": tx.PayloadSignatures[0].Address.String(), + "key_index": fmt.Sprintf("%d", tx.PayloadSignatures[0].KeyIndex), + "signature": util.ToBase64(tx.PayloadSignatures[0].Signature), + }}, + "envelope_signatures": []map[string]interface{}{{ + "address": tx.EnvelopeSignatures[0].Address.String(), + "key_index": fmt.Sprintf("%d", tx.EnvelopeSignatures[0].KeyIndex), + "signature": util.ToBase64(tx.EnvelopeSignatures[0].Signature), + }}, + } +} From c7a4d5820fcb1d705c606a2a70f03a17683cd1b2 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:30:04 -0400 Subject: [PATCH 118/169] Update config/default-config.yml Co-authored-by: Yahya Hassanzadeh, Ph.D. --- config/default-config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/default-config.yml b/config/default-config.yml index 7788aa6a91d..9834694b0e2 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -64,7 +64,8 @@ network-config: # Note that we purposefully choose this logging interval high enough to avoid spamming the logs. gossipsub-score-tracer-interval: 1m # The default RPC sent tracker cache size. The RPC sent tracker is used to track RPC control messages sent from the local node. - gossipsub-rpc-sent-tracker-cache-size: 10000 + # Note: this cache size must be large enough to keep a history of sent messages in a reasonable time window of past history. + gossipsub-rpc-sent-tracker-cache-size: 1_000_000 # Peer scoring is the default value for enabling peer scoring gossipsub-peer-scoring-enabled: true # Gossipsub rpc inspectors configs From 252dd2d2addb7e98f670f7fca9962b7287d700df Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:30:19 -0400 Subject: [PATCH 119/169] Update module/metrics/labels.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- module/metrics/labels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/metrics/labels.go b/module/metrics/labels.go index d57dd418b56..9febc9ab391 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -92,7 +92,7 @@ const ( ResourceNetworkingApplicationLayerSpamReportQueue = "application_layer_spam_report_queue" ResourceNetworkingRpcClusterPrefixReceivedCache = "rpc_cluster_prefixed_received_cache" ResourceNetworkingDisallowListCache = "disallow_list_cache" - ResourceNetworkingRPCSentTrackerCache = "rpc_sent_tracker_cache" + ResourceNetworkingRPCSentTrackerCache = "gossipsub_rpc_sent_tracker_cache" ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel From 647065b7f909915cf3c72a22a05becc935130fb5 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:30:26 -0400 Subject: [PATCH 120/169] Update network/p2p/tracer/gossipSubMeshTracer.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/tracer/gossipSubMeshTracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 48054427b1d..6ea32c1b464 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -69,7 +69,7 @@ func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTr topicMeshMap: make(map[string]map[peer.ID]struct{}), idProvider: config.IDProvider, metrics: config.Metrics, - logger: config.Logger.With().Str("component", "gossip_sub_topology_tracer").Logger(), + logger: config.Logger.With().Str("component", "gossipsub_topology_tracer").Logger(), loggerInterval: config.LoggerInterval, rpcSentTracker: rpcSentTracker, } From 72e6b7db62a82452760e26a3fdd988d343b032a9 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:38:54 -0400 Subject: [PATCH 121/169] document irrecoverable error in NewGossipSubMeshTracer --- network/p2p/tracer/gossipSubMeshTracer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 48054427b1d..21a93cddb22 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -58,6 +58,12 @@ type GossipSubMeshTracerConfig struct { RpcSentTrackerCacheSize uint32 } +// NewGossipSubMeshTracer creates a new *GossipSubMeshTracer. +// Args: +// - *GossipSubMeshTracerConfig: the mesh tracer config. +// Returns: +// - *GossipSubMeshTracer: new mesh tracer. +// - error: if any error is encountered during the creation of the gossipsub mesh tracer, all errors are considered irrecoverable. func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTracer, error) { rpcSentTracker, err := internal.NewRPCSentTracker(config.Logger, config.RpcSentTrackerCacheSize, config.RpcSentTrackerCacheCollector) if err != nil { From 1dcb3a4408a998cf454a313a4d67beaefe3ed0ba Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:39:39 -0400 Subject: [PATCH 122/169] Update network/p2p/tracer/gossipSubMeshTracer.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/tracer/gossipSubMeshTracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 6ea32c1b464..7ea6b0c7b78 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -61,7 +61,7 @@ type GossipSubMeshTracerConfig struct { func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTracer, error) { rpcSentTracker, err := internal.NewRPCSentTracker(config.Logger, config.RpcSentTrackerCacheSize, config.RpcSentTrackerCacheCollector) if err != nil { - return nil, err + return nil, fmt.Errof("could not create rpc send tracker: %w", err) } g := &GossipSubMeshTracer{ From 2b2d2fc2651e38e2c20f14f50f297f1f1f39da44 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:44:28 -0400 Subject: [PATCH 123/169] use cache size from default config --- network/p2p/tracer/gossipSubMeshTracer_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index 4b469beb2e7..f9d409a762d 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" @@ -30,6 +31,8 @@ import ( // One of the nodes is running with an unknown peer id, which the identity provider is mocked to return an error and // the mesh tracer should log a warning message. func TestGossipSubMeshTracer(t *testing.T) { + defaultConfig, err := config.DefaultConfig() + require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) sporkId := unittest.IdentifierFixture() @@ -66,9 +69,9 @@ func TestGossipSubMeshTracer(t *testing.T) { Logger: logger, Metrics: collector, IDProvider: idProvider, - LoggerInterval: 1 * time.Second, + LoggerInterval: time.Second, RpcSentTrackerCacheCollector: metrics.NewNoopCollector(), - RpcSentTrackerCacheSize: uint32(100), + RpcSentTrackerCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) require.NoError(t, err) From 6b6a74a0386dacdf74aada3985f6aee949d0b63e Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:49:34 -0400 Subject: [PATCH 124/169] Update network/p2p/tracer/internal/cache.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/p2p/tracer/internal/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index 34f34c4fa01..a16c502f907 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -34,7 +34,7 @@ func newRPCSentCache(config *RPCSentCacheConfig) (*rpcSentCache, error) { backData := herocache.NewCache(config.sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, - config.logger.With().Str("mempool", "gossipsub=rpc-control-messages-sent").Logger(), + config.logger.With().Str("mempool", "gossipsub-rpc-control-messages-sent").Logger(), config.collector) return &rpcSentCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), From 45889037518c7a67efbbd7e685b2a5c9a2a8acab Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:51:11 -0400 Subject: [PATCH 125/169] rename RPCSentCacheConfig -> rpcCtrlMsgSentCacheConfig --- network/p2p/tracer/internal/cache.go | 4 ++-- network/p2p/tracer/internal/cache_test.go | 2 +- network/p2p/tracer/internal/rpc_sent_tracker.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index 34f34c4fa01..668e05e3199 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -11,7 +11,7 @@ import ( p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) -type RPCSentCacheConfig struct { +type rpcCtrlMsgSentCacheConfig struct { sizeLimit uint32 logger zerolog.Logger collector module.HeroCacheMetrics @@ -30,7 +30,7 @@ type rpcSentCache struct { // - *rpcSentCache: the created cache. // Note that this cache is intended to track control messages sent by the local node, // it stores a RPCSendEntity using an Id which should uniquely identifies the message being tracked. -func newRPCSentCache(config *RPCSentCacheConfig) (*rpcSentCache, error) { +func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) (*rpcSentCache, error) { backData := herocache.NewCache(config.sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go index fc043ee116f..ae8671c8f68 100644 --- a/network/p2p/tracer/internal/cache_test.go +++ b/network/p2p/tracer/internal/cache_test.go @@ -211,7 +211,7 @@ func TestRecordCache_ConcurrentInitAndRemove(t *testing.T) { // cacheFixture returns a new *RecordCache. func cacheFixture(t *testing.T, sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *rpcSentCache { - config := &RPCSentCacheConfig{ + config := &rpcCtrlMsgSentCacheConfig{ sizeLimit: sizeLimit, logger: logger, collector: collector, diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index 66791217c57..9b08a79efc5 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -18,7 +18,7 @@ type RPCSentTracker struct { // NewRPCSentTracker returns a new *NewRPCSentTracker. func NewRPCSentTracker(logger zerolog.Logger, sizeLimit uint32, collector module.HeroCacheMetrics) (*RPCSentTracker, error) { - config := &RPCSentCacheConfig{ + config := &rpcCtrlMsgSentCacheConfig{ sizeLimit: sizeLimit, logger: logger, collector: collector, From 8d95bb4e2373438dbb9316443b93bd3cea3ff857 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 09:52:06 -0400 Subject: [PATCH 126/169] add godoc for rpcCtrlMsgSentCacheConfig --- network/p2p/tracer/internal/cache.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index bfa35ba6b35..556463256f0 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -11,9 +11,10 @@ import ( p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) +// rpcCtrlMsgSentCacheConfig configuration for the rpc sent cache. type rpcCtrlMsgSentCacheConfig struct { - sizeLimit uint32 logger zerolog.Logger + sizeLimit uint32 collector module.HeroCacheMetrics } From cfc36723646b2a7e5b88ec4322d1b23ec4375d9f Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 10:00:46 -0400 Subject: [PATCH 127/169] remove error return from NewGossipSubMeshTracer func signature --- cmd/access/node_builder/access_node_builder.go | 5 +---- cmd/observer/node_builder/observer_builder.go | 5 +---- follower/follower_builder.go | 5 +---- network/internal/p2pfixtures/fixtures.go | 3 +-- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 6 ++---- network/p2p/tracer/gossipSubMeshTracer.go | 11 +++-------- network/p2p/tracer/gossipSubMeshTracer_test.go | 3 +-- network/p2p/tracer/internal/cache.go | 4 ++-- network/p2p/tracer/internal/cache_test.go | 3 +-- network/p2p/tracer/internal/rpc_sent_tracker.go | 8 ++------ network/p2p/tracer/internal/rpc_sent_tracker_test.go | 9 ++++----- 11 files changed, 19 insertions(+), 43 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index cbfad7b6e05..5edd2629ee2 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -1199,10 +1199,7 @@ func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.Pri RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - if err != nil { - return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) - } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) libp2pNode, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 9eca3748a63..78ddc464fb7 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -710,10 +710,7 @@ func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - if err != nil { - return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) - } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) node, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/follower/follower_builder.go b/follower/follower_builder.go index eaa2a4a0c82..e2eb43cb49c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -612,10 +612,7 @@ func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.Pr RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), RpcSentTrackerCacheSize: builder.FlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - if err != nil { - return nil, fmt.Errorf("could not create gossipsub mesh tracer for staked access node: %w", err) - } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) node, err := p2pbuilder.NewNodeBuilder( builder.Logger, diff --git a/network/internal/p2pfixtures/fixtures.go b/network/internal/p2pfixtures/fixtures.go index 2d2c18983ab..29ee0509fbb 100644 --- a/network/internal/p2pfixtures/fixtures.go +++ b/network/internal/p2pfixtures/fixtures.go @@ -111,8 +111,7 @@ func CreateNode(t *testing.T, networkKey crypto.PrivateKey, sporkID flow.Identif RpcSentTrackerCacheCollector: metrics.NewNoopCollector(), RpcSentTrackerCacheSize: defaultFlowConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - require.NoError(t, err) + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) builder := p2pbuilder.NewNodeBuilder( logger, diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index da8768402ed..8e550b4fa94 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -503,10 +503,8 @@ func DefaultNodeBuilder( RpcSentTrackerCacheCollector: metrics.GossipSubRPCSentTrackerMetricFactory(metricsCfg.HeroCacheFactory, flownet.PrivateNetwork), RpcSentTrackerCacheSize: gossipCfg.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - if err != nil { - return nil, fmt.Errorf("could not create gossipsub mesh tracer: %w", err) - } + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) + builder.SetGossipSubTracer(meshTracer) builder.SetGossipSubScoreTracerInterval(gossipCfg.ScoreTracerInterval) diff --git a/network/p2p/tracer/gossipSubMeshTracer.go b/network/p2p/tracer/gossipSubMeshTracer.go index 5135c920390..1cc25fd2565 100644 --- a/network/p2p/tracer/gossipSubMeshTracer.go +++ b/network/p2p/tracer/gossipSubMeshTracer.go @@ -63,13 +63,8 @@ type GossipSubMeshTracerConfig struct { // - *GossipSubMeshTracerConfig: the mesh tracer config. // Returns: // - *GossipSubMeshTracer: new mesh tracer. -// - error: if any error is encountered during the creation of the gossipsub mesh tracer, all errors are considered irrecoverable. -func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTracer, error) { - rpcSentTracker, err := internal.NewRPCSentTracker(config.Logger, config.RpcSentTrackerCacheSize, config.RpcSentTrackerCacheCollector) - if err != nil { - return nil, fmt.Errof("could not create rpc send tracker: %w", err) - } - +func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) *GossipSubMeshTracer { + rpcSentTracker := internal.NewRPCSentTracker(config.Logger, config.RpcSentTrackerCacheSize, config.RpcSentTrackerCacheCollector) g := &GossipSubMeshTracer{ RawTracer: NewGossipSubNoopTracer(), topicMeshMap: make(map[string]map[peer.ID]struct{}), @@ -87,7 +82,7 @@ func NewGossipSubMeshTracer(config *GossipSubMeshTracerConfig) (*GossipSubMeshTr }). Build() - return g, nil + return g } // GetMeshPeers returns the local mesh peers for the given topic. diff --git a/network/p2p/tracer/gossipSubMeshTracer_test.go b/network/p2p/tracer/gossipSubMeshTracer_test.go index f9d409a762d..a2da0584f94 100644 --- a/network/p2p/tracer/gossipSubMeshTracer_test.go +++ b/network/p2p/tracer/gossipSubMeshTracer_test.go @@ -73,8 +73,7 @@ func TestGossipSubMeshTracer(t *testing.T) { RpcSentTrackerCacheCollector: metrics.NewNoopCollector(), RpcSentTrackerCacheSize: defaultConfig.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, } - meshTracer, err := tracer.NewGossipSubMeshTracer(meshTracerCfg) - require.NoError(t, err) + meshTracer := tracer.NewGossipSubMeshTracer(meshTracerCfg) tracerNode, tracerId := p2ptest.NodeFixture( t, sporkId, diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index 556463256f0..1094aeca95c 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -31,7 +31,7 @@ type rpcSentCache struct { // - *rpcSentCache: the created cache. // Note that this cache is intended to track control messages sent by the local node, // it stores a RPCSendEntity using an Id which should uniquely identifies the message being tracked. -func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) (*rpcSentCache, error) { +func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) *rpcSentCache { backData := herocache.NewCache(config.sizeLimit, herocache.DefaultOversizeFactor, heropool.LRUEjection, @@ -39,7 +39,7 @@ func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) (*rpcSentCache, error) { config.collector) return &rpcSentCache{ c: stdmap.NewBackend(stdmap.WithBackData(backData)), - }, nil + } } // init initializes the record cached for the given messageID if it does not exist. diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go index ae8671c8f68..3383382a488 100644 --- a/network/p2p/tracer/internal/cache_test.go +++ b/network/p2p/tracer/internal/cache_test.go @@ -216,8 +216,7 @@ func cacheFixture(t *testing.T, sizeLimit uint32, logger zerolog.Logger, collect logger: logger, collector: collector, } - r, err := newRPCSentCache(config) - require.NoError(t, err) + r := newRPCSentCache(config) // expect cache to be empty require.Equalf(t, uint(0), r.size(), "cache size must be 0") require.NotNil(t, r) diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index 9b08a79efc5..4ab68cb6e19 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -17,17 +17,13 @@ type RPCSentTracker struct { } // NewRPCSentTracker returns a new *NewRPCSentTracker. -func NewRPCSentTracker(logger zerolog.Logger, sizeLimit uint32, collector module.HeroCacheMetrics) (*RPCSentTracker, error) { +func NewRPCSentTracker(logger zerolog.Logger, sizeLimit uint32, collector module.HeroCacheMetrics) *RPCSentTracker { config := &rpcCtrlMsgSentCacheConfig{ sizeLimit: sizeLimit, logger: logger, collector: collector, } - cache, err := newRPCSentCache(config) - if err != nil { - return nil, fmt.Errorf("failed to create new rpc sent cahe: %w", err) - } - return &RPCSentTracker{cache: cache}, nil + return &RPCSentTracker{cache: newRPCSentCache(config)} } // OnIHaveRPCSent caches a unique entity message ID for each message ID included in each rpc iHave control message. diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go index 5ebfc2da1d1..f113f862cbf 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker_test.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -15,13 +15,13 @@ import ( // TestNewRPCSentTracker ensures *RPCSenTracker is created as expected. func TestNewRPCSentTracker(t *testing.T) { - tracker := mockTracker(t) + tracker := mockTracker() require.NotNil(t, tracker) } // TestRPCSentTracker_IHave ensures *RPCSentTracker tracks sent iHave control messages as expected. func TestRPCSentTracker_IHave(t *testing.T) { - tracker := mockTracker(t) + tracker := mockTracker() require.NotNil(t, tracker) t.Run("WasIHaveRPCSent should return false for iHave message Id that has not been tracked", func(t *testing.T) { @@ -46,12 +46,11 @@ func TestRPCSentTracker_IHave(t *testing.T) { }) } -func mockTracker(t *testing.T) *RPCSentTracker { +func mockTracker() *RPCSentTracker { logger := zerolog.Nop() sizeLimit := uint32(100) collector := metrics.NewNoopCollector() - tracker, err := NewRPCSentTracker(logger, sizeLimit, collector) - require.NoError(t, err) + tracker := NewRPCSentTracker(logger, sizeLimit, collector) return tracker } From d270551bb6f3a2280833ff5e1c051e611c3df30c Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 10:10:25 -0400 Subject: [PATCH 128/169] rename messageId -> messageEntityID --- network/p2p/tracer/internal/cache.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index 1094aeca95c..e5a8e3da373 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -42,26 +42,26 @@ func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) *rpcSentCache { } } -// init initializes the record cached for the given messageID if it does not exist. +// init initializes the record cached for the given messageEntityID if it does not exist. // Returns true if the record is initialized, false otherwise (i.e.: the record already exists). // Args: -// - flow.Identifier: the messageID to store the rpc control message. +// - flow.Identifier: the messageEntityID to store the rpc control message. // - p2p.ControlMessageType: the rpc control message type. // Returns: // - bool: true if the record is initialized, false otherwise (i.e.: the record already exists). -// Note that if init is called multiple times for the same messageID, the record is initialized only once, and the +// Note that if init is called multiple times for the same messageEntityID, the record is initialized only once, and the // subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). -func (r *rpcSentCache) init(messageID flow.Identifier, controlMsgType p2pmsg.ControlMessageType) bool { - return r.c.Add(newRPCSentEntity(messageID, controlMsgType)) +func (r *rpcSentCache) init(messageEntityID flow.Identifier, controlMsgType p2pmsg.ControlMessageType) bool { + return r.c.Add(newRPCSentEntity(messageEntityID, controlMsgType)) } // has checks if the RPC message has been cached indicating it has been sent. // Args: -// - flow.Identifier: the messageID to store the rpc control message. +// - flow.Identifier: the messageEntityID to store the rpc control message. // Returns: // - bool: true if the RPC has been cache indicating it was sent from the local node. -func (r *rpcSentCache) has(messageId flow.Identifier) bool { - return r.c.Has(messageId) +func (r *rpcSentCache) has(messageEntityID flow.Identifier) bool { + return r.c.Has(messageEntityID) } // ids returns the list of ids of each rpcSentEntity stored. @@ -69,14 +69,14 @@ func (r *rpcSentCache) ids() []flow.Identifier { return flow.GetIDs(r.c.All()) } -// remove the record of the given messageID from the cache. +// remove the record of the given messageEntityID from the cache. // Returns true if the record is removed, false otherwise (i.e., the record does not exist). // Args: -// - flow.Identifier: the messageID to store the rpc control message. +// - flow.Identifier: the messageEntityID to store the rpc control message. // Returns: // - true if the record is removed, false otherwise (i.e., the record does not exist). -func (r *rpcSentCache) remove(messageID flow.Identifier) bool { - return r.c.Remove(messageID) +func (r *rpcSentCache) remove(messageEntityID flow.Identifier) bool { + return r.c.Remove(messageEntityID) } // size returns the number of records in the cache. From 8edb7444a56f2ddbf3fbe6338a2668681a7235e1 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 10:10:55 -0400 Subject: [PATCH 129/169] remove unused funcs remove & ids --- network/p2p/tracer/internal/cache.go | 15 --- network/p2p/tracer/internal/cache_test.go | 106 ---------------------- 2 files changed, 121 deletions(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index e5a8e3da373..b5f0a635c47 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -64,21 +64,6 @@ func (r *rpcSentCache) has(messageEntityID flow.Identifier) bool { return r.c.Has(messageEntityID) } -// ids returns the list of ids of each rpcSentEntity stored. -func (r *rpcSentCache) ids() []flow.Identifier { - return flow.GetIDs(r.c.All()) -} - -// remove the record of the given messageEntityID from the cache. -// Returns true if the record is removed, false otherwise (i.e., the record does not exist). -// Args: -// - flow.Identifier: the messageEntityID to store the rpc control message. -// Returns: -// - true if the record is removed, false otherwise (i.e., the record does not exist). -func (r *rpcSentCache) remove(messageEntityID flow.Identifier) bool { - return r.c.Remove(messageEntityID) -} - // size returns the number of records in the cache. func (r *rpcSentCache) size() uint { return r.c.Size() diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go index 3383382a488..fa746adbc8a 100644 --- a/network/p2p/tracer/internal/cache_test.go +++ b/network/p2p/tracer/internal/cache_test.go @@ -1,7 +1,6 @@ package internal import ( - "fmt" "sync" "testing" "time" @@ -104,111 +103,6 @@ func TestCache_ConcurrentSameRecordInit(t *testing.T) { require.True(t, cache.has(id)) } -// TestCache_Remove tests the remove method of the RecordCache. -// The test covers the following scenarios: -// 1. Initializing the cache with multiple records. -// 2. Removing a record and checking if it is removed correctly. -// 3. Ensuring the other records are still in the cache after removal. -// 4. Attempting to remove a non-existent ID. -func TestCache_Remove(t *testing.T) { - cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2pmsg.CtrlMsgIHave - // initialize spam records for a few ids - id1 := unittest.IdentifierFixture() - id2 := unittest.IdentifierFixture() - id3 := unittest.IdentifierFixture() - - require.True(t, cache.init(id1, controlMsgType)) - require.True(t, cache.init(id2, controlMsgType)) - require.True(t, cache.init(id3, controlMsgType)) - - numOfIds := uint(3) - require.Equal(t, numOfIds, cache.size(), fmt.Sprintf("expected size of the cache to be %d", numOfIds)) - // remove id1 and check if the record is removed - require.True(t, cache.remove(id1)) - require.NotContains(t, id1, cache.ids()) - - // check if the other ids are still in the cache - require.True(t, cache.has(id2)) - require.True(t, cache.has(id3)) - - // attempt to remove a non-existent ID - id4 := unittest.IdentifierFixture() - require.False(t, cache.remove(id4)) -} - -// TestCache_ConcurrentRemove tests the concurrent removal of records for different ids. -// The test covers the following scenarios: -// 1. Multiple goroutines removing records for different ids concurrently. -// 2. The records are correctly removed from the cache. -func TestCache_ConcurrentRemove(t *testing.T) { - cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2pmsg.CtrlMsgIHave - ids := unittest.IdentifierListFixture(10) - for _, id := range ids { - cache.init(id, controlMsgType) - } - - var wg sync.WaitGroup - wg.Add(len(ids)) - - for _, id := range ids { - go func(id flow.Identifier) { - defer wg.Done() - require.True(t, cache.remove(id)) - require.NotContains(t, id, cache.ids()) - }(id) - } - - unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") - - require.Equal(t, uint(0), cache.size()) -} - -// TestRecordCache_ConcurrentInitAndRemove tests the concurrent initialization and removal of records for different -// ids. The test covers the following scenarios: -// 1. Multiple goroutines initializing records for different ids concurrently. -// 2. Multiple goroutines removing records for different ids concurrently. -// 3. The initialized records are correctly added to the cache. -// 4. The removed records are correctly removed from the cache. -func TestRecordCache_ConcurrentInitAndRemove(t *testing.T) { - cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) - controlMsgType := p2pmsg.CtrlMsgIHave - ids := unittest.IdentifierListFixture(20) - idsToAdd := ids[:10] - idsToRemove := ids[10:] - - for _, id := range idsToRemove { - cache.init(id, controlMsgType) - } - - var wg sync.WaitGroup - wg.Add(len(ids)) - - // initialize spam records concurrently - for _, id := range idsToAdd { - go func(id flow.Identifier) { - defer wg.Done() - cache.init(id, controlMsgType) - }(id) - } - - // remove spam records concurrently - for _, id := range idsToRemove { - go func(id flow.Identifier) { - defer wg.Done() - require.True(t, cache.remove(id)) - require.NotContains(t, id, cache.ids()) - }(id) - } - - unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") - - // ensure that the initialized records are correctly added to the cache - // and removed records are correctly removed from the cache - require.ElementsMatch(t, idsToAdd, cache.ids()) -} - // cacheFixture returns a new *RecordCache. func cacheFixture(t *testing.T, sizeLimit uint32, logger zerolog.Logger, collector module.HeroCacheMetrics) *rpcSentCache { config := &rpcCtrlMsgSentCacheConfig{ From 145de0517a1eff4a1d32d87a70531d73e9f56059 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 11 Jul 2023 17:15:39 +0300 Subject: [PATCH 130/169] Updated rest integration test --- integration/tests/access/observer_test.go | 30 ++++++++++------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 57a1553fc3b..9806685217a 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -174,7 +174,7 @@ func (s *ObserverSuite) TestObserverRPC() { } // TestObserverRest runs the following tests: -// 1. CompareRPCs: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN +// 1. CompareEndpoints: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN // 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rests handled by the upstream // 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all others queries func (s *ObserverSuite) TestObserverRest() { @@ -184,27 +184,20 @@ func (s *ObserverSuite) TestObserverRest() { observerAddr := s.net.ContainerByName("observer_1").Addr(testnet.RESTPort) httpClient := http.DefaultClient - makeHttpCall := func(method string, url string) (*http.Response, error) { + makeHttpCall := func(method string, url string, body io.Reader) (*http.Response, error) { switch method { case http.MethodGet: return httpClient.Get(url) case http.MethodPost: - var body io.Reader - if strings.Contains(url, "/transactions") { - body = createTx(s.net) - } else { - body = strings.NewReader("{}") - } - return httpClient.Post(url, "application/json", body) } panic("not supported") } - makeObserverCall := func(method string, path string) (*http.Response, error) { - return makeHttpCall(method, "http://"+observerAddr+"/v1"+path) + makeObserverCall := func(method string, path string, body io.Reader) (*http.Response, error) { + return makeHttpCall(method, "http://"+observerAddr+"/v1"+path, body) } - makeAccessCall := func(method string, path string) (*http.Response, error) { - return makeHttpCall(method, "http://"+accessAddr+"/v1"+path) + makeAccessCall := func(method string, path string, body io.Reader) (*http.Response, error) { + return makeHttpCall(method, "http://"+accessAddr+"/v1"+path, body) } t.Run("CompareEndpoints", func(t *testing.T) { @@ -215,8 +208,8 @@ func (s *ObserverSuite) TestObserverRest() { continue } t.Run(endpoint.name, func(t *testing.T) { - accessResp, accessErr := makeAccessCall(endpoint.method, endpoint.path) - observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + accessResp, accessErr := makeAccessCall(endpoint.method, endpoint.path, endpoint.body) + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) assert.NoError(t, accessErr) assert.NoError(t, observerErr) assert.Equal(t, accessResp.Status, observerResp.Status) @@ -236,7 +229,7 @@ func (s *ObserverSuite) TestObserverRest() { continue } t.Run(endpoint.name, func(t *testing.T) { - observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) require.NoError(t, observerErr) assert.Contains(t, [...]int{ http.StatusInternalServerError, @@ -252,7 +245,7 @@ func (s *ObserverSuite) TestObserverRest() { continue } t.Run(endpoint.name, func(t *testing.T) { - observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path) + observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) require.NoError(t, observerErr) assert.Contains(t, [...]int{http.StatusNotFound, http.StatusOK}, observerResp.StatusCode) }) @@ -395,6 +388,7 @@ type RestEndpointTest struct { name string method string path string + body io.Reader } func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { @@ -415,6 +409,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { name: "createTransaction", method: http.MethodPost, path: "/transactions", + body: createTx(s.net), }, { name: "getTransactionResultByID", @@ -455,6 +450,7 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { name: "executeScript", method: http.MethodPost, path: "/scripts", + body: strings.NewReader("{}"), }, { name: "getAccount", From 60b764dc9ce99628578fb3407689316c3e22c3f9 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 10:40:28 -0400 Subject: [PATCH 131/169] improve cache func signature cohesion --- network/p2p/tracer/internal/cache.go | 34 ++++++++++---- network/p2p/tracer/internal/cache_test.go | 46 ++++++++++--------- .../p2p/tracer/internal/rpc_sent_tracker.go | 16 +------ 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/network/p2p/tracer/internal/cache.go b/network/p2p/tracer/internal/cache.go index b5f0a635c47..b916133b270 100644 --- a/network/p2p/tracer/internal/cache.go +++ b/network/p2p/tracer/internal/cache.go @@ -1,6 +1,8 @@ package internal import ( + "fmt" + "github.com/rs/zerolog" "github.com/onflow/flow-go/model/flow" @@ -42,29 +44,43 @@ func newRPCSentCache(config *rpcCtrlMsgSentCacheConfig) *rpcSentCache { } } -// init initializes the record cached for the given messageEntityID if it does not exist. +// add initializes the record cached for the given messageEntityID if it does not exist. // Returns true if the record is initialized, false otherwise (i.e.: the record already exists). // Args: -// - flow.Identifier: the messageEntityID to store the rpc control message. -// - p2p.ControlMessageType: the rpc control message type. +// - topic: the topic ID. +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. // Returns: // - bool: true if the record is initialized, false otherwise (i.e.: the record already exists). -// Note that if init is called multiple times for the same messageEntityID, the record is initialized only once, and the +// Note that if add is called multiple times for the same messageEntityID, the record is initialized only once, and the // subsequent calls return false and do not change the record (i.e.: the record is not re-initialized). -func (r *rpcSentCache) init(messageEntityID flow.Identifier, controlMsgType p2pmsg.ControlMessageType) bool { - return r.c.Add(newRPCSentEntity(messageEntityID, controlMsgType)) +func (r *rpcSentCache) add(topic string, messageId string, controlMsgType p2pmsg.ControlMessageType) bool { + return r.c.Add(newRPCSentEntity(r.rpcSentEntityID(topic, messageId, controlMsgType), controlMsgType)) } // has checks if the RPC message has been cached indicating it has been sent. // Args: -// - flow.Identifier: the messageEntityID to store the rpc control message. +// - topic: the topic ID. +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. // Returns: // - bool: true if the RPC has been cache indicating it was sent from the local node. -func (r *rpcSentCache) has(messageEntityID flow.Identifier) bool { - return r.c.Has(messageEntityID) +func (r *rpcSentCache) has(topic string, messageId string, controlMsgType p2pmsg.ControlMessageType) bool { + return r.c.Has(r.rpcSentEntityID(topic, messageId, controlMsgType)) } // size returns the number of records in the cache. func (r *rpcSentCache) size() uint { return r.c.Size() } + +// rpcSentEntityID creates an entity ID from the topic, messageID and control message type. +// Args: +// - topic: the topic ID. +// - messageId: the message ID. +// - controlMsgType: the rpc control message type. +// Returns: +// - flow.Identifier: the entity ID. +func (r *rpcSentCache) rpcSentEntityID(topic string, messageId string, controlMsgType p2pmsg.ControlMessageType) flow.Identifier { + return flow.MakeIDFromFingerPrint([]byte(fmt.Sprintf("%s%s%s", topic, messageId, controlMsgType))) +} diff --git a/network/p2p/tracer/internal/cache_test.go b/network/p2p/tracer/internal/cache_test.go index fa746adbc8a..c92b42b5e02 100644 --- a/network/p2p/tracer/internal/cache_test.go +++ b/network/p2p/tracer/internal/cache_test.go @@ -12,59 +12,62 @@ import ( "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/network/channels" p2pmsg "github.com/onflow/flow-go/network/p2p/message" "github.com/onflow/flow-go/utils/unittest" ) -// TestCache_Init tests the init method of the rpcSentCache. +// TestCache_Add tests the add method of the rpcSentCache. // It ensures that the method returns true when a new record is initialized // and false when an existing record is initialized. -func TestCache_Init(t *testing.T) { +func TestCache_Add(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) controlMsgType := p2pmsg.CtrlMsgIHave - id1 := unittest.IdentifierFixture() - id2 := unittest.IdentifierFixture() + topic := channels.PushBlocks.String() + messageID1 := unittest.IdentifierFixture().String() + messageID2 := unittest.IdentifierFixture().String() // test initializing a record for an ID that doesn't exist in the cache - initialized := cache.init(id1, controlMsgType) + initialized := cache.add(topic, messageID1, controlMsgType) require.True(t, initialized, "expected record to be initialized") - require.True(t, cache.has(id1), "expected record to exist") + require.True(t, cache.has(topic, messageID1, controlMsgType), "expected record to exist") // test initializing a record for an ID that already exists in the cache - initialized = cache.init(id1, controlMsgType) + initialized = cache.add(topic, messageID1, controlMsgType) require.False(t, initialized, "expected record not to be initialized") - require.True(t, cache.has(id1), "expected record to exist") + require.True(t, cache.has(topic, messageID1, controlMsgType), "expected record to exist") // test initializing a record for another ID - initialized = cache.init(id2, controlMsgType) + initialized = cache.add(topic, messageID2, controlMsgType) require.True(t, initialized, "expected record to be initialized") - require.True(t, cache.has(id2), "expected record to exist") + require.True(t, cache.has(topic, messageID2, controlMsgType), "expected record to exist") } // TestCache_ConcurrentInit tests the concurrent initialization of records. // The test covers the following scenarios: // 1. Multiple goroutines initializing records for different ids. // 2. Ensuring that all records are correctly initialized. -func TestCache_ConcurrentInit(t *testing.T) { +func TestCache_ConcurrentAdd(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) controlMsgType := p2pmsg.CtrlMsgIHave - ids := unittest.IdentifierListFixture(10) + topic := channels.PushBlocks.String() + messageIds := unittest.IdentifierListFixture(10) var wg sync.WaitGroup - wg.Add(len(ids)) + wg.Add(len(messageIds)) - for _, id := range ids { + for _, id := range messageIds { go func(id flow.Identifier) { defer wg.Done() - cache.init(id, controlMsgType) + cache.add(topic, id.String(), controlMsgType) }(id) } unittest.RequireReturnsBefore(t, wg.Wait, 100*time.Millisecond, "timed out waiting for goroutines to finish") // ensure that all records are correctly initialized - for _, id := range ids { - require.True(t, cache.has(id)) + for _, id := range messageIds { + require.True(t, cache.has(topic, id.String(), controlMsgType)) } } @@ -73,10 +76,11 @@ func TestCache_ConcurrentInit(t *testing.T) { // 1. Multiple goroutines attempting to initialize the same record concurrently. // 2. Only one goroutine successfully initializes the record, and others receive false on initialization. // 3. The record is correctly initialized in the cache and can be retrieved using the Get method. -func TestCache_ConcurrentSameRecordInit(t *testing.T) { +func TestCache_ConcurrentSameRecordAdd(t *testing.T) { cache := cacheFixture(t, 100, zerolog.Nop(), metrics.NewNoopCollector()) controlMsgType := p2pmsg.CtrlMsgIHave - id := unittest.IdentifierFixture() + topic := channels.PushBlocks.String() + messageID := unittest.IdentifierFixture().String() const concurrentAttempts = 10 var wg sync.WaitGroup @@ -87,7 +91,7 @@ func TestCache_ConcurrentSameRecordInit(t *testing.T) { for i := 0; i < concurrentAttempts; i++ { go func() { defer wg.Done() - initSuccess := cache.init(id, controlMsgType) + initSuccess := cache.add(topic, messageID, controlMsgType) if initSuccess { successGauge.Inc() } @@ -100,7 +104,7 @@ func TestCache_ConcurrentSameRecordInit(t *testing.T) { require.Equal(t, int32(1), successGauge.Load()) // ensure that the record is correctly initialized in the cache - require.True(t, cache.has(id)) + require.True(t, cache.has(topic, messageID, controlMsgType)) } // cacheFixture returns a new *RecordCache. diff --git a/network/p2p/tracer/internal/rpc_sent_tracker.go b/network/p2p/tracer/internal/rpc_sent_tracker.go index 4ab68cb6e19..6d44ac984a3 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker.go @@ -1,12 +1,9 @@ package internal import ( - "fmt" - pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/rs/zerolog" - "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" p2pmsg "github.com/onflow/flow-go/network/p2p/message" ) @@ -34,8 +31,7 @@ func (t *RPCSentTracker) OnIHaveRPCSent(iHaves []*pb.ControlIHave) { for _, iHave := range iHaves { topicID := iHave.GetTopicID() for _, messageID := range iHave.GetMessageIDs() { - entityMsgID := iHaveRPCSentEntityID(topicID, messageID) - t.cache.init(entityMsgID, controlMsgType) + t.cache.add(topicID, messageID, controlMsgType) } } } @@ -47,13 +43,5 @@ func (t *RPCSentTracker) OnIHaveRPCSent(iHaves []*pb.ControlIHave) { // Returns: // - bool: true if the iHave rpc with the provided message ID was sent. func (t *RPCSentTracker) WasIHaveRPCSent(topicID, messageID string) bool { - entityMsgID := iHaveRPCSentEntityID(topicID, messageID) - return t.cache.has(entityMsgID) -} - -// iHaveRPCSentEntityID appends the topicId and messageId and returns the flow.Identifier hash. -// Each iHave RPC control message contains a single topicId and multiple messageIds, to ensure we -// produce a unique id for each message we append the messageId to the topicId. -func iHaveRPCSentEntityID(topicId, messageId string) flow.Identifier { - return flow.MakeIDFromFingerPrint([]byte(fmt.Sprintf("%s%s", topicId, messageId))) + return t.cache.has(topicID, messageID, p2pmsg.CtrlMsgIHave) } From 000581eab6c47e710a6b80405bb469cdafa66dfe Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Tue, 11 Jul 2023 18:33:24 +0300 Subject: [PATCH 132/169] Fixed typo, updated last commit --- integration/tests/access/observer_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 9806685217a..1cc222cc990 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -176,7 +176,7 @@ func (s *ObserverSuite) TestObserverRPC() { // TestObserverRest runs the following tests: // 1. CompareEndpoints: verifies that the observer client returns the same errors as the access client for rests proxied to the upstream AN // 2. HandledByUpstream: stops the upstream AN and verifies that the observer client returns errors for all rests handled by the upstream -// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all others queries +// 3. HandledByObserver: stops the upstream AN and verifies that the observer client handles all other queries func (s *ObserverSuite) TestObserverRest() { t := s.T() @@ -189,7 +189,11 @@ func (s *ObserverSuite) TestObserverRest() { case http.MethodGet: return httpClient.Get(url) case http.MethodPost: - return httpClient.Post(url, "application/json", body) + if body == nil { + return httpClient.Post(url, "application/json", strings.NewReader("{}")) + } else { + return httpClient.Post(url, "application/json", body) + } } panic("not supported") } @@ -450,7 +454,6 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { name: "executeScript", method: http.MethodPost, path: "/scripts", - body: strings.NewReader("{}"), }, { name: "getAccount", From 64f629d6d65a2db87b2c1e1a3ce83f3647ce7805 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Tue, 11 Jul 2023 12:18:04 -0400 Subject: [PATCH 133/169] update WasIHaveRPCSent to check multiple topic IDs with multiple message ID's --- .../tracer/internal/rpc_sent_tracker_test.go | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/network/p2p/tracer/internal/rpc_sent_tracker_test.go b/network/p2p/tracer/internal/rpc_sent_tracker_test.go index f113f862cbf..7b9c4ec9acb 100644 --- a/network/p2p/tracer/internal/rpc_sent_tracker_test.go +++ b/network/p2p/tracer/internal/rpc_sent_tracker_test.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/require" + "github.com/onflow/flow-go/config" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/utils/unittest" @@ -15,13 +16,13 @@ import ( // TestNewRPCSentTracker ensures *RPCSenTracker is created as expected. func TestNewRPCSentTracker(t *testing.T) { - tracker := mockTracker() + tracker := mockTracker(t) require.NotNil(t, tracker) } // TestRPCSentTracker_IHave ensures *RPCSentTracker tracks sent iHave control messages as expected. func TestRPCSentTracker_IHave(t *testing.T) { - tracker := mockTracker() + tracker := mockTracker(t) require.NotNil(t, tracker) t.Run("WasIHaveRPCSent should return false for iHave message Id that has not been tracked", func(t *testing.T) { @@ -29,28 +30,41 @@ func TestRPCSentTracker_IHave(t *testing.T) { }) t.Run("WasIHaveRPCSent should return true for iHave message after it is tracked with OnIHaveRPCSent", func(t *testing.T) { - topicID := channels.PushBlocks.String() - messageID1 := unittest.IdentifierFixture().String() - iHaves := []*pb.ControlIHave{{ - TopicID: &topicID, - MessageIDs: []string{messageID1}, - }} + numOfMsgIds := 100 + testCases := []struct { + topic string + messageIDS []string + }{ + {channels.PushBlocks.String(), unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {channels.ReceiveApprovals.String(), unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {channels.SyncCommittee.String(), unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + {channels.RequestChunks.String(), unittest.IdentifierListFixture(numOfMsgIds).Strings()}, + } + iHaves := make([]*pb.ControlIHave, len(testCases)) + for i, testCase := range testCases { + testCase := testCase + iHaves[i] = &pb.ControlIHave{ + TopicID: &testCase.topic, + MessageIDs: testCase.messageIDS, + } + } rpc := rpcFixture(withIhaves(iHaves)) tracker.OnIHaveRPCSent(rpc.GetControl().GetIhave()) - require.True(t, tracker.WasIHaveRPCSent(topicID, messageID1)) - // manipulate last byte of message ID ensure false positive not returned - messageID2 := []byte(messageID1) - messageID2[len(messageID2)-1] = 'X' - require.False(t, tracker.WasIHaveRPCSent(topicID, string(messageID2))) + for _, testCase := range testCases { + for _, messageID := range testCase.messageIDS { + require.True(t, tracker.WasIHaveRPCSent(testCase.topic, messageID)) + } + } }) } -func mockTracker() *RPCSentTracker { +func mockTracker(t *testing.T) *RPCSentTracker { logger := zerolog.Nop() - sizeLimit := uint32(100) + cfg, err := config.DefaultConfig() + require.NoError(t, err) collector := metrics.NewNoopCollector() - tracker := NewRPCSentTracker(logger, sizeLimit, collector) + tracker := NewRPCSentTracker(logger, cfg.NetworkConfig.GossipSubConfig.RPCSentTrackerCacheSize, collector) return tracker } From c66755b7956b703689cd668ab8d44608a96f807a Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 11 Jul 2023 11:30:31 -0600 Subject: [PATCH 134/169] fix mock bug --- engine/execution/testutil/fixtures.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index 21717f8da61..f85ff2f4f42 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -649,6 +649,6 @@ func ProtocolSnapshotWithSourceFixture(source []byte) protocol.Snapshot { func ProtocolStateWithSourceFixture(source []byte) protocol.State { snapshot := ProtocolSnapshotWithSourceFixture(source) state := protocolMock.State{} - state.On("AtBlockID", mock.Anything).Return(&snapshot) + state.On("AtBlockID", mock.Anything).Return(snapshot) return &state } From 67b43905f130db9d8d4e96d54a0b59f3679d51f5 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 11 Jul 2023 16:29:54 -0600 Subject: [PATCH 135/169] add error description for newSource() --- network/p2p/connection/connector_factory.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/network/p2p/connection/connector_factory.go b/network/p2p/connection/connector_factory.go index 4547d1cb301..10003895953 100644 --- a/network/p2p/connection/connector_factory.go +++ b/network/p2p/connection/connector_factory.go @@ -81,6 +81,11 @@ func (src *source) Int63() int64 { } // creates a source using a crypto PRG and secure random seed +// returned errors: +// - exception error if the system randomness fails (the system and other components would +// have many other issues if this happens) +// - exception error if the CSPRG (Chacha20) isn't initialized properly (should not happen in normal +// operations) func newSource() (*source, error) { seed := make([]byte, random.Chacha20SeedLen) _, err := rand.Read(seed) // checking err only is enough @@ -89,6 +94,7 @@ func newSource() (*source, error) { } prg, err := random.NewChacha20PRG(seed, nil) if err != nil { + // should not happen in normal operations because `seed` has the correct length return nil, fmt.Errorf("failed to generate a PRG: %w", err) } return &source{prg}, nil From 1f1a3410ac356b3b0b12a6d2563b0dbc1a64f078 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 11 Jul 2023 19:11:59 -0600 Subject: [PATCH 136/169] revert LRUEjector and remove random ejection fallback --- module/mempool/stdmap/eject.go | 68 ++++++++ module/mempool/stdmap/eject_test.go | 230 ++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 module/mempool/stdmap/eject_test.go diff --git a/module/mempool/stdmap/eject.go b/module/mempool/stdmap/eject.go index 2e52d7320bd..7cea5214b3d 100644 --- a/module/mempool/stdmap/eject.go +++ b/module/mempool/stdmap/eject.go @@ -4,7 +4,9 @@ package stdmap import ( "fmt" + "math" "sort" + "sync" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/rand" @@ -96,3 +98,69 @@ func EjectRandomFast(b *Backend) (bool, error) { func EjectPanic(b *Backend) (flow.Identifier, flow.Entity, bool) { panic("unexpected: mempool size over the limit") } + +// LRUEjector provides a swift FIFO ejection functionality +type LRUEjector struct { + sync.Mutex + table map[flow.Identifier]uint64 // keeps sequence number of entities it tracks + seqNum uint64 // keeps the most recent sequence number +} + +func NewLRUEjector() *LRUEjector { + return &LRUEjector{ + table: make(map[flow.Identifier]uint64), + seqNum: 0, + } +} + +// Track should be called every time a new entity is added to the mempool. +// It tracks the entity for later ejection. +func (q *LRUEjector) Track(entityID flow.Identifier) { + q.Lock() + defer q.Unlock() + + if _, ok := q.table[entityID]; ok { + // skips adding duplicate item + return + } + + // TODO current table structure provides O(1) track and untrack features + // however, the Eject functionality is asymptotically O(n). + // With proper resource cleanups by the mempools, the Eject is supposed + // as a very infrequent operation. However, further optimizations on + // Eject efficiency is needed. + q.table[entityID] = q.seqNum + q.seqNum++ +} + +// Untrack simply removes the tracker of the ejector off the entityID +func (q *LRUEjector) Untrack(entityID flow.Identifier) { + q.Lock() + defer q.Unlock() + + delete(q.table, entityID) +} + +// Eject implements EjectFunc for LRUEjector. It finds the entity with the lowest sequence number (i.e., +// the oldest entity). It also untracks. This is using a linear search +func (q *LRUEjector) Eject(b *Backend) flow.Identifier { + q.Lock() + defer q.Unlock() + + // finds the oldest entity + oldestSQ := uint64(math.MaxUint64) + var oldestID flow.Identifier + for _, id := range b.backData.Identifiers() { + if sq, ok := q.table[id]; ok { + if sq < oldestSQ { + oldestID = id + oldestSQ = sq + } + } + } + + // untracks the oldest id as it is supposed to be ejected + delete(q.table, oldestID) + + return oldestID +} diff --git a/module/mempool/stdmap/eject_test.go b/module/mempool/stdmap/eject_test.go new file mode 100644 index 00000000000..398c74938aa --- /dev/null +++ b/module/mempool/stdmap/eject_test.go @@ -0,0 +1,230 @@ +package stdmap + +import ( + crand "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestLRUEjector_Track evaluates that tracking a new item adds the item to the ejector table. +func TestLRUEjector_Track(t *testing.T) { + ejector := NewLRUEjector() + // ejector's table should be empty + assert.Len(t, ejector.table, 0) + + // sequence number of ejector should initially be zero + assert.Equal(t, ejector.seqNum, uint64(0)) + + // creates adds an item to the ejector + item := flow.Identifier{0x00} + ejector.Track(item) + + // size of ejector's table should be one + // which indicates that ejector is tracking the item + assert.Len(t, ejector.table, 1) + + // item should reside in the ejector's table + _, ok := ejector.table[item] + assert.True(t, ok) + + // sequence number of ejector should be increased by one + assert.Equal(t, ejector.seqNum, uint64(1)) +} + +// TestLRUEjector_Track_Duplicate evaluates that tracking a duplicate item +// does not change the internal state of the ejector. +func TestLRUEjector_Track_Duplicate(t *testing.T) { + ejector := NewLRUEjector() + + // creates adds an item to the ejector + item := flow.Identifier{0x00} + ejector.Track(item) + + // size of ejector's table should be one + // which indicates that ejector is tracking the item + assert.Len(t, ejector.table, 1) + + // item should reside in the ejector's table + _, ok := ejector.table[item] + assert.True(t, ok) + + // sequence number of ejector should be increased by one + assert.Equal(t, ejector.seqNum, uint64(1)) + + // adds the duplicate item + ejector.Track(item) + + // internal state of the ejector should be unchaged + assert.Len(t, ejector.table, 1) + assert.Equal(t, ejector.seqNum, uint64(1)) + _, ok = ejector.table[item] + assert.True(t, ok) +} + +// TestLRUEjector_Track_Many evaluates that tracking many items +// changes the state of ejector properly, i.e., items reside on the +// memory, and sequence number changed accordingly. +func TestLRUEjector_Track_Many(t *testing.T) { + ejector := NewLRUEjector() + + // creates and tracks 100 items + size := 100 + items := flow.IdentifierList{} + for i := 0; i < size; i++ { + var id flow.Identifier + _, _ = crand.Read(id[:]) + ejector.Track(id) + items = append(items, id) + } + + // size of ejector's table should be 100 + assert.Len(t, ejector.table, size) + + // all items should reside in the ejector's table + for _, id := range items { + _, ok := ejector.table[id] + require.True(t, ok) + } + + // sequence number of ejector should be increased by size + assert.Equal(t, ejector.seqNum, uint64(size)) +} + +// TestLRUEjector_Untrack_One evaluates that untracking an existing item +// removes it from the ejector state and changes the state accordingly. +func TestLRUEjector_Untrack_One(t *testing.T) { + ejector := NewLRUEjector() + + // creates adds an item to the ejector + item := flow.Identifier{0x00} + ejector.Track(item) + + // size of ejector's table should be one + // which indicates that ejector is tracking the item + assert.Len(t, ejector.table, 1) + + // item should reside in the ejector's table + _, ok := ejector.table[item] + assert.True(t, ok) + + // sequence number of ejector should be increased by one + assert.Equal(t, ejector.seqNum, uint64(1)) + + // untracks the item + ejector.Untrack(item) + + // internal state of the ejector should be changed + assert.Len(t, ejector.table, 0) + + // sequence number should not be changed + assert.Equal(t, ejector.seqNum, uint64(1)) + + // item should no longer reside on internal state of ejector + _, ok = ejector.table[item] + assert.False(t, ok) +} + +// TestLRUEjector_Untrack_Duplicate evaluates that untracking an item twice +// removes it from the ejector state only once and changes the state safely. +func TestLRUEjector_Untrack_Duplicate(t *testing.T) { + ejector := NewLRUEjector() + + // creates and adds two items to the ejector + item1 := flow.Identifier{0x00} + item2 := flow.Identifier{0x01} + ejector.Track(item1) + ejector.Track(item2) + + // size of ejector's table should be two + // which indicates that ejector is tracking the items + assert.Len(t, ejector.table, 2) + + // items should reside in the ejector's table + _, ok := ejector.table[item1] + assert.True(t, ok) + _, ok = ejector.table[item2] + assert.True(t, ok) + + // sequence number of ejector should be increased by two + assert.Equal(t, ejector.seqNum, uint64(2)) + + // untracks the item twice + ejector.Untrack(item1) + ejector.Untrack(item1) + + // internal state of the ejector should be changed + assert.Len(t, ejector.table, 1) + + // sequence number should not be changed + assert.Equal(t, ejector.seqNum, uint64(2)) + + // double untracking should only affect the untracked item1 + _, ok = ejector.table[item1] + assert.False(t, ok) + + // item 2 should still reside in the memory + _, ok = ejector.table[item2] + assert.True(t, ok) +} + +// TestLRUEjector_UntrackEject evaluates that untracking the next ejectable item +// properly changes the next ejectable item in the ejector. +func TestLRUEjector_UntrackEject(t *testing.T) { + ejector := NewLRUEjector() + + // creates and tracks 100 items + size := 100 + backEnd := NewBackend() + + items := make([]flow.Identifier, size) + + for i := 0; i < size; i++ { + mockEntity := unittest.MockEntityFixture() + require.True(t, backEnd.Add(mockEntity)) + + id := mockEntity.ID() + ejector.Track(id) + items[i] = id + } + + // untracks the oldest item + ejector.Untrack(items[0]) + + // next ejectable item should be the second oldest item + id := ejector.Eject(backEnd) + assert.Equal(t, id, items[1]) +} + +// TestLRUEjector_EjectAll adds many item to the ejector and then ejects them +// all one by one and evaluates an LRU ejection behavior. +func TestLRUEjector_EjectAll(t *testing.T) { + ejector := NewLRUEjector() + + // creates and tracks 100 items + size := 100 + backEnd := NewBackend() + + items := make([]flow.Identifier, size) + + for i := 0; i < size; i++ { + mockEntity := unittest.MockEntityFixture() + require.True(t, backEnd.Add(mockEntity)) + + id := mockEntity.ID() + ejector.Track(id) + items[i] = id + } + + require.Equal(t, uint(size), backEnd.Size()) + + // ejects one by one + for i := 0; i < size; i++ { + id := ejector.Eject(backEnd) + require.Equal(t, id, items[i]) + } +} From 183ce49f4019b910422809b924258923cb49f114 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 11 Jul 2023 19:24:53 -0600 Subject: [PATCH 137/169] use fatal logging when system randomness fails --- module/mempool/herocache/backdata/heropool/pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/mempool/herocache/backdata/heropool/pool.go b/module/mempool/herocache/backdata/heropool/pool.go index 1a84ee78fdd..92f926c2239 100644 --- a/module/mempool/herocache/backdata/heropool/pool.go +++ b/module/mempool/herocache/backdata/heropool/pool.go @@ -180,7 +180,7 @@ func (p *Pool) sliceIndexForEntity() (i EIndex, hasAvailableSlot bool, ejectedEn // we only eject randomly when the pool is full and random ejection is on. random, err := rand.Uint32n(p.size) if err != nil { - p.logger.Warn().Err(err). + p.logger.Fatal().Err(err). Msg("hero pool random ejection failed - falling back to LRU ejection") // fall back to LRU ejection only for this instance return lruEject() From 6ddbb2b43f070975baf9425f264bf636596fb6bd Mon Sep 17 00:00:00 2001 From: Supun Setunga Date: Tue, 11 Jul 2023 13:12:05 -0700 Subject: [PATCH 138/169] auto update to onflow/cadence v0.39.14 --- go.mod | 5 +++-- go.sum | 10 ++++++---- insecure/go.mod | 5 +++-- insecure/go.sum | 10 ++++++---- integration/go.mod | 5 +++-- integration/go.sum | 10 ++++++---- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index ef9c29a3a43..760c4c327ed 100644 --- a/go.mod +++ b/go.mod @@ -51,11 +51,11 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.6.0 - github.com/onflow/cadence v0.39.12 + github.com/onflow/cadence v0.39.14 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 - github.com/onflow/flow-go-sdk v0.41.6 + github.com/onflow/flow-go-sdk v0.41.8 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d @@ -227,6 +227,7 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onsi/ginkgo/v2 v2.9.7 // indirect diff --git a/go.sum b/go.sum index ecf5dfae9f4..4d3d0326089 100644 --- a/go.sum +++ b/go.sum @@ -1218,8 +1218,10 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= -github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= -github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= +github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= @@ -1228,8 +1230,8 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+K github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= -github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= +github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= +github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= diff --git a/insecure/go.mod b/insecure/go.mod index fba888f2997..aec63ed4b7b 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -181,11 +181,12 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect - github.com/onflow/cadence v0.39.12 // indirect + github.com/onflow/cadence v0.39.14 // indirect + github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect - github.com/onflow/flow-go-sdk v0.41.6 // indirect + github.com/onflow/flow-go-sdk v0.41.8 // indirect github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 82276186e6f..439a76760f0 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1172,16 +1172,18 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= -github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= -github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= +github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= -github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= +github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= +github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= diff --git a/integration/go.mod b/integration/go.mod index f59d02427b4..0a80f6dbfd3 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -17,12 +17,12 @@ require ( github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.3.0 - github.com/onflow/cadence v0.39.12 + github.com/onflow/cadence v0.39.14 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-emulator v0.50.6 github.com/onflow/flow-go v0.31.1-0.20230607185125-e75265a6c631 - github.com/onflow/flow-go-sdk v0.41.6 + github.com/onflow/flow-go-sdk v0.41.8 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 @@ -227,6 +227,7 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect + github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4 // indirect github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e // indirect diff --git a/integration/go.sum b/integration/go.sum index 0b3079bed0f..1cee24f6dfb 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1354,8 +1354,10 @@ github.com/onflow/atree v0.1.0-beta1.0.20211027184039-559ee654ece9/go.mod h1:+6x github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= -github.com/onflow/cadence v0.39.12 h1:bb3UdOe7nClUcaLbxSWGLSIJKuCrivpgxhPow99ikv0= -github.com/onflow/cadence v0.39.12/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= +github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= +github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= +github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= @@ -1365,8 +1367,8 @@ github.com/onflow/flow-emulator v0.50.6/go.mod h1:0avs83tvFDt8vyMcm4AYOcHDSRJHY5 github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.41.6 h1:x5HhmRDvbCWXRCzHITJxOp0Komq5JJ9zphoR2u6NOCg= -github.com/onflow/flow-go-sdk v0.41.6/go.mod h1:AYypQvn6ecMONhF3M1vBOUX9b4oHKFWkkrw8bO4VEik= +github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= +github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= From 82d6e5f45ca11b1d3dfca02c678a41e8c728fa4b Mon Sep 17 00:00:00 2001 From: Supun Setunga Date: Wed, 12 Jul 2023 11:20:38 -0700 Subject: [PATCH 139/169] Update to flow-go-sdk@v0.41.9 version --- go.mod | 3 +-- go.sum | 6 ++---- insecure/go.mod | 3 +-- insecure/go.sum | 6 ++---- integration/go.mod | 3 +-- integration/go.sum | 6 ++---- 6 files changed, 9 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 760c4c327ed..1f2fb8c4a53 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 - github.com/onflow/flow-go-sdk v0.41.8 + github.com/onflow/flow-go-sdk v0.41.9 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d @@ -227,7 +227,6 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/sdks v0.5.0 // indirect github.com/onsi/ginkgo/v2 v2.9.7 // indirect diff --git a/go.sum b/go.sum index 4d3d0326089..fc28af16c8b 100644 --- a/go.sum +++ b/go.sum @@ -1220,8 +1220,6 @@ github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= -github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= -github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= @@ -1230,8 +1228,8 @@ github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+K github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= -github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= diff --git a/insecure/go.mod b/insecure/go.mod index aec63ed4b7b..94bb5ad0881 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -182,11 +182,10 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect github.com/onflow/cadence v0.39.14 // indirect - github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect - github.com/onflow/flow-go-sdk v0.41.8 // indirect + github.com/onflow/flow-go-sdk v0.41.9 // indirect github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/sdks v0.5.0 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 439a76760f0..e18fd0b1029 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1174,16 +1174,14 @@ github.com/onflow/atree v0.6.0 h1:j7nQ2r8npznx4NX39zPpBYHmdy45f4xwoi+dm37Jk7c= github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVFi0tc= github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= -github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= -github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= -github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= -github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= diff --git a/integration/go.mod b/integration/go.mod index 0a80f6dbfd3..cbd6c8d457f 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -22,7 +22,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 github.com/onflow/flow-emulator v0.50.6 github.com/onflow/flow-go v0.31.1-0.20230607185125-e75265a6c631 - github.com/onflow/flow-go-sdk v0.41.8 + github.com/onflow/flow-go-sdk v0.41.9 github.com/onflow/flow-go/crypto v0.24.7 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 @@ -227,7 +227,6 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/onflow/atree v0.6.0 // indirect - github.com/onflow/crypto v0.24.9 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v0.0.0-20220727161549-d59b1e547ac4 // indirect github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e // indirect diff --git a/integration/go.sum b/integration/go.sum index 1cee24f6dfb..5b3a101da9b 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1356,8 +1356,6 @@ github.com/onflow/atree v0.6.0/go.mod h1:gBHU0M05qCbv9NN0kijLWMgC47gHVNBIp4KmsVF github.com/onflow/cadence v0.20.1/go.mod h1:7mzUvPZUIJztIbr9eTvs+fQjWWHTF8veC+yk4ihcNIA= github.com/onflow/cadence v0.39.14 h1:YoR3YFUga49rqzVY1xwI6I2ZDBmvwGh13jENncsleC8= github.com/onflow/cadence v0.39.14/go.mod h1:OIJLyVBPa339DCBQXBfGaorT4tBjQh9gSKe+ZAIyyh0= -github.com/onflow/crypto v0.24.9 h1:jYP1qdwid0qCineFzBFlxBchg710A7RuSWpTqxaOdog= -github.com/onflow/crypto v0.24.9/go.mod h1:J/V7IEVaqjDajvF8K0B/SJPJDgAOP2G+LVLeb0hgmbg= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3 h1:wV+gcgOY0oJK4HLZQYQoK+mm09rW1XSxf83yqJwj0n4= github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.3/go.mod h1:Osvy81E/+tscQM+d3kRFjktcIcZj2bmQ9ESqRQWDEx8= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= @@ -1367,8 +1365,8 @@ github.com/onflow/flow-emulator v0.50.6/go.mod h1:0avs83tvFDt8vyMcm4AYOcHDSRJHY5 github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= -github.com/onflow/flow-go-sdk v0.41.8 h1:Anfj7lK3YM53qqomrkdkD9F5oOost1LUPrk40k3DYeI= -github.com/onflow/flow-go-sdk v0.41.8/go.mod h1:QNEJ8amKeIZZWAvo7I2Mn/o0sPQ21H1iEdox0t94anY= +github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= +github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= github.com/onflow/flow-go/crypto v0.24.7 h1:RCLuB83At4z5wkAyUCF7MYEnPoIIOHghJaODuJyEoW0= github.com/onflow/flow-go/crypto v0.24.7/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= From a2f5f4b221cf6dd10c28d415509b89a07b9f05ce Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 12 Jul 2023 15:10:11 -0600 Subject: [PATCH 140/169] update grep command to work for different grep versions and remove tmate --- .github/workflows/ci.yml | 2 -- Makefile | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc57f51ee31..5772ef5dcb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,8 +64,6 @@ jobs: with: go-version: ${{ env.GO_VERSION }} cache: true - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - name: Run tidy run: make tidy - name: code sanity check diff --git a/Makefile b/Makefile index 741862396f2..003a0a6a84b 100644 --- a/Makefile +++ b/Makefile @@ -94,8 +94,9 @@ go-math-rand-check: # If this check fails, try updating your code by using: # - "crypto/rand" or "flow-go/utils/rand" for non-deterministic randomness # - "flow-go/crypto/random" for deterministic randomness - grep --include=\*.go --exclude=*{test,helper,example,fixture,benchmark,profiler}* \ - --exclude-dir=*{test,helper,example,fixture,benchmark,profiler}* -rnw '"math/rand"'; \ + grep --include=\*.go \ + --exclude=*test* --exclude=*helper* --exclude=*example* --exclude=*fixture* --exclude=*benchmark* --exclude=*profiler* \ + --exclude-dir=*test* --exclude-dir=*helper* --exclude-dir=*example* --exclude-dir=*fixture* --exclude-dir=*benchmark* --exclude-dir=*profiler* -rnw '"math/rand"'; \ if [ $$? -ne 1 ]; then \ echo "[Error] Go production code should not use math/rand package"; exit 1; \ fi From 117fcc06984a5de4d21c015407a40a7c458ff1c8 Mon Sep 17 00:00:00 2001 From: "Yahya Hassanzadeh, Ph.D" Date: Wed, 12 Jul 2023 14:43:11 -0700 Subject: [PATCH 141/169] Verification Node documentation (#4528) * adds readme for block consumer * adds readme for assigner engine * adds documetation for chunk consumer * adds documentation for fetcher engine * adds documentation for requester engine * adds documentation for verifier engine * adds the architecture overview * Update engine/verification/Readme.md Co-authored-by: Leo Zhang --------- Co-authored-by: Leo Zhang --- engine/Readme.md | 1 - engine/verification/Readme.md | 170 +++++++++++++++++++++++++++ engine/verification/architecture.png | Bin 0 -> 489749 bytes 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 engine/verification/Readme.md create mode 100644 engine/verification/architecture.png diff --git a/engine/Readme.md b/engine/Readme.md index 8faebe0b332..cd082cdf557 100644 --- a/engine/Readme.md +++ b/engine/Readme.md @@ -1,5 +1,4 @@ # Notifier - The Notifier implements the following state machine ![Notifier State Machine](/docs/NotifierStateMachine.png) diff --git a/engine/verification/Readme.md b/engine/verification/Readme.md new file mode 100644 index 00000000000..ff527a432b0 --- /dev/null +++ b/engine/verification/Readme.md @@ -0,0 +1,170 @@ +# Verification Node +The Verification Node in the Flow blockchain network is a critical component responsible for +verifying `ExecutionResult`s and generating `ResultApproval`s. +Its primary role is to ensure the integrity and validity of block execution by performing verification processes. +In a nutshell, the Verification Node is responsible for the following: +1. Following the chain for new finalized blocks (`Follower` engine). +2. Processing the execution results in the finalized blocks and determining assigned chunks to the node (`Assigner` engine). +3. Requesting chunk data pack from Execution Nodes for the assigned chunks (`Fetcher` and `Requester` engines). +4. Verifying the assigned chunks and emitting `ResultApproval`s for the verified chunks to Consensus Nodes (`Verifier` engine). +![architecture.png](architecture.png) + + +## Block Consumer ([consumer.go](verification%2Fassigner%2Fblockconsumer%2Fconsumer.go)) +The `blockconsumer` package efficiently manages the processing of finalized blocks in Verification Node of Flow blockchain. +Specifically, it listens for notifications from the `Follower` engine regarding finalized blocks, and systematically +queues these blocks for processing. The package employs parallel workers, each an instance of the `Assigner` engine, +to fetch and process blocks from the queue. The `BlockConsumer` diligently coordinates this process by only assigning +a new block to a worker once it has completed processing its current block and signaled its availability. +This ensures that the processing is not only methodical but also resilient to any node crashes. +In case of a crash, the `BlockConsumer` resumes from where it left off by reading the processed block index from storage, reassigning blocks from the queue to workers, +thereby guaranteeing no loss of data. + +## Assigner Engine +The `Assigner` [engine](verification%2Fassigner%2Fengine.go) is an integral part of the verification process in Flow, +focusing on processing the execution results in the finalized blocks, performing chunk assignments on the results, and +queuing the assigned chunks for further processing. The Assigner engine is a worker of the `BlockConsumer` engine, +which assigns finalized blocks to the Assigner engine for processing. +This engine reads execution receipts included in each finalized block, +determines which chunks are assigned to the node for verification, +and stores the assigned chunks into the chunks queue for further processing (by the `Fetcher` engine). + +The core behavior of the Assigner engine is implemented in the `ProcessFinalizedBlock` function. +This function initiates the process of execution receipt indexing, chunk assignment, and processing the assigned chunks. +For every receipt in the block, the engine determines chunk assignments using the verifiable chunk assignment algorithm of Flow. +Each assigned chunk is then processed by the `processChunk` method. This method is responsible for storing a chunk locator in the chunks queue, +which is a crucial step for further processing of the chunks by the fetcher engine. +Deduplication of chunk locators is handled by the chunks queue. +The Assigner engine provides robustness by handling the situation where a node is not authorized at a specific block ID. +It verifies the role of the result executor, checks if the node has been ejected, and assesses the node's staked weight before granting authorization. +Lastly, once the Assigner engine has completed processing the receipts in a block, it sends a notification to the block consumer. This is inline with +Assigner engine as a worker of the block consumer informing the consumer that it is ready to process the next block. +This ensures a smooth and efficient flow of data in the system, promoting consistency across different parts of the Flow architecture. + +### Chunk Locator +A chunk locator in the Flow blockchain is an internal structure of the Verification Nodes that points to a specific chunk +within a specific execution result of a block. It's an important part of the verification process in the Flow network, +allowing verification nodes to efficiently identify, retrieve, and verify individual chunks of computation. + +```go +type ChunkLocator struct { + ResultID flow.Identifier // The identifier of the ExecutionResult + Index uint64 // Index of the chunk +} +``` +- `ResultID`: This is the identifier of the execution result that the chunk is a part of. The execution result contains a list of chunks which each represent a portion of the computation carried out by execution nodes. Each execution result is linked to a specific block in the blockchain. +- `Index`: This is the index of the chunk within the execution result's list of chunks. It's an easy way to refer to a specific chunk within a specific execution result. + +**Note-1**: The `ChunkLocator` doesn't contain the chunk itself but points to where the chunk can be found. In the context of the `Assigner` engine, the `ChunkLocator` is stored in a queue after chunk assignment is done, so the `Fetcher` engine can later retrieve the chunk for verification. +**Note-2**: The `ChunkLocator` is never meant to be sent over the networking layer to another Flow node. It's an internal structure of the verification nodes, and it's only used for internal communication between the `Assigner` and `Fetcher` engines. + + +## ChunkConsumer +The `ChunkConsumer` ([consumer](verification%2Ffetcher%2Fchunkconsumer%2Fconsumer.go)) package orchestrates the processing of chunks in the Verification Node of the Flow blockchain. +Specifically, it keeps tabs on chunks that are assigned for processing by the `Assigner` engine and systematically enqueues these chunks for further handling. +To expedite the processing, the package deploys parallel workers, with each worker being an instance of the `Fetcher` engine, which retrieves and processes the chunks from the queue. +The `ChunkConsumer` administers this process by ensuring that a new chunk is assigned to a worker only after it has finalized processing its current chunk and signaled that it is ready for more. +This systematic approach guarantees not only efficiency but also robustness against any node failures. In an event where a node crashes, +the `ChunkConsumer` picks up right where it left, redistributing chunks from the queue to the workers, ensuring that there is no loss of data or progress. + +## Fetcher Engine - The Journey of a `ChunkLocator` to a `VerifiableChunkData` +The Fetcher [engine.go](fetcher%2Fengine.go) of the Verification Nodes focuses on the lifecycle of a `ChunkLocator` as it transitions into a `VerifiableChunkData`. + +### `VerifiableChunkData` +`VerifiableChunkData` refers to a data structure that encapsulates all the necessary components and resources required to +verify a chunk within the Flow blockchain network. It represents a chunk that has undergone processing and is ready for verification. + +The `VerifiableChunkData` object contains the following key elements: +```go +type VerifiableChunkData struct { + IsSystemChunk bool // indicates whether this is a system chunk + Chunk *flow.Chunk // the chunk to be verified + Header *flow.Header // BlockHeader that contains this chunk + Result *flow.ExecutionResult // execution result of this block + ChunkDataPack *flow.ChunkDataPack // chunk data package needed to verify this chunk + EndState flow.StateCommitment // state commitment at the end of this chunk + TransactionOffset uint32 // index of the first transaction in a chunk within a block +} +``` +1. `IsSystemChunk`: A boolean value that indicates whether the chunk is a system chunk. System chunk is a specific chunk typically representing the last chunk within an execution result. +2. `Chunk`: The actual chunk that needs to be verified. It contains the relevant data and instructions related to the execution of transactions within the blockchain. +3. `Header`: The `BlockHeader` associated with the chunk. It provides important contextual information about the block that the chunk belongs to. +4. `Result`: The `ExecutionResult` object that corresponds to the execution of the block containing the chunk. It contains information about the execution status, including any errors or exceptions encountered during the execution process. +5. `ChunkDataPack`: The `ChunkDataPack`, which is a package containing additional data and resources specific to the chunk being verified. It provides supplementary information required for the verification process. +6. `EndState`: The state commitment at the end of the chunk. It represents the final state of the blockchain after executing all the transactions within the chunk. +7. `TransactionOffset`: An index indicating the position of the first transaction within the chunk in relation to the entire block. This offset helps in locating and tracking individual transactions within the blockchain. +By combining these elements, the VerifiableChunkData object forms a comprehensive representation of a chunk ready for verification. It serves as an input to the `Verifier` engine, which utilizes this data to perform the necessary checks and validations to ensure the integrity and correctness of the chunk within the Flow blockchain network. + +### The Journey of a `ChunkLocator` to a `VerifiableChunkData` +Upon receiving the `ChunkLocator`, the `Fetcher` engine’s `validateAuthorizedExecutionNodeAtBlockID` function is responsible +for validating the authenticity of the sender. It evaluates whether the sender is an authorized execution node for the respective block. +The function cross-references the sender’s credentials against the state snapshot of the specific block. +In the case of unauthorized or invalid credentials, an error is logged, and the `ChunkLocator` is rejected. +For authorized credentials, the processing continues. + +Once authenticated, the `ChunkLocator` is utilized to retrieve the associated Chunk Data Pack. +The `requestChunkDataPack` function takes the Chunk Locator and generates a `ChunkDataPackRequest`. +During this stage, the function segregates execution nodes into two categories - those which agree with the execution result (`agrees`) and those which do not (`disagrees`). +This information is encapsulated within the `ChunkDataPackRequest` and is forwarded to the `Requester` Engine. +The `Requester` Engine handles the retrieval of the `ChunkDataPack` from the network of execution nodes. + +After the Chunk Data Pack is successfully retrieved by the `Requester` Engine, +the next phase involves structuring this data for verification and constructing a `VerifiableChunkData`. +It’s imperative that this construction is performed with utmost accuracy to ensure that the data is in a state that can be properly verified. + +The final step in the lifecycle is forwarding the `VerifiableChunkData` to the `Verifier` Engine. The `Verifier` Engine is tasked with the critical function +of thoroughly analyzing and verifying the data. Depending on the outcome of this verification process, +the chunk may either pass verification successfully or be rejected due to discrepancies. + +### Handling Sealed Chunks +In parallel, the `Fetcher` engine remains vigilant regarding the sealed status of chunks. +The `NotifyChunkDataPackSealed` function monitors the sealing status. +If the Consensus Nodes seal a chunk, this function ensures that the `Fetcher` Engine acknowledges this update and discards the respective +`ChunkDataPack` from its processing pipeline as it is now sealed (i.e., has been verified by an acceptable quota of Verification Nodes). + +## Requester Engine - Retrieving the `ChunkDataPack` +The `Requester` [engine](requester%2Frequester.go) is responsible for handling the request and retrieval of chunk data packs in the Flow blockchain network. +It acts as an intermediary between the `Fetcher` engine and the Execution Nodes, facilitating the communication and coordination required +to obtain the necessary `ChunkDataPack` for verification. + +The `Requester` engine receives `ChunkDataPackRequest`s from the `Fetcher`. +These requests contain information such as the chunk ID, block height, agree and disagree executors, and other relevant details. +Upon receiving a `ChunkDataPackRequest`, the `Requester` engine adds it to the pending requests cache for tracking and further processing. +The Requester engine periodically checks the pending chunk data pack requests and dispatches them to the Execution Nodes for retrieval. +It ensures that only qualified requests are dispatched based on certain criteria, such as the chunk ID and request history. +The dispatching process involves creating a `ChunkDataRequest` message and publishing it to the network. +The request is sent to a selected number of Execution Nodes, determined by the `requestTargets` parameter. + +When an Execution Node receives a `ChunkDataPackRequest`, it processes the request and generates a `ChunkDataResponse` +message containing the requested chunk data pack. The execution node sends this response back to the`Requester` engine. +The `Requester` engine receives the chunk data pack response, verifies its integrity, and passes it to the registered `ChunkDataPackHandler`, +i.e., the `Fetcher` engine. + +### Retry and Backoff Mechanism +In case a `ChunkDataPackRequest` does not receive a response within a certain period, the `Requester` engine retries the request to ensure data retrieval. +It implements an exponential backoff mechanism for retrying failed requests. +The retry interval, backoff multiplier, and backoff intervals can be customized using the respective configuration parameters. + +### Handling Sealed Blocks +If a `ChunkDataPackRequest` pertains to a block that has already been sealed, the `Requester` engine recognizes this and +removes the corresponding request from the pending requests cache. +It notifies the `ChunkDataPackHandler` (i.e., the `Fetcher` engine) about the sealing of the block to ensure proper handling. + +### Parallel Chunk Data Pack Retrieval +The `Requester` processes a number of chunk data pack requests in parallel, +dispatching them to execution nodes and handling the received responses. +However, it is important to note that if a chunk data pack request does not receive a response from the execution nodes, +the `Requester` engine can become stuck in processing, waiting for the missing chunk data pack. +To mitigate this, the engine implements a retry and backoff mechanism, ensuring that requests are retried and backed off if necessary. +This mechanism helps to prevent prolonged waiting and allows the engine to continue processing other requests while waiting for the missing chunk data pack response. + +## Verifier Engine - Verifying Chunks +The `Verifier` [engine](verifier%2Fengine.go) is responsible for verifying chunks, generating `ResultApproval`s, and maintaining a cache of `ResultApproval`s. +It receives verifiable chunks along with the necessary data for verification, verifies the chunks by constructing a partial trie, +executing transactions, and checking the final state commitment and other chunk metadata. +If the verification is successful, it generates a `ResultApproval` and broadcasts it to the consensus nodes. + +The `Verifier` Engine offers the following key features: +1. **Verification of Chunks**: The engine receives verifiable chunks, which include the chunk to be verified, the associated header, execution result, and chunk data pack. It performs the verification process, which involves constructing a partial trie, executing transactions, and checking the final state commitment. The verification process ensures the integrity and validity of the chunk. +2. **Generation of Result Approvals**: If the verification process is successful, the engine generates a result approval for the verified chunk. The result approval includes the block ID, execution result ID, chunk index, attestation, approver ID, attestation signature, and SPoCK (Secure Proof of Confidential Knowledge) signature. The result approval provides a cryptographic proof of the chunk's validity and is used to seal the block. +3. **Cache of Result Approvals**: The engine maintains a cache of result approvals for efficient retrieval and lookup. The result approvals are stored in a storage module, allowing quick access to the approvals associated with specific chunks and execution results. diff --git a/engine/verification/architecture.png b/engine/verification/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..a1a16dec61b58fd83c91bd412194130a668fafb1 GIT binary patch literal 489749 zcmeFZXH*nT*ER}*BuN5FW)M(7B_kOIBnt=#2ofacFeCv<0|JsW3P{dDqC_POL6T&U zIOLo&0|OJzpwIKX_d4HN=X^iDKkqrOwPw+F?Y*nJs;hd}wXfab8ZQ+|i0Fv0u&_v! zmE>MwVd0-(Vc~KU;9<@TFTFFt!orrdm6g>{mX&4Ja0OY}I#^<1am1UOm{2G^<>)mv zH8JTOe8NfO>iH@xEb5gBc&M$Ld9baMxs5q9#n5nxhI|P}y8|mvv%SHdIEwk|8+Y&1 z64~KgB6hjQs03)&%3&5|ObOF%=YstE>^R#%>GT9c4egG$wiGOjC&|fF0n=R4^-6a~ zG^Jt0ASZ*k1nWv21l?4Ex}58QhqT`%pbU`7;&#O<9_eompTXePUVyXnF2>7Vnx z^K88-0p`lLAC}yE$UG|w$^OP6AekrEewXHX2~pZiRsnXmU_xo zYHC=IF~;RKsbl^p$ztQ~?kvW`_=*J>cEXWblsJqmEjm^K5|5^DjMG2mpd;dpY{O#yJXE8l3MI^!VpHY({(p)KZ z#ln)tQkIj^_QKw4A#4We>O5*ZY&mRc9>6gr7bGC0k(Z`oBOu~RwEU{{VD|CrCm)PL zUfbH*+rNX_%|IR-s7P}a)#Gaa#7_HW!BKaQiuP65M;Tse;u?*(IKq=|A9PD=9jpO1 zi(YP7qmx+^?-Tb?^Eq^DMbp7ruE9g|-caa5tIZ(luxIk~-N^EKYyLQi`T+}&Prrp} zD=&S+Gp5f_gUL<{e-n`!M;`?}#jRbvILB13fq^#E?e+AUFUFVsXuAOpR1!D|tOM2t zJC+Ba>rf+C{de*wv-H8cn`B5tK^E}*)Ds;}2zfbj2spc@MxvA<;PFGi`L!+*^#%g& z*|_cqP`-!$_M|)HJW-rSo#+B(3b@s}i5q>(} z5hejtBD~{lh0>>$OPq*xa0G<&tG3m{x23P()}o_$VyX=elpl=%yIQ?y=} z47_O8w%qunD)iC}=zxtG_%Rd_Y`6;wk3-zX53$BPM z*1sPA$EX4vpMadSp7XM4bFsWD)3WT_)Z0xrx}Ad&Q6QB{PlS_l)lY9wa=E7V-B#;t z_$p3m~9mu-4$#{cWu8pK5;^v&e9HJIY1)S?_S`+p~zruTb!{ULf@J zk(AW)YnGTS7dqi(_g~_7H->&okxxT8ondGh63L&InQySi&A!4@e`2rU-V2~WNw>!a zss*s1ah^s1yL*qw+Hw&%ht8^2Cd}OL6);1TOqzQo%O;`Z{0gbYfhSZ7IiMpA+u<>{ zxWr#kIH#6cNf~I9F1y<0Sjs;Sodgh*mjlD(2xHkoU!G41gyQ60eZ8W8z`xSzRSsmJ z>aoDH>dknC)Ik;m6{9E!$w<<3e~XE`mqJLRF`_B}S`;>f5a2RXrg!{s9dIijA;upeyg_>rOu^FwLIz97!ZL@Zl9EZtJG?;FR1VIIaj`XuPE;5 zr4zD2jjA&Vnd(I|;FFbvJt1SI92&jmedoAO@f%g7* zltlObo;Ab061{ta3004+4{b1y@gJ#A;7?$C(tk<)-epH3X0)u%sV1_Hq5}%Y7@rD~ z?`{IhfwidT%w-JvU|VFtxo&wW!QFC*gr=8r^p71&cji?~o!!{B?&_T|T%#e<^bBn< z5l4_VQu9vt5yrUDhK)MRzu-s`l=_ zm*3aAwl#b~`r5~-u!xV{s%|)9wg{HEA>4Qd_soh%SK8@2 z*fvSvyxC$@qpk=|VAxLOgOxt2?>Jj$jj3swdc`_OmtHXi90+M*-zP9(%t_@=><&nq ze!_T|h*W_&_sTxZi%(`BOH2|pq{j2~cPUEDf-x`9IJ9rA6oRXGu8S$crO`W;*0=Fl z$u*JB<$iqwt%%qr5IG?Kd5Og=J7)8hO zR#ihh;N@_d+xVS3b@$aHyD~3=GeAG_Gi?bv-X*TV__V;hBCeSuSMT|W1=5AXLfdVN zOMo;CUx3bATVe{78b*$Tc*ndVB+t(AgJdu9NjnxZ+WjnClSpHq(f{mR68!xeKuiiZ zUSdYOSvvz#845bh|Dk}Q7U@OL(atG8Qrte$tC5i)ekrSlY+s5)-zGf)dg5Kg_i2hr zOgfO01&wRmT*?|aFbFaH->3<+5(1Q#wNuwW_@x<)MOTMSSDsEz$MoQ!=|_1YSNbj>rn3Ckt?m;;V(2!IOXT; z{pB|=Y5g`EWM9D6jO?yDR?fL~Z1lmsy3wms0P4!k9#pL-$1xQ&t-bqf z%b;7|9v4STFQ{)S&BlM345+`$cmj#;O?iHd-xF?Le$9dAKz}hWMiryJ(3h#-)d$-k z11Y{uir1p1!=gBAQ30zzt0(XR9gX}I@IdFhvvWVbj-I6AJv&*Otc983)(g(M> ztmy4Z-DsP}K$yI_tc}+Zhepv zWtIw5w)YjT>IP=n4fBnx>~dMPfe6Y{G);xZi6@*@PI+=zF&@KR%e!xBV;(%U?#jZv zA1imrUKBIdm3_>9Ql^RI+M@bqi!`m^f8TS&HUW4Fn!>#w>`{y=MSTf6>Pi3ddJb5u zfyxk2z5N1C9d1$L;Te#GS)JpR?3%sIw%I2De~PhjeuXlnK3FdI_m%xy)8F2?=gIJ} zN#lu!{E0Tml5=+ctv!Z#Etg5mps1Gvffc*M$8u61DDD9333^EsQyhd0C!VU{Wl<5< z({FqfQ5B-Eu>1TlNIe4AU=VUQFu+DYeiL)>Vxf1U_I zE8yPxN2}22ntt?u%gcZz5`%qMAON-_M>^oh{U33rKbE0(lcBQbfn~G_G~Hh85nO75 z;7}5*dn0#fH3-4aDFWi3NKM2iq4^rOG$VlP09?iJdq2WXK*Mys?5suFui(_=C8!bc z4%6OGDMTUW^895t&&cxKF5M(BtDH92yHvW6MzkQI6d0g%9=dRsRJLL*Nkil{Tjz^& z#dCZnO<(zQ_UCfvMMXpw5%F4&ORjLjw82}9%e-qZUK0K>sEu*DmoNsk!7Tq<2GbCv=RNHM+~fro2PwkPxJ z6yOM{KAd`bT1F|I?WWJ1x}UBcPjYmNs4Lq?@wIgc)JW zSU(ep$C(nYoS{llb4(ZWIX&6&%NX-*0ltz%Wiz3-oudh--_*3|@=YE+ddIIk&}}cG z@A|gh_C*lYz|Z`bkHTlXNEB6u`i0bN<36OO{O)7pqpw`H>3cs@$Vu{wmoiZ-mCb7P z83UIn4^*%~hRDk76?@`OhvtuY4>v8-3I&zgRaJK^5>le4oNL#b9mbRX=VJb+Xa47z z`ENa&g4TuBzK4U{eYMP(a1GHr(tBg#z?XZmYQ78(K{X3T2dmSZ-FM2NWUo%eUrUD- zQ&h&#HCt?FKXkFCx3c0vFcK*az3eBT0MhiK8E=b|Xy2-g`GO$BQp1loo2w>a?NmG7 z)M&(?De-w_ri5VDK!_4~&=uppWqj5J!1_YU($#2AukbN#WjM!~?3>m_{?E8ene0n9 zsy#ss!e%kM2q2aqqg;ebk5Ck`a95V>GLuEofpYkR;~?qdO^#{XYznV;zp21jHez8^ zt1*r=v`!Jg&<@Gom-C4fvMcz3E5n*ql6EJjWu5S-s@xWEXcITN0Q!FyiA$<=?Uy{DtO)0?A3zFEgXP1l5lH z$3_2wVpc_S9(_#EOaceAakg{X-1}?vFKqM}?frk$>LA<`@gP!1HZh6|!Ean$hQni^ zEw%TtPAf=l{Bfx}9oyYmEoi<=sH>j%vuVMy+!sYM6giT0YIOGgK^>^EjI6*2t)P(J z*vF-U=XoUiC5}P{K9FY_UyH}g*$8q&Ky3HIr}Jj9gG)(0C`1XZa}f*zb>~QO-uw|Ji1?mmlpF% z{?p4Kw&y-}X@Z~8v{WaMWv-VbLW~6wk2CO;gJHjgY@eb9IM%?O2I+JTVlVzrUpV*> zwOKjY3@NQLVOMYs!BPu=EgKs7e%+OV@Vpg_=6EChqF#ka^qByo3jziC4;Veq4 z_qv&(mvHa_Vbc~)$&oPm9WS|tur=^}@-0mLbN0^U$8Vh$u8GBpt0JZ1deOu`Z!wK> zVSvA;m=o|Np5#wy7k;yntH{m?K6-ZzQ))kEO$2S&ZHZh1xd@;qW0&#&CuK6!V=il= z5tjs~IBV>)>zB2%$@jOhPKBgEA{}f^p1i$0E3xEW9N%A@iW7%d9Fa9-clM&Gz;3}D z=rfBPMB1Bi3AuB26@30~69)Q|Qpg!?JvNo-iXhhSEA}eo;3Tj()h|dFdlFc2M+3#U zJwE01+1y9+71DH|N(UJjNltR5EEZ+^Jo&zD8Nz^96UA7OhY;^jgin!cAWfZ%QT*25 zbTC-r;6wSoG{5#z%x2a2t!ye!)HVhMzfD;Vd}G!rlOi?eS&L#8(FgNdO)v26l0kd3 z@Y%!>RZek@x*N9kG~Z8Ovc~g8vrDX+woZsgR|xJU43V~M0QA_-UJ{@q@WIRDQ)cc@ z1me-OG%z4S>6m&{&@3Wm>|q9@_j*>Ke6;XsfpX4ennZXMW;@H1P>GCn)>gED|Bs<&ZF3eI&~KSH1-?Z8?yH_!{7ne?>oIu|_5KBH7(z>k}CNey_kPrt9=L z(4{EV)ZnP^y+zNF$GW>b5?V`^l&qExj$W<{Ts9jfw3Ik<1^i2RA4f=}*3KCMi{XJm zN6BcussVZ`a6AKl8m^H4Bh_EPkMV&y2>RAB@JU$QvEW$e_>kVFPd^-oU|Y3$B$v(+m;>x93?xy9NjFo{8cx*uFbOcwH2o4 z!Y-8NNtk_J^v7OKbcZlzH$^r&>E=lv%RXT|VeElqYNrkE5Jsp9yM#q)Vutnh zt$7|AQEpyZxXteO&JE>%itC01B=WHv{nwAHCZDndm ziSYVro0UbfOW7Eoe4Y=0{h8%7cTm%+=7^+yJg}#MO9W)==x_L3uLmnqvetjuPmaDq z{kqQu?J<+3fp`WvI^=^9hJ)c*2zAtEPSK?-fAQ@(uSsy+$Qy%QBo-+VP#Y~yiv3#A zQqb=`mELmMxuRFn3fWNhB|YKKBMsDXm2{>CN(Y1Mo~H*mA7Ox&pa%lMvixJ#Mz*+y zBDy8xH_!@p&&bu2TyxyGpoGzaKX_ojoGDjd6ghSDQOT!nog9G0ZnJ!%P8-k)@wp(NbGDum>-U-sVO zCn*V^s%X*O_fhR7WA&8NL5nqLC#mC65A$oiCV!XO7WB{P{U7tY(NbeE^@;xNjR~jq z!*-qvl3(Xnk20Vc4a2sZP8svu_2AHYvN}e3>;93>s>{tqE9V^m3<)iOvBtw2SB7VH zH|eUn&X=NsvW)9i@70j>do!6=foC7dT(dayd!@TnK{fy)@n-Z&6K*!8Qrm1c#<(x> zVg*Q-0T|fvV^*A*O0WX`mr`{xLVfDM#P-!F**}2f|Bq{6BH}_PTIq6!SZe4W(xnMY z*|l{ZUd~A3&bI4nH_)b4!EgstR*xqxY>%!2q&TYr@-_0A85OSG^^~$B^O-J`Y*=b= zhy)KmYG8!7IFd%R*7E_H&gvJN=SedprllJEysy91FwHUf4q5Jx9JJR*OI37rU9>jG zsvjY*pWsUcUfTPg;x8W_Ls?m6**^S8AJ6WeJfVVlIKA$9jp)Oy@)1DUfK+qxhZg~= zLEOguTGP<)Zv;NlVgfE2EDayrLVe^j8Fw~iOhKCpmz0@5f8(A2;aLyjD&lc7@*Db3 z>Lfy1GSc`7djxUdSwS*u-S`g>?6?%nS21-6Q972X@!|$IRy3;w<8vg!F@OWzD4cK4 zCd0nH|IL?6{l@6Aq8CBAKRWwT3#EzT!{vX**3LExwS? z#gW+qhsIo0YL{eBog8?Yw|~K|TI`|t7ZsqU`md0N8u~O?Js+e>%_k~AceBU-7%6}K zXvS$8+JjBcR%=u<({2U1r$FqoF|_Pq+oa;N*N$J;lX?1mKPp7B2r`(w8%x~##DYF= zXfucYx5KE5p(eG=cl2!XeTT0xXCovLV)YeoG^qnOhO-}9*@TlrOIan|gK}fa?O*?v zXGVm8jcdoPN5*8QUo&p!&{Ms0FsjMrzV&C(>T&M-e78gBUI@((nR|7Kf6T0?9~WI3 zkkU^8WpRPgNvhR%^R(V@qG@8DFy9aS<4+%U&%I)?%!@u5wn{R*v9%7sl&t9Q1#-U> z=-0CbIlf!T8{z&9&CP#ulqMVKe~BlSSp)8jw}-04K&o3{(+0CH(08#&Plpa@cZ=i5W+KeQ z;na^_eE|xlAyc2~5o(U!f?(evrN2IQZ{2oe1%O*SuTxLw91lBSmZ5;(OyCWiStHVP zYuB3gyekQzh2N!SIvp}E>=pVijB5)8xw*xE_WRORK7ResR)Roa46)3Elk}V`sBxfZ z8zY&}9mLDJ%!9B;(GQva{zl1lYTI0@_bLCFc|Zxpj!r~Ru!wia9$V{a+Xy}Sq|;TJ zzx3i2CJ^jti&URF@|X$#3>SQ>Q}$xai`**zst8kb6A=Vt6Tr^g=QVsKWte7f>n33O z3{ncdZ}zO0hGCK$f0RF~@ZUY7jn+ZmgwW%ui0abiFxHIzZS7xs8G(f7IiC}I|Dlou zM!BqKVc=>hD|A5ra20*)!$c;{=109OTdQ#W`7RMtA{FELs7arcp0z_!R6HzF7*d98 zXz{E|#P@^>3C$hj@B3LUrZHXHcnh;SumXTIrZ@{i8`a8(Mh<$ueXM(J0o@3N6lBHz z1n5x4iD#itCN8$Q&4J2RkU;w|~Z7bp<4f_BN4o=0Il)l9R6+V_tB^3Y~Ta zzZ@}Qr{O!5*vnOa!^nTa58-Y1X;o&%Zlm%_tclp})My|d(0z5p=Ivz(c=iV5@<9pn zPzxw(AS}JSFh;jFfk?!ripK@vjoES_zhniup>YZeMw$;JJ$B=@+{9vP<+9JM3l-I) z;u-2tbDLKJLMLyWDMmGeo@)~my&1TW=k}#+siUkn@jXzhwBZqZZz*2?ZVWXf`8Ep@ z4Sav7KHBrh&tc^H$?3k;6eGFf_x#C(LS?T$4whQyUxl_Xd2ELRXiV!}@yWfq$orf1 z=+!;;x?%BCf$Kv}GwrC)K`s#x6>;tXoF-Cvon9i{UONcS7xQYapn=?ud*4zHW7qT? z`Z;QfokN~(T(gSahw}e=`q=P6Bpjs_(^VhEABj;hYXv4?RPz2u<$rqYzxe|b6FPld z*S~HaeNGpD;rd?lQL5a5A6fYRxOvdL(8!|48O}aC5azT0Wgk+v?E960uE=Bo9h??xP`~GH)PM8a({3r)b!r|2+ZBZ!R zQBd;Ps99l+E}7agL3yoNHW&Wn{_V>_6VT4mNv=exZ8DuV&pU**%&} zTS0}N@aAQFY%RshVpixA{Haji6LiT+)<3qVH%fkQ@*`#Hoty?axZ!f<7k}*-n}a@&srup@hx&H%y9Ll-SBizB|&gk zE*6A#PmS@9-3JH~ory)!jgOvX-ir?3XC4}@;(QuSP<@VhmuVqV$)dtiwwX--H}0nt zHN+>}Y(8;q)j!C+O$R8B`fNdQ0#V0C*F^jJRJVXWhIU59Ld4^qB~^!f6oi_#sOm(% zgbrODclE&(0oPw)H4Uw`^R7!4&}yIw#9F<}yoRZ=w>xVM zz2F&*DfEiF8`ex>`AWj?P!V#!rVR;7qoF8rWNG#XOu3@x)j^AH?Yi|oOg|F0`I0jl z>gPQ~Mb871>4tDz3T*G6UVBmr={vj(-c?xZ9eo}7V*JG^cn+C0;`0O%0&I<{r7L%* zRNyk`Rjw)6w=>{+tfunEQ7XEO^sSPTwLw-lPqMyrNLAB?+G*PA->}R6mRo`6w68D` zL2DSv6`{fF@rinU?Ju88A%9Ce*`zhWpKx;)9ew`Hw=Wg-6S-rx+L)c|-*WrbuE5&% z6kt6ZAr&nAuJQ4rppGv=!+NuQXU4SnWG49ZE3|UIwRE4+u$GZ>1qd+&iS4Hu$gF<4 zLj7*Jdmm;>^a&|Tez^dFuHuF;vW*_h5E|q0jv3E@aA9W+KAUQ%dC-uaVFxs<8RT#Q zTxAlPBHRTNeVKjpGDPwh)uf>;G1H{1RX-WFbonz@(e zE6Pd<%+3v82Nx7>_yZhhUf;h}7ze~$D*GHTpZzqLRs*M`h4i(>`Qfc$I`JzEC0LL!Wc0g{h#ClH_mh~niK==lr*KX&3GPhR61@f`r_T_&vTcz{}V-nEq6cbvsmfiQ23d z$$c+p+xuZr9i@dB?Z3Ijtviy)V$^N&Uyxsr`*%u#D(~C%CX7vT&+#=zuI_OByz9cp zukeVBdV6#D194tYr%l;jp!udYm@zVNMw-*2{RxT%=1gjcXBGj8iI&YpXi3PzdyG2C zs|ES_?*wO@D(o&U3*Z#Q-L6DH-J+fa_Die68_ZYZIMDZpHBj%Sc;%i(M-ALE-E797 zXHo(hL~o9MK_q+hC>x1?V&FF+(!L&c^o9aTA|vG8xwc_4{LtCB;=ckXwV0i3&0Aa>tETI`x?C^e>xa@y0*f zxE{%GE+pSF?8we(&O5#!6ulwvCCc8wiK?V`>3#tPn3PrqW@F;06JvA;KD4L)cm+1> zd+-S?{u)>AhV*&)Ja}Z_$sb0n&ukQy@*`)?tzr}|hOqci7J=9g;4QF2j+%!C*dTMx zdya04g!-dWk4LUlb}PSiNC%uTyfBRTEH+_mK@t3tAz1a?laTxdnZi?S#BhG1LU(v} zRn-~G__`JR2cO!1h{@QA%}eYZDZn67ly~FQQCBr}(>09Oi2PNQvk%oW=M|SzA@zGj zmbBTP5S0(Ck@K**G*n~D(NBA5A5J|OV!t%=$UpX3+Hk{a`I8s=#)xV8>GPS-kZozG zZNFGHnU~?@E%o`lx{s9uo*iqjqP^=xdOo3{)uA6{kuQM|wQo&O(x*cpcv(`a+;c*J2$ANZbDmoYPD!;IiW>ph(w z%Vm0|%Yj+z7VhpI?r5DoTQ|Nbk$CIIg}5O53XQ+qL|n~PGa?%E*X}AG9(w$op_9LN zGuI=4U4R>2Yo_!sTloeyJ+qdg{z;BfCCpIV6nw(GjgSOy1@4lLhpP`g&feq|v;5sr zj&B0?;*)V6XoD;H(g{o@QagADWbPRetf~O&IS&uBjcb$VPJY0|>%$?QZJ8Frv~R!0 z9- z1jfsru|MCZCjh#3-hqaCg2MaHXIv&Ecs&LV@!Ic8COtTl%44P4(sytf7F?2SR8GH| z?)Ph>L`zxFQvTv63sqzvJmer&3WgwJ7qTx|DVW$;jmmyTeVv;=Q61LQGRAe&`1H*7_j3q`(&*5@#r`=PjF9ntY-N`2aEXy(< z5_i;}^HpggU8bC+i6akKaR8TU1V%zn7=|C*@TLodInWg0s$!D$ZDd9Sb*S5qKg##m zEaAA^$c)%_ygF74paRQvWZ)JZ7%+!&2S-3i<5tL|snEO%aBCC(Tlj6n-?jA77#fVs zOCvC1;;OouWLSn>L~B@TDdQ(+rgc#_u6Yej6u=ZeVLWSY!LAZ=J`3wtG#>SEJZ+t^ zx{Tp$*aHokJDllFo1eGdJ&YZLK7zCe0r9G$J+sI^LY*QZ>Hzr;IU06i>1jv{f&y!! z2k1*^;xbk-pP)EUH&42^T!(@UuNlC+(dCJe_2#wxdlU^v>p0Cn7WcXl;vUGz`lgmC zDfi3IutO&=O>wJVl1(t0nMbFzy!He8))0bXgo5Gb6kSpl8ga z$nE~(Oj^muzO?PLqAR%CWxRXD9pd-J%Yj&$O#0wa;juSjLfuV7>|Fp2=2uu8!uD;K zcSQnm6Z~H)siz0b8Ps-zmXcC{7?+xI;O zdgMQrCcxR!`B-r~kh}mbV265}#r+q_k1-H9IWOoFc#D4fDPtBi6TGJ$f5zyvSRG^0 z)pC3gO}Ukp;>{W5Az=xwWwcjEAMw;(92h23Q`aLu!BcS47+_!gEbx2}nbA^22Ty#u z)N(7edRbb?yc~*FvG!@Zg_f!D_e#LfVu;t_o=0Vnr+rqdpQbqI+JO#U2mue9TKZqJ z1p19G{mKK`>jJz03O{vBO|rTm)+XbA`+=bipr)mGSXZ#DAB);g&F3M-EtxcT}% z_zNRXy(i-ktenbW`fz3sb^Bg=RnCIqPjZ>OkP2a|21gIRm+s+3Cyoq0U;Kwvk6lqX z^>yfgF6;VADZ~iusD{TL3Kf5JPED?7+?)v9xZokZr1JBJc9F>4-=6{ayIjxXfWNz- zWSj73)w9sG9`HbMsN_$2yeldi7Y2{&%U3~z)0Rl9y%(i|NxuVEjXqxsb1LS@tAoSu zu8rrwG0ar;#1E(OnXM+TnY@tU?)u-0S`>JnJ@vR*i<^PaE*ZcbK}8U z+!yCXxfI;84sC(me77p?atK>hl0WgbnYzM_<-bVxp&viqemX98_Z2*AXK@X0`LRcg z($4YEsuzyzsuc(=)@W%?w6=DKK^Te1h{%mc^GMjavW>I3;9B)vMlAvBdKR56?hE*T z+PE16f$xDnw_h%E&0Cl5=F8J?qJJx~YWxD;`{Yb(hC?bydIK!>k<2}DCIa#Ce4Co3 zH)#aI>GPTI8_?hVHeeaE_4|8JgR}h*6ta67QiBuW zuPi9gWDUM7*!LuB^c|kt5&o?c2$SSpM*qxP1z*c&n3?d3atcu<&IIVT2g$eGkh?EH zj}WCRLStIsWk7Ks`o~4E=jD6Xq-DeZVq#+=9~`Lp|K^12RX{3w=Dz3LU7_ioL3sHF zr|~F0c20Vmqur8N4ExND5E;%Ff$Y!lj2|qpwnM7AFHSD^#29 zwcD9dsPgKM2LT0s@326fAmblC5LmzGdxfT{&yFilaO8e@YdB!_eZ#`NJZFX>0Drop zs!u4YWMdIJV+WZ2UV@y;1HpJ`mPFC(ormoq4W1}IisK$nZeFI=X=j62aaVs;f2eiL zli!H5D#-OF&!3zn=odJK9}<9pzClktwZRgeS64L~anjcBkirg)sG9^89VDZRivh=< z4IV%)VyAMa@_XPUKsBtz*9Li;xfZooJ=&5hqz}H(DSDPsdX%278uYqajOkZqOjWRh-AL zkALBCU^J3zpdaT#{>@!2Mx`$+R2ReWCd#Oaj2(iCUL&lBWP|E#FQB88DEd`(l=aYZ zb&XJf1a-w!N*X6hp{)G2^0DWSE6nQ}^Y&zpE>mOv1)-1U3)vxKV2`Og$iu|L&0|cR zsxS=G8`pcnIDe#$Q2X4ImA_B}8S;m`u8qld)4|Y9Z~XRQk|$9GNcPW^^(`ObG7Kgb z14DNX+gLbng4B)XQvm#7Npi#qGX;K^dd~ozqO_v0#3;seO*j47tsgd(Z`t?^iaS&q8s+}PAhNs zp)qKs+2MN6!&Iu;_&TM%w(ICJJaIo3bwmb*psNA)hkh4LE%q&F5WK#q4uEI}IQYOq zf`(fWCmwrSd<}#Sw;FU2+gbJf+%2`F^vTZj8|Wrq6vCSt6zZtzWIYJ*_j!R9c$obS zEOT!txF`aMS$yt_*GrF(VbBgTnSx7kd88IfrXs~nakESlWrU6I+~tB_2ChYEAlg%@ zz?!A3n20?C$qg)@-yJ$NP24O^;>%Ms&iaEgbkZN#~S7KEF&Av8f@&h$l9i!?U1G=jie(O_D*Ne0sTlAP!0!7ItmC(e7 zqmKJmD?oy3kt(zZu$9bYV8$mDI=pypyFa(z>D2GDvfh<&3wa%%xa9idlX3F)>bDQh z-lqt!EpY!v&od;d57KMDnR1N?`OJbgfR|#Dfk@vESQVH+UmBtlJSou!*Ilp z<%1lX!WG1KGZ%i&+V)u)T+C^u%-4bBha2x-5} zbj>HA3kvC5uF|V0Ie=@Sc=`&N4UL9?2zI>vbD8rVR#cTD1AaK-A*Gb2f9d4?gk)t+ zViWp-MW7ja6RNu0#8sBg+l=IeV@AM zlQt(fd2i~d3;)2navZ~^RO0>ZOH8=u$A81)zd{DJl%JY6dvZz5Exca|@=)^u7s06w zeDYzKn0#9F9bIJs zw#TFcEdSb(sWADnvxO~G7Uh{cJ(j;(3CA93t4-(tWU`_2Jh7s8o)8zrtxB+s_IR;V zfy0Rj*Ru|UoXNdY>OzQ%sldT(Sgg8crK}~F=FX$hQJ#%NuLUvgJ&FNaq&KYO#bY!U z^SZHjXp@Qyd0|2v{_Fg_o2*VAiRb6+<{`m5AoY8geI2BXck({}bNTb+IJvgCC!%$V z3H~Tk*8+OlL+5*CHo1teh+Tp<83k3HS8#kRPV|`goKZv&@26miDi%h+o1t62TMDFo z!X~A>TiH@1C8dFPrRTmpbaHUj;2hbjTu4#-S#J9bAv2z=J?x_`BJ^0T`@jn44ue1a z)=F@Op^T%xyNa-@_Z-k+C2W{P@QMsN^2Hr+0gM&s{*7SNdA<^M>PCsGS;T#fArk`f z=lr5H5uJ*}B={e<+*6y#;>@Zh5Qo2avq&4wkc1tc zCp$MnTK1#hX2bVRcjPKLenrzP>V(IzbHw>3hvEAaR!O{85}5HHvNv9U=k0Z4g$TuP zzF!b$64}nc2VzDWZ*lS!UTUJ=Y2dNNkh;imq$f z;^;)>;ZriSaUxi( zc()n;ZkqX{_)amtj9g<_!~8l#3>F9*ZOM}&MRhj=VBB7NPkNX{!dmxZq=mhQE5~fE z7qfnO-XcbI?O!VSiBRhsH$O+Y)l22guSSV?Wxh)>xb^N1bn1ZqTnLpZ$qC*)7=>k0 zPBnP$8%-^=T<5^J5zuSr>$LgZS@CH9%?e2!@JZ*s?wq=>Q}eCB{feFW;qr}q(;IHYcrtE=xXI)&8j}CycsS!U)aOjp+g%# zo6s)P$W7m+Xyt`4F^D=znv6&?Z0MNk6^#ABp=Yxa&W?yuULaW6EcRHuhawC{43jH? zO}Fl8ZvipfV+_doE{87~R64qB4;j=SSc0z@4dBL#YA9g@Nrib;K9Ck8Q2GBXKx7wz`2fVC&)7G{!1Uwr5gdP?@4qy#HqZAr zkser+-9(16sXq3bpkFkp=|T2XxuCr)(VMM%wXU7jlSVG&mJztqGm zt+)7B$kUe#5xX~Zd}X+2g=i@-)$RxA%y7J`{axyb`UYjL0@`%Jl-|2j4c>FZ$4|s8 z@nq(BF`BmJ=s?Bcx_D`^Da&7wRCRHQZQwwYgDj|#6D@Ybe}9F_aEm!1V-0T|byK6p z$4CKF)x+qJq3=T-w@8H70e?vOH?)~3S`meUIus6*@%$5Yc-#=8GgA z#NNCWwIvSSWN5~B3yqF^5*>-Nei7`kUu`42=#^*7TO7S(#OH0FJij{zGvM(x6~CBs z%jikDW5}wnwD;;?MSm?^Be>{EX)ELNX)wF*B z&@F+BTf`Wdv0yf7MZJ&+;GMWnvKXX8_Z`Tk8(Y@T*#>eD)*UFG=H>YLr(rsgFEHbk3D8YADhe{}Pu{v~={OWvOe zZ;7S^%GtQxof(vJUMci&-_*N5z#K<#wOHB1epo*pTpwQzGA4CUZMywv`!r7&b#COS z(Q4CGvk9$#0A83(<&M;%EsI0%PhQlX|om`C~GlBZ?`*6I?CYY`s3{wiO^;+<{}&W(9H zvhOQ7`^|GdJ>}aMpI##@iIU&-bI06$;AU5~KN-6)yKs7NQEy+_Qr~jncj(x_^u2Bo ze*@67L`-Y$^aH-(u}UIbOYig}$8U{w@+~^vqf-$>4P_AJ7ZZ&!=?U&X4drv;AL=IE z999{n>2Cln~i7;HV zO7fFN;r&er38KM3^8~%*yQCYR;+-)m35Rmx<~N^yn17@EgWX=4rNr7HAWb{FMOVCVA&>d>0$deR&u+wP3sKi z1&38Q)}~Q(f_(1H*jvxgpq9R0ES0a&-)fYi9~Qa>x}PX$pnm@VW4Bp%5!a`Jg1GAU z>))gR$1F^!_~=AE-z-!XMj1&Ff)+ zT76MPqAh|LVJOw;xJ;ep$V3bDHXNg`AjcwVxAcH@RLGm3lv z+>8Z0#W_qyZ$FuBIed5M!sB6lNrBtyeiRA0ytBUvUf#U^idcQS^KJ!&(?2-`Im(3z zrtQ+APsH~k?@ar~;>aR@3uI0B@5~IJG{t_nbF|ljXCS_>272!%zM1ZpzM0H?v`OxN z8HyT)*+Wj|usV1+#?Cqcs4y5}6O#RAYD)ksIJ;%J^y@5T<`M293O^Q(J&rs8qhSfj zE%}Q;L61iMUT@=+iWP{3-n1fr9PPkuRk|K`QJJ~f-@Mw54SvmEU1f+n)`RSV^Et`eQu zG~2&y%{>3}^$uMixY8iqpzsD+!hF`D7WGXVLlZPo82nm%yc_rM{r)%ZE`7xtY-!}m z+{pPRikvL}aS{Lq?UMPhWR((!GU)DIiZ75FXZsCegu+S}3t_s!24L;K(U?6`K(d7YLqJR^e>jiWwj>@&Dn`xSjv zz9gdUyCh{T@y`ELO}H)%^n>qg5)J|$9r?>kMO<_t8a*N4rjahEfmfSAe}t0HPJ^$# z&-vH4j%iDU7rTl$(Dx{fgPE_Vz%l<5_N~X^S#fNu1Tn+Lm!Z}qXcmnV2q!va5tl!6P|WB9*%fxKfx1!xU-XLc zanIFF04ZpI{}n!E$PFtR{@gQk8Bhw;N5|Cp83o!Pm-{P7``UWVLv3|#$fR5)cF3+p z6*)`V^N_%?ho!(wbv&sX3@oqy%xo3&Ihgjgvp@!D`|#wd?)7Qx%Q-{ zaVp1;{2JLxOzwxcoeB@%0_H0)#Kc+Z&dBqguja;rLh^8#W7|EK0)l;|k0=8TUVq9* zhxA{k4*kkzWRFj|Qc0cKeqwR|==RFaYieNuiJI8&2E5}-1@`a9`doZ%+p}6XW^UW?9|+8B zL^s+A^U)bA@`ewC?!&+iD`{Ajv{g7i66@RjRK$0hl@Z{mP}W&*()X&7KtMqI;E*ag znoI8G-sR&@ZYEqhwhnXp4hro` z*p%iQQEzE66tpy|%CDF32O*llf1xO^4`$sQv7FU?z;{U??cR0C9^scih@F?=X$%Ej z;RBx7ETtXmg=CaA74{*}o@9OE4~98X#_dCh$lBpgJ$HxQ)=(pMx^u!e{a18@KdMV; zl)LKDZm%UU59~+aq_TE+|oV$4wN1%F!VvDztHM%5Z1=-LQ_COk~ts%&XP2Q-NXQLztFnVx#f4zBbsL^2T7( zqb~qLP6sqb0hiXwDZ%J=g`mkRGjXE4PF+pEYaKT@{bffWeH;%lV0}-X14C{OXn=>h z+{~Ig0>_F{^?((3i&LwIZvh6_=MOP=fzGX@2z@>AS3%UO{%cEm|Iy-jWx=U8w;!$J-hvv=p!lg*J2tAvzj1P2rTL3B z7!+7PH~GS!>YK!<2K!Nb+J>Zk>yafpY_u)i%^M&{4c0)x|2;vW!3Zaqc0ol_-2^-x}cJVirGO7p6 z%5k9}M>p4@2~o;Q5BqGdmX!HxHO zx$^AT=98KWXfiLdrcol@=}}kq(4c#^o1bp^{j2^E@BmW*jL~vbcwRZy<|}DoYDag^ zlz&~{R$7##CsWS2#3i$Q;n1eVCMv0(WU^4l8wW8su*#1_hf!Ng-jWO^OEH07!&I_F z8sl5HLkP>^yAk2=!tnvTp5h@N;Egefxc5uK;KOq&rg!y1S`N%`ZyjLY3kx^{e*($1 zxI>7tYPCE{`%p8+pU^Q+IuZ`BxucP6RMA2-st^9g?Rc1zol z*Q)O#_?cw$c0o_(QTg&bZXUb!dM>;A`e{v@lTtooPsI)Gd5+Cz==}y=>+l6Tc`g6- z`b9H(F1$Gv)=x%Y-VAHL)`jxQbPBrG z(aiFfR`o!ve6%jbn@xXR>{_E3qC#(0b4&Eg@7I?O3UqVpIi>PIN$?mDl(LiGXE_OY z>oo>Y7@+`%wj0q8Hsk~WCFYNzZ<)29Lr43~D%FZBGx@PK3mTmJ4z7b@>DMk;c=`Ji z+!Ia9N~C~RYor$@c$DmYED*w@)bB%cnPERM8xAqKU%mnn(APX`ucgeF0_d#=d}`Ky z?dDyv+*~qHE;Lkb!Mnj@hwL$W-y)`VvjfJqSn6py=nf<`vn;nLO@=;so_#`^*|B#+wURs+&w-|aKqJKd}p|M(T6)s+0=%&+-|{*9zxIFS6t*)<}(Q!mcK0PYpv4> zp8LfQ_1RC(wbqG$TZJ*7hCu@P)Hn0-Uz(n;5Cp`r>0ln$zN`4E&_9H~;LEW&0nmjh z#PIvYdQ@(g@kb3>9@5$UVYs(TgXDvVqPT^$`Mn%dhJg_5#`i9<@5m3YUJozE@AC{o zlvMgaC+Di_6<~3~?39cbaIXC@Y&=d5X z#@Q4`C3-k)K`$)V@gX6&%-x*)Vy%jkK46T`9^7+djj;e98e>o6_z=ma#^GlFO5fl* zVOZzbN`^6`@SB{oM+PRKxpjDpOtmHzAol{J53npnpehFqWUIJ_;7I}nX?-H>Qn2k{}rnNZ*XJxRGx z#jUy)kHKWFbucm*Bx}Xqb2&ccG{eut5yC~M^Qm@%dv=)Z3-V^#`D3r!^ZMEN5Z9e| zKY=}4O6T=uuRfHE=i{$uJ$TQhzfI{~2e$5)J}#lzi?G|VwCfY@jHY+$;q?G{yTXS{ zCWaHbpKy|Dy~NcF9XP)r;q6O6b?~DZe(krC!o0V1v2>5kTVAgR(i^pIS2WcqOR(V4 zbnAtY$)I`tuikFF`0W%Hs=>1{^&su2Y2G3$e(kn}=SPEb?c|9+O^UBFBpP+|%4U#G*P-&CoXx-i{~JipZ4N3? z_bIq4Lz8LZ5+j0B&lTg)r?@AXheOX8eS-{Tjcbni=?p!l_18a*)yQ#^Q9T03pfaha zA%cF)hM78UAEmCNel1u((5IRZN=bjkv^cO&D$enj5tcMO>=4@Q(9C<~JObRut#J`~ z*Ek@IDOkPW1(aG)q<U-{hj|qT0FkG9P4Io=d+k zy%+g7hep~0DmjMa`;DMD*KN}Kk|Zkf+igWb^BwyfALO0KYbsoL;xhm6izCFf1W zOL3xd=Tj^l`mQijzFT^AT^>a>2MdYS8Ffgl?*qP727SAtA+rQJ&*JjVUpAh z!Z^$FjoW}%JS%}){iOXP8HjzJ$zP#bhcnLKveGu*R9PJuAeIF#g)oK66V9Cvf1J3v zIa9h}55dr;&izr%_53z;Y_iIgz0njlV?pxytV3=LHA|vMo(c>3s)ntu!+( z0lE&iBB_e}vIxi5OXr?Oe6($Hlho`bhQ|hIpU*zev$bl-{D&$$Ffn~zMq|B#>|6aF z?aBS3Y4dhREzRUVp8E)DYfha01WzwCsHJA=deiVTx2W~oGCtYvu2ap z*eV|Dtv>uHriY#AgYT=8cot;y z>wn9^0W{7E-!O5o$}*2Dhf9Z6v(CWqvT&CrBZx-WY?5`1-Kw5L<%k9nE?gT?5Hk+Wojp-omZRi&q zE7}s%Cq8soCYv6!Sy$|wx`)4ocglr84O=POewXd4p6Me8fNgIaRIx`WN;9LdH6@m`2^od$0XQWxRis$G4L+bj#~Ut8_S zdf8tVXEs&8h#_(vQs5Q}lK`6I#CP0|bEUPHr1J*TZp-i7_Vhttb@6>k*{B{4+R&yAA@8=JD?~p3>0D@JuMU8v|@;_K0+7p7^A$umVX-`kbm0eN^C*C4YK7_8!i! zf}SrL(k2qmA*-sJA<{xhxL#JEli@jP@_hxj@ky5m=81yMuCu9sdu;hSun|fL zU%^dc{`E)h%y29U<`sLQaCIz$90unl>nmVQ$FL$i1&v-9Ib#I>p*9; zvfYmiUqc#8aR`S4E)1KF4}U4Un{qyUqV zqYmX-V*a6Lk|Hk=&&&)rX9>8HV1BQ&4{6048V9V7G^`zR0w1WX1=NhmbT4kJ4erd5 zE!P{!i%KSsaze@;tJ!?DZbPar!G@R2q}7Ob{ms2|zGf-MAqwnCg@TrwLYtu;8pj~4 zfj&{!%{Bxw$8*l6z_p8Oecw!E5KL%ivd%GupZ4RH9v>^8m`(lr+{7baWh$5r}F7MXt3;mqPuM#cR(R7?j%#P}OM@~J@hsf~b;4fd^ znx@CCP%fGo_Gc+bt%`o(mP9K>?Y*D1!r#uot#VdpWNq^~tj6H=cuz&kt#c>IDF4bUI6)+}lGhq9)DgPu`x_HggTR2Jm_*mOM%g0B%7dxA! z8#b`J%k(|t7D5@t;bu+}$V&#Y@1&a!<@_R=1pHzt@r??HZ0`Nje$`%hsL$jO?F&Zc zKJ4PsTgxi0jMA-(3)i)7O#Wnfk&)2rlQJsK?qW)=`#JIgPRW#P@3INSJ~1A(4hyOI ztH_o(hp7&vu2mm5Hnrel`j23r~w&jZlNkI1P6xU*hGZ9^`VI5J2isYEc z;TxhICX9uOwG?<9E3vTX9v+20tOF{OOEF=WM{q<|1!OwPn)G8PRXE)2t)+xg{0OI) z+#K6lM7UgJ<(~}x##Wo>$XUbH<0Cf3e#`b@dUIm)L+nkxo%2Kc+(-MjG{JO{-^me3opfHqgy;_Y8m_eu*CCjdjyB}u$IxKA><&@{s!C2e5$_s6TLd(ee3 z({{I$w?Fm21YSECMpuvr8*sr1eAvQF(Hky?yUBXC_s6TYr~4QiA87VMkqS1Z^Ygme ztuy!DvR~!6S#_?5Y~o)mNsWo9rEP{IkA30VdvEl92AD{{Qu#xSNS?R5qo@^mg}+`L zi++S$)}4)4>4g#Ah$hSXbM%0SoEr?AIyOw;$BmJ1~ycUj!&wg)rFc`38WIG;6b#_$^ z;x<00gw@jTw4r7=s3Bdj!LLIpKn&+q<62>EW{y-@Ui-~Z7xLL+vDp^wj~N|0H)P4w zXj(P|aEb~tXK(&iY>DoKuAvN;^Ba8HoegoYK4$k!fI_&MjClyMgi>7fV~Kds=ZTZ9 zVJ@n(G}|{2r}s$2A~f8ip#y_qe5O?!vNv;KEFILZpEczq;t+j}K}Ild;Jg@5(p*P8 zM8Wd0pm*ksK-{DY^zj!DpgWk?!|}7poSWa{%opAoDq`n4;m{4Z-+j)7;8O7CXnJS> z`PO`#8MA(I#COt`s|RP1Kf|~}pDG{*4nhv2cpR9q&&ZJ(_{@m!;&YBeR;a4u=|j|w zBM;zUMWnKVrQgjYKA3b1&@S^@)Wx>ShN*PB%J@YSPql(N8#bNHMLPHS7-k(;teHfG zS^ggYtSoM;kAwHE-Tcvlbr+WkNo1K86;zj-lOCEy-)_+W@}9+TU?^K=#Mz ztBpY8I&022it|i#pqpWHh1i)9R1jo=Yu~~*T3sKi#Sbe0XgwX}+BGypKa18@xwceTH03 z4v|bKY|b>o&dQ;RCnDlxBGbEq7y16v_5*NP_9QIXj~riHJQMo(G08Bb$l|qe7BCrCJ1u6g zA$#dn1I)t3=NRs7t3lirbUGyWTHrL*m?Wro3NB=oWSA;_Z5?{d)lbuS_XN(!x*&TQ z;-_1xPHg*yGgAMB`)p+XxVS~RkS%=X+b`w8P-}X>kL_*ci=S}b%BJG{a~%Nqjz zVs*#EXxMd5viV2`&YUnbX{&p1wC+VJ*ce5AgA zGttagg>MvdF5x}h!4K*9>gEqD#*hMm&0PzxtsW-cMSRb840f#Hna(i%IQ5%ydO8M= zG;;dy(x(@yE?ImZAh&>%Y&?g4c>*> zK>dcr)1KWSTY;n+FE+FY@N(9gFzop?J$)c>*xV;~^ilWO#Uq5k2f+hM=AUcX&w)Ev z4rSFVI7C+7sGiL=Ee#%_J=zf5(#qi$KeNwmY3aE2#Im_*Hqysnk$F`oN%ysRQ9lN4W`}J@=Ygfc%&nNh5`VsWWFVd-( zfRlH&@@dwc?R0)1Z~A$T$KIow`RjJEve4+>OTbn=U-N{k5`0SQJ8C^iN-m$~p8<0Y z=AH2}gJZkE`0fS%fhkk7-tvFYY1c##auV0Gz|}SNtt)=XUi-8txZz`LjsMP^EmdFq zh>MBCgE!(_Nn1^Sui>ENK#F|#3GbIK{usv+PJ}%HS&~!YdyeDp&u&>mN?W`Q?R^_p zinQR)uC;B-=~MTf&H0clHR|K#bK&a@*28#_pSI;c&JJ6oac33E=e4+A^=ycuk1M(Epu%oKf5|KR;!s6Y6I)FPD-crFmAQPyn4{cEv_$p zrtv!I#^c}m#688gQXuN65Kfd6Daa1(_XiwI1I8uu)c^%{fS*Za^)NV(8SBCRd4;PS zov~ne$y7nY!zl2FOt}iic;pO)#N_%waSO;{_)H+dD^6OiWB1C}r0V6|O+#A{%+6i{ zQA!k)VROYw`M!B4p62JgIUE-q@#G?k@zsd3eX#s%nhDLE6=F z0_3ydCGoD<$-yYPEg!@JGmJ-dFPZ0=^CJU)-wr4za1 z!tM>Z+>b*S3wZVi_wG#-db?BT@|NebI)=7ARyJ1OFI&dcOFV4*8rtDMsFuk}YJKgy z$jfFh2bj!j;|FsLTa0Exh+px_zDp%*oUqSuk$9e1TCsMQ3db&5)3p)2UvNwC7IY%{ zc9_R#F?28G>X{(to;Dsn_l@zOZvd#)H4YA+YVLMz`p}8F5pIh)a@gHtORApLx$_ZMWcJ;H2uWmO&^<=9uEMEI|IPpM66@G^zHarQTiQp zt28R+XRdIO!PMQAS4%h+Guv)#wOfySW5&cfc*uLff3PSXDXQ;#0@p=vY!p^>t~(MS%c>E zdAckEEKmj`4{xHL@Jt$=%oH|mrxJ_wG0h)-UJD5=dD5k5)pq%Li@W~$kp0U%1g#oH z$prGj>FZADB8&wvC%^5_eAE87fVt@u%#zFxX*D#P=`OxJA@TX%xf$o zqm-8P5frwC=iT6_L+23owbM+FfSWiWcCOv%OIrhdZ_-{(^^zcW?hD|hW?st%+nry* zJ<_BWFiKs%-NowUBj34XUjMI;+9^x#wT?E<`&?|^eP(H+=bN#XX8x&LXTE`{DrtLc zHus@pP9hiJ+cw;OqASipoj8Uht(aYi;rUx>TtD?wbqTJ8xakcYdtmh~i5dT3A7*!L zRDsAEEHQGd<&V$B7@oV8#>G#!r8$HjN=ggxAIUZua4$tCwqls60W9ReMt zhgS&v$OXnYc)UJ#weZ{P>yOo6JTAdXSsc_}7{C&lT_C>W)FNg&X?6GA7#30@nDTZ7= zH(yqkwetIMG1&J-1Xk=wbb5=k5B}+T2lk#EoZgRT`VHc#ZYNVuIN5&2k?lhIsS`55 z0N8Y#SkOhUonsD<;S8UJJr2S9hv2Ccs1iClmU1tM<%p=oL-Wib29!erYdP{K%Y(*S z5-)VqYe@(u8>6BX>ZHb7o`@o80n>z2T)O)kUlenby)_vEM^PbD6&chZR#F&&-fWn0 zP(GfZ3cRor&i8fpy!O*!chLliz3rQvS__;ecBF|*%T43FRDWsg((vb7}YbgUs=g6aduL{sb1Hi#;BY?JD0NCtvTXlbp-)hcfkp;yi{?bqh zrOc0m={!s0E%#LikMu2nL~$^&PdJz5`2RpK1aAa=(_z{is{eEr|XvIWZ5QH2s*&))hi} zr@F-M1EqWB$Vq=5r6Sw{gWrAWV~hg*)KUk;)}Mv2;yy7vHuBuwt6{|qd@R*fpGR34 zxuQPy>vnk&^5^!Fk3ISG4}0m0m0yRNzQ6={!)OqPLYsz7R6w&>!x{SH@4SPtM?#MC zbOnRo-sv91Z`}X(u8qyLGZ1t9B)kS({YkZkg=ODs$UKdAG3;$#$h&+&$;I$!T{{p=C=-_V> z57}q@gsV{}!;OKNVIeV~POq)tEQXBZnCq*``vrA8DY) z_1Lb;e0g!f*zxMBS!7tbMpQaU?CinDossRcH>q+*G07VnE~661b_j1J7E2N6*|Lv+Wcs-OFFY^UtShE94V3^ z%QB{L9^67Cx6a2rCxI;1M7M{Vpy24|aV@zXoNmkqXwVv3X2Jjxg>_%q2>X&wcdL0s zU!kJ0O#77rAh0bI&#uZa-;vorg+6yNQB!ReaTg=gvIwjk9h}riBI~XE^kzW-INgdu zhS4Z>M8(;0zd4vV>zkW%=``kin4jot1! zA{@A#aUheb0H-S8VVusm6yutSBf%S}AgOeYlL0RaSRSG7g>f74C~yGq4?%^vfhvDw z1SXq*EMS+ehQ`Oqz|@KCFT{KB`v(Wv4{806RQbVfXv~LTaT$k&{IN7geE*OBH5@sH zp{ifAyhMue$5}6L!dDY{MI`@3-k7;n+t`p_C}XM?3jiggm|q@kWi1H&ZlpnrY*N!g z5&E!L0{Q-C>!bsn+hnVifobCGg_km-X)5>5x3LRv6?7eoHan+v^WN)y{6!wNM!axj z1gL;kxQ2e>Ik*6Dl65+KrFvTaBLB(WJ=v4ox8Lf1>oJob5f?d7%5Q%JR27U4J;C;E5 zHUJ8&8|zcd3q6Pn{ib^BsaW^QPP!N>N0qq7rI@O#5(XW9ax$@~@|`Yc?*D+iH9}L& zbl1eBg#f$*K&Q47SB?dC?v=VJTIc{b2PjzX$H~&(BSk4l_6MYiI*xld2Qz%cwgMT{ z{i>WyhHc>bU7LvmX5&yerV{j>HlL!ol|nH>Y-ggZ{P-$4H)oRs#__@DCRvbX(~U1>o`=euv{AbXkHI zZJeb*4GKl$FjIxSiq$e5<}4xkn3?ztXvO>JG0t6Ydv6aR=wA<3B9}<0-hF z!@)@@4LxdSuqKWcCJ?%oZU;jILW|!%fy0!CU-aOwT*^TOLksg9Rro3u7s!<{hU;7Y zxj(b`vr6@iTbLTDUwLnXN)!gU=}G(dIi5_cl!GUFpAu_Wy_Um75m~kPP?iTa>~U@F z{7vy|Q>`tbPHjRol9{(jQ2*LTV6dNKOtBGlcS$GmZ2ZyvRz+w%sd zIt(;)F%99d#9-l6ckW*QGKou|@D}+GHfQy34o|efPw$gxoU4CE$Fp5|wn~GCWY^H$ zKjNc!O4S5gwl~CI97AI``UKch(P6dN8H`+(*P{xEU&F9Jd8x(>Sms8Edk%{c5N3d_ z%QCl_miew8d&1^nmMi%W%aTpD*gpaUf_2lu8QLQN0?9#!lwx_a^qIs~^*5wMgtMy1 z>cB6t6H}!fL7~3f4C5Lavx9^R)Gl@W2_lBuYC0mL^XtV=ZWFb!JKk68M+~M8IM-~k(hqr+cS%IU7LI> zKEzJQrtLn{*LYvAOKTcBHF90KTY-05Y5FEO zVUbum=6*O0eDZOEAOcp`v(U9n^Cfs@FrPrgED%&%9gawbf%$_v)^SUDF&guCGf$hfRhN)-1 zlk+e37KjOjH@T?H{rlU<=S}JM9SMc*W$)C*1>_UG-u?UA!;*bx`1f0sGx#)UyMWD} zvb2AH`_W!yaOB@zd>!~thyOLe{~6MMzb5||y)-5lK-TlMH2CM<7-wTLa43oL$CeKg>ZQQw&cu2Qj2Bdg}_Zn9P{v z0J)n7fEuGW_K&{Y7|moXZxi1u zfp~)hej42k{EZELb8+ld-RO;T$2-e1Cet}&-mQv|RVRzl zeX2ZdY}hEa$R#xR0P=E^J~0(AOS2nj#U0m&^kmq7f*0wxs>4dLr+C^pr7AYPwBl2k zBf`t>^2^iAm8wK?ch7paNV?JxGda*hECk~vu2{c(Z#DqprGqXaZ=g};_ zy1+M)zls<0b%C@y&b+#meH>crN5yw%au5v>KB*JShGtL`>lH_zfmp)j!ELI?S!iA+ z2h|hhmCfOmEPV(T{hazZ`b<0jB8F#u2xRE}Rpk?B^b^B9GuIXQpS1i-QEF<88GO*o zk$=Pq0H;_t&N(8qXY4puVY{Qw3$4x%csT!}c&&M>?*Hfl*kWBE9MlB)aet(vGweFZ z;dA%5a9%B~yDCsvn0vBeEGgHi;XW0!(6@cD0*HJ`SaeIZOxA(u47cKr-5Fj4|%GM(Hesk*HXWUUwz&PQAgA zw<1{sg&TGIVkC9NZ_U1`w3O)lsUmge^QNtrWE(n<`Nqk=NjKh>byaEq$L-N?qo$>G zj_M%~7<}D#`}{F+OR;vfQ{m^vJe*w|f4qs5jg@|B`b;3ZrmL>0D)ra~4N0{F{yPLs z0`npg9&1ExJg#iwv2G36{rD@~I^J7<*TKYizuf3_&AXH$=m%q6!F^{Q^4-4$vv)OYll{8QhX2-U7Fa*K&e5Uav;wY9?_Opc!(soqa6ky5K1nuVe?%NH`B`f5ITKQ-Lg>03(qhtp z?CW!`@q(THWOB{7+%Hf^KGGzN85kH3EpG^nhxf@wPy>jm9jVVnkM%dPmuRLfuZEum zJU&_Mv-adftpRw5QNap1R61G_)-qZ83z7>0xjL*Y;05x@RbZ=3AG+oEZtao=vfCvo z4xrn5G=B4%qmBRR5z%_h9p!RAv&2Zfb=;C2KlbKLQny`j1@DFr#n@<2wr6YmxPsSq z?1`E>w11~dJhLEz?tUTl#>r;V!Q2*hw?OHpwBw`4Ol&7x*d^z$ivIMveX&WFYA;MC z4A1=SzFjcg~} zv(IYdW3f5y1Qjyj_^qwl)vO$)cI=Jojq#r(zPFahHr{-OC33>dqPqUey|wa zv+-f;$AFrXJz?p6=uk}rq6)!wIo+qS=MeSESz@|k^57@ewv6NIAMVHQV05)p9pEf` zJh6g>K;gj)!G>h#$7HRL8wV6OzG{cP{5o}mkFRCdAvT9rEZM*R{;AKJip5%Vm-j~^ zHwXYu2(a!bwiR*m=Rkfx-gHU>1ehr@FldA~gW}*-nhI#np=lhxaxAJ~`p2TpDg7t5;MoCNI!fZGq@PE^ zve?I^X5Iuyv3G_-@?lUD_TnnQ-a~|h+|I{6Xul@G<|4qu^t5EFTXYttYr8}S1_;X4xj&Y z?DK~wQL=Tl<-U<-4|fj6UX*eff}}=;%mR=Ag@tUYlq4J{=Yp90@kn`_TrM*g(@SX+HlcZAb4V zA6cOahTpl=7l8c6X~dnxx0kdod@7jSF5*pDe7)TW(%#&LYg4o=IGKCNzWJE=_}K7~ zV|PH?9sGH^H}+z(Rl)xwYa72JumQCHhUEKg2^Y_95z`3g-=fG1mL6YVxBrR27!Ehl zZVD~d1lK{Ndt05zeROEPZJ*XuAr&)a%Th>#GbZ)b;oM0(RcLA~afQwT18&Z!5~z5) zWBoYAS7w)C=y``ZK;n<@px(mbPONlSSHr>M*%Hb!H2T3wEiQX$B!!NdB7t^)v{*PO zfJk*GvN39iwehnJSFks!YeE&~OYUnX6~0x93KDg@cPE7l&I_oJe6rd;xJbV9t-6Bu z3B7%$=|JR`H=<{jUpU@fKF01Ws+3YM!T?X~H8&d^mC717s{TD5wfff4SH&y|P9ku3 zZ;>}Rx>vd0-S2(9*qmkc^r3+DDba^0NDUXfS1)c+vRlf+1~zoCyuOj|;`$1niV=}K zzv<+E=g$v2py&>}w4KAR8$=QoPDRQ5j`M5xS}tCnUAT)Hg2FwKav^sdwuPixe82x7 zD()X|+4GGO%5M~t{<}o~L@+RyZ0ZC`okh@t&qZPHWENizkKf|=yYvYjn2JQ4N54?m zx@CK)!n~x}vf8`p&2DDVz%If7=1*XSMJFx|@BlLrAe|Hk4x(THgQ*jog)+mfphoR> z-`V;xZ?DEnJ#K(A;`KHW$RzX_I|(5+tjF%ZW-xan$HL5FbzCJy1`dfW62uqiOdGY^ z4%d!9`i%3P=Ae+W%HLUz&b(%IuEq{j2HSfi3z&s}h8JwTA3m$K_rj^+ z7YlEc2DkF>1+8Au()-l%*;lB9KN5UTmi+TC=bxAil2ap0qj;I?jYB9p;tlbAeY-3{ z^o;(x3jZmtAneI6(!gzurQhBRh-a%=x9*&+o*ZXI&B)}y*ZuYBGsqkqB9Z1FqKV@w zSq&%9;G1->cYM=xA03n&b*tqh+5JIEgmKdW4%5yFEFg6|=qGjoq?*d0?RGp^vT&|? z+U*5dll^FVwcNhX`KoGfC`aL^^$E)-*c_^5{{EP^XslACx+O(oZnrv)%#0|9sH9pM z+(Lz3dYSO2WaV<~`G7k{k22}%mLlPQ%94U|e;pc$iw-_y>3@x4XeDsDU)6DJPII{IJ*RxmSgZ1j93S>D!oVyg>0t zMxpK}&&E)j3ip3^aP!~=H9`Bk)poqQAKwZ0H7Ga~D);H5T`^B0dY(f*DdeqF<8K}aGW~O)aVj0jUBox$Go}*h zf1YAwmhZNXPGp_g+)ZX*wGOSaSgqxT;L&8Q&|hfJXXvku5A`hwhOEjmZX@AZJeOqz zU4wSSf4L1E<2XYJgh$WRZxwT91QN-k&S>lKP8>xq>YRPN00Mn%b#WAmcR3B8;1C z2dpv&30xkjp8~KeD#LcUYrJ3-e6PF@Mb{oO<5+=Ace2;E7guV3mH9ch&LSk@8K+>a zn*uZ^u?9Imgo9YF;lV2xxC}LYRbWaAnxu+N34+5#I4kk~6`oQiytt zV8@O)c2R*Xi%Kr`~rB+8q{Pzp`QcnKLb_84)vI4CZ(~b3E za8?4&mrKxx8c9$sI6&RX>cLvf_KO4sQLd{uz)Jzz;&cwoakY>}|4qlNvTDQbA-?Zs z2@3>10DcgKRqdv#S6~(vDgGnDG^!~QJIT!BXHZZhjIj>a8t8W1VWQ{{Hubu&cyp19 z!EDtSJ?J0>*VaatYu&av(c4h@ks~F@5LAy_LHyRtJT-N6+kV%0<_2Sd*v(PL`sqbV zExx=Y1=)AjpTJ`x>SEJ~ zX$e_v&1b*OdtrwK2AZ8aAJ>wbxwON4YrrmiyWHDnO9fz^VBJ2^s3NkX{xtgk?X8d# z4IyqJojQFwWRK=b=LjwX`);buFn8n*=SrN&E0-K`Yhd#$f-0O-fG`VN zF%AQnF@*p>-I4K|dR2Xz>_#BRL}Bu~VDFQjRfA}=(oC^cM$U*T0Up>&)i-Ax#Fv{x zkMRy8!I@l*@DMkgWFK5T6~!Ko3ghg+w(lw@kT~U;b#yqAg=elW8$(K=H&nk=(LD;~ zMj{O)Z>Y6sR9~IB1G)0k46mI3`1;T5W;RXSDtrJ4YUl~rZc|HbsG3szXCff=^LJ!i zaF8ttEo?k|TgeRCZYF1MXuuB($Vb>D3)no>aQ}OAzcjP##oOSUL^U(v;w`un+l8V> z#N~bR?+IvLRYH`F9}e?n(+Wqdm|~EP`k+dP3Fg6YlD^oTB$mC z8Us5O;2lcH=_UR!Q_u4m!n*QSPSMWprA)qd%V~C$ya|6=CaNqK`PpK8RVPyBw^?Gf zvNF&D95_3bbn3SFZMtB-zFbJ^^tF-ZI<6~d2%X=jaX}+>V#oH-geSvq0u}uY|Jpf~ zo)wU-)v9sy@15sdAE_krtS2i(zA6s@gZyR}e7fM7JwM?x@L5|)MPdX$RuadX`Ck6) z)R8~M&}-|T>)oygK?ZWsPH|?nX4nz2A?Lz@Nf>@*S*%S@C~p5VD)tq4X-mp7(3hdpyWDR z{J$RR8e$1P0?Y%?cXFm`5;}s;|4XjF-jzPYp_s^N-TmZaj%tNEKO~}?{>fEr1k=yH zeN&mx_HS~inBQu`*n|Kfi+eXW-P9O^gev-O#`EAWh-&>RP@j_d1!+IN78n3s9n$Vc zc@Gc}E3H6F<3bDJ9@reYQ5g~!BOxW~xrs@gY~?4mwY6*zk?+oPD|hAHy;BUipS*gw z=H^vEOL+l4rpLW0xnMh~v0vrstnma3?o=q0gVR(6OKwA!^VgkI$93r~Y-^iUW0&!l zR(C9AuR-eDnzV!t3UKVKntT&@`R_MwYIu`MwM}+~4j^@ZPVE=nE3xIH8Gh2rfakLsLyV7W)*1)wGL8J8oUs}WcgMR)f(<~;yt+R2OTy6{ltp~O@E z;*8g<1BXu^Kd{-(znGr@*LPNrrY)~~)<$EmVdD=Sn?iGjU%H3#h+3nxp6=Qa1GSpe ziRPQB%G(nk|8w~!aWZn;MnG6p-2S?#)Ys_STx*dmz+A`5a^;a2nZWi!jiiw>L%qjZpg{+UL*9Fc5q??P_b6CHPAX9fXjOH*`X~0=lz7 zLw7yf{2|n^EV(!Ejga?@oHX!4kgWqL$&V*I4`S?~Py1L8-VC@g)X9|g97IHLv~mwKM`8~{ zEm%wLV_@te!qnQ*Rs{jK0HnEDYZ9lpV$GJ*Q)0ebHfZTp66mv+>MA7wt& z#z{xiZ?4gtURHO0vGpy_pN)A+&6UC#CX;WqS_SKV11a>po|yta4Y6LLd1zB9)ut%g zaU|+Q%h}ZO_0-wmxs`}r;$tT}RB9jSxp~OJW}sqj?#7UL<2Y|9BQ|Nq${FMj9BzEHyne1*eALZlk~m#^hL1sPyyfON9HSJvh%OeLOd-m^cl7I-ynrZx@J z1ouS?H#Mc=I0t9_#SyJ{fYVopl-ZR#7dvahY_fN|q>*l%pI6_63!7)%X@1%tW}`?9c^X z<^utayet$w$^M0x!KAq1k&imcNuwecBGq(L(0?%*Ip0BwG$d@b8F5gnyj?xF*a3>i zdg3N`N`4! zHOIFe&sAi--S4`0CvNJs&h_!Q4>eX5Pn}l+_O!#*8-tu?q{^c~tGYHHKh&f|l@0qZAXWpY{x zj~3~lgiLN>j}9Ss$nQ$6n?}=rnPeQavSPlTlozJ_c2?!Vx1@WLA}BhlsP0Jp9@Q_8 z#4Mk!=>&grk)AzfKCUPgXKB@6d|9&IoFNjl8@SI8Sb z(ML}pO3wil%Pri@C>Y_&AP z1E;0m>kUZnTjVO@~qA^4C?0H6FQn=k-?F zHBDQ8v3OevCr=5z@@zI%xI6?WWW>9)jeu`ITvVL0{b!wLpHh2i4t0QoJmSKGH#+oc z0bWLJGTQ!deE8x>_UZm(AT{s|?quPOD|JzU9ew3o;89MzG%Wd|+CK^Zshfp4&iy{a zW&l z;$a=Dw&#nFg4} z=CDmo(p%|t{6uSbZa10wc12Lsb!0`cb9l+q88Ez8Z*Fc5w-Q~%JBDF*vVTVLlhryE zepDbR_#D;sbnt+CIL|=nF07D6<>dF7A?9A#OH`Ym zpSSlSWMI!bY$W1jW!mqd;?EZj7@IWqg|}QE2(mrdlMMq`yG+HIEgMvhiV8y|M4JKr ze!UZA@6&y`gzdbHm_LhHh0Msjr^g#+u4~<9XD7~u^vo{`sG;L1Sig0m&9PvZ#k~Qv zyEKhvX6yIa;sf_@Ij>hq;)05EUfsls%?nk_g%oI&izo}S15%wnqbv=BFNPz-@xY5 zYGWD&-_MVT-cwb(Ycryu6Q?Eu=vnpgOfp}@Q>&Iy72|>072OZry0=Dss0!8vRLItD zXpo8903$WOoO3N`qi|D6u>#|+$>&j`7GY1Lw|J?%fKgIUK>n%kXE}b^?#qInpg>SW ztM3(d5&s(RByW_j!O%gsEa4(PJo191u9E`CVx4#Uo!Rv}F=*4{m&hdkSy-D=GZ1T1 zki+&uBMb~AR02h8#$2Hh4PDQOZu}8*bTFw4*!CI59ISc|Q@PMRWLMMC&YxZ`eJHEj zu*E)jQ3c46Y<*NnG81cee5NFd@2l}uoPXt zHc;?9a9hYr0ehI49fye^7cNHZHJ|QC`n-soSgoHg7b0fowTB4Os@)fuQZKkw5E9}>aN zm+#(8+>Chr{{bNqDdP2NK?9VD{RltOlhrYTW9xpw$e2kfkL~E1HJwrJD`XoN_5mf6q6rv=${M z_eY^v?^%LQQZtyyhB8gwi#~%=SP3!Fe*P~c9(jqx7i?c5@yP$hNIbOk$T_bs{+WYE zEng~Tua`a6u&#@7iP|# zOKbe>ejQN>-Ne68Yxb0Wi4-@|ihuId-T^nlfGmkzed;FdT@bU5A)BRjrRpc zA!h%grXjE!*f7-pG{pKp0~5Z#c@NmCI6C^{CU;m)vc1>$Ex-n^Ki|)T4%&NR+}yd~ zeMTd0ZqPbPL!+-{zn!)Bqn-55R1=;EY6xy2Xrfk?Iks^3;|s=G6bCV<@7nF|E|jQO zzkLWEYuMe_Zfznn_|Ir^R_!N1QHLh=H>+KD7ovW49<|dFM3|UQb7)b1W;ahKthv?q z{GQ4!(D7n-gQ?I`s}iEgIGMMKBcc>+H{xEl5)OVsqFa;%N1M=mqWjRe(76&h_X(rM7F;D)QLu`0pSp%AcR7 z_cSmm8}~esM8%Vu@%&tWmCiV57g)Ld$>uSV48I4y6uP8Y{K`p-FUhHpFy|~HUEa?o zpH+uU2rVXyZ}gFZq8upBB~gq?)h$QMQXNBydLnT_A39+%E)LxS<@H67?E+JzDY5Wz zu$<(PQ@Ap;EJw;xZA0i{$w;;#eY9#MgkoL17(HMqPyj}g4ORGVyt@dp*6`4<)%d$! z>Q)A|({ih4n9`X22NB1VN|?`+Zgj0en6e`Euwsx+U7<)xP1TAhr0L8+GY1VjQ_U1o zm1(JnrbmX+ZAD%-pGD|)O%ex=*esc-7C)1_6nj&3Pd)=a+f-92XzU9gB3XgrdfaJhtu^xvAXhR@8gH^F_Lw;%3y=Bo4cPw3Z2ELe0yku{gVAz7(O3p{q6K)5o zz?2uL1iVSC!VD7;P}GK8c=QA#n^XcXbYsHb)3gde^CQDnp*O-2@SZT_RIB?hXeL(=1;PG56 zHfa0Hn^ezNi+&6wq6;$5qZWUPz2GBvGz-r}3C1FOKUn}h_Zwe+m-~dMNY?#Ik16(I zyzmq&mp4;j6#DJ;hndZvQC85!cC@3$EQtt=MOGfszAN!mHGPN^|60%QWb&A?$f*pC+5 zXT?!rYLw_iS?JpJq@76VLEin|XQxFpBS?FL!?gqX)|4oEkS$b*R+(}ge#`>6#rDJ3 z_b@sc4a*;yp(q8oDK&LiVv}B-3ytB}IC?K~jnyzba!6M+sjvli826`U7V0Vv7#YHK zaiH0#KAenJ*z3(&l5obo#&MV6GvlgoG*Klc4(>a624v=P)G4L6(P2sM>4{`paO!As zlB)dLj$MQ-2CHzHX(JulUSvwqHJ~QKM$i|lZ;jT>%sL}uf1~URc=i&a`}fOoObCW* z+{Dq@0xQaFT{&nIZ#eYyUDZnSSFL5~0(n1N8`}}(CY|WLE|V@zC5bFU&M9`w4h(Q= z1TPeH*ZtpDSv_nEG~%A+R)6wu!+RbOL^Sb)38}BW;{E2=k-I=J?~R?;sKz+x`LCWe z+n_$_A-21*o}~4k|BTb{_JZDLh)CF(WOU3?#C>oZB7w6ut(PWeugUScG1p>_n`B{1B+XD)|j*jb~x=+V_GpU)KTugxQ_bLS~I~3+^ts z$KpsLYz@AC%O0?{??Lf{PHNDJm#K6Nnjn2qr*?N?r>2{)b3}vl1IVEnY(3-J-*BYl z3(D@~5#edCcokXpJkX9*J!I%Oq6HFJa_KVVH19wE?69qbnjRSe{W~N-SJ)!;CzzIF z3-p(IK+Meiwy&G7g_ckE`S?4BKfVD`uF(`HkT{Bq_|@Juer*S0*WyDvfbshNh_Bw8 z4|YJUR_)88mgO@}n`p31PZYBEph%oaH?=O}vY;)nr$qBdDYdKyTq5-RL+n81wi9W( zDPLWtyNj5YpGVR6lVwHk)O}Y>Ex3u!1mw%6R4xzzYS$H%F$Cxzb`iFW_UjaPVJ=pu zbCCNrtAf(}ac__KcB!rxQ6p4$zUaqOj&el9z1sYa1?b^g283WVo-;~2HI>SkZ#gA- zE6fgVI^#4sP(L(q8Az1&Sw2keHkNR);OLO(@WO4JOZT_rYwZaW5S7!Z>dNRkPu{l( zHY2?9a(U0#0Ev-nHH0OG**$9#m;B;Mp1zj8d`UUlp1O>pPrV+99~Am=;E=QYW%nhq zciGZPU6eNzjo%H!R`o^DOtfz$-gY{s~)h%`nJ0*T*R$nEq zk56S93pXkGKb*3<|4pjy;Cvl&{Ow$S-r{%pZ{#}khVjqFwhPFq{Rh!TsR8lui{kQG zDavAXtY@*LA>V~=Or2bjXEZ=$Ow~@H?bm)ZvHQ9u%-~`3-Leu z5cY4`H`%|~H<7>DxBq198%Q}Lc~msAq%LNzbKdbl@=GAe;s{2H5#7vNY`rd+_TBT0 znK!EQga>(AzWDS&k_Q%($YMf8DGq9s!jGI?*~t<`1_lf+K{bbefNEd_Kilu;iL_vH zL^gZYd6c0@$VGffxRTNjm!~G;#9u(V@rqPiMTz>w;>ByK#3G+{^(cToT9FXslz}1} z+d~eS7`h0jV5FosSJ9Hh)OVQb|6ok+Z1!-@H(~B|a$R*sd5z}2q@l$TqNC-b?>v3p z770oCTAtlBb_kcEoezk;lAv+N^4c-^IM=G?j_og?ezB%Bt7otUVK*fuMKk*u1epx8 zuorim0lPffs*sCeJgf!s)>!KJ3y{3zrd_27;^Ezfvp*vxMJqv~H^rozfn(ptK=(aS z3*2LxFI1wQ0Bs@&%k}*WXe$Q6y8jxLQ@ujv-mHN%5h?vPJdH7t7F}~ustKsXirjO1TJ-&cB5tFZ2tdlR!pvr?Uf7gqi9f!oGC7Aw3TFP$-&dqO&Wly2 zeucyJCmam$Lxo_?qe9foT)~&UTxO96$7&}A$hAo)?Txm1g4J~{WN@K%NCz+&j*U#) zc_M9N-J5+F_`d*mlOS*hCLRBO0`6e{Z-BezB47ZDSo_8_{le$f4YAc8)8g3dSjEH^ z0YmTF^}Z#;?{Y11y2q`;JK%$^!QRa1K-5c_B|4UrRhed8+;yUA?cR2qyyJ*W&F65q zgdNMQmEFM!c_$PMX{nf05fxcnTD-UTzCOXyj`%7tMN^++>58K`f$rdq-*%s=*r`p9 zxTwJz0>sBNW!k?(Z|R{GoQ?xk5R|+6hI<_6`g$3~e*wA{rCeR6Dl@>O$gANyTvV&5 z&j+u26zn<=RMoFa@3{>#@9UEF8TxNReuZMVCEPGRlN&O5f=j>*|$U9+Ra!Ij*X}42$=QcKWjlNO9S)v zyKZS3_}a!b!cJu4EvBeKg$TlOe)-y3)D*^}adH1fTk-B&R?uLVjtMhYIaR&ZUAger zdrgE*3htn_tegPULvSSct6Q}sYNS6cVsg4Cnvx}Or2VFdilAk%A z9)oBaxDJ6x`rJbh05yGQ;|n^mHIyERj-qL~bElj3_CX+K{?8M(;dsR8Tx+wifTnqr z`#(}!gLvH+r4!<~uUE;f?{CBZ3FEZ@Y*JTH< zo(fA)`+{6>A2^d=vTNDFe(t~}C3Tk{TAck>Tur2BfBk+L0GY=bvaF_= zLdHb)PlWC&mNJbn4a`iT;;WRMp8Ocs4Q%f!EYn=X>{Wvj?L3*Moz7>*(l}G)tFTDt z|4kb3%~_fge_6PgRn?odsqhIlN5qzZo8wu{JLmldaiq_Y_{uwMQ-{_e4)$G#{G58s85>Ggt2RzBoe%-0v;X_*F{q{22wd@CV%C zJi181!-Lgt<5^ksfc~pI{?L1ElD(7e^c%m}Da?9lC?2ZYzaT4!+2I9OyD5erPPhj< zVj>GRNY{naMr|-)vV#9=2|JM?L>mvj|Em2d2hwXleVA!IcM-zG%+B%)XsWG@2_4j( zBsP7lk2XlX>zZ}~88~Ol3UrcT7fQsBHza5qjvAO_xZ@z=Upa zirev)M4X!`@(N5x+u^%@nEVvNU@!EU)^Gq)^%~bAzanvOIMNy&bTcgWfb+731Mk&y z&CK==)VPcm5#>dV7{&2iCOH^sAf^Of#Qs`;qKwehbbO9_$JOEr8lIllfXrLoyN#Eh z(TJg}%zk?Zi2UXKkavgcF@j&NQXYbbCdpn)oY7^&4=Bz)Wa2Q9Yd>et@zSx9t>-+F zyOxmUW9u~_?RhkGl|8pM?#aLYdo&r=&%4_7P1MqW1nXK_z8&cMz#5T4t9{DK$vHbiw`vbas@(Q<{`JH4~6nkEY4i}?C?t}z_b%({V;c?e!HPpwYl)riJFS9fC0#n(Mhg~plF#{R`Qm@u|kgCihl+p$SST# zwy>jdFe$!d(EIm1U&AgKWZ$0_7`Zzq` z__5B%-H+BktvyUn#&wNPF1A0;8QQy5L+~}MohVrs!W-5D6E_$mjGxTMMK;PZ;EQxs`I z1>Xz~$54FIMyL+Hzyd6NO2DvGI&ZABkHgEK?u7Vq^maca2`otq`a8)EhF>v2CRyBhtpqUk=i${zQU^i^ZqVTqj_!$Y47?pEJA`1De8^x^noTgNwDK+l zqYf6f{91b-t#!G*`NPmEai@ff?sJ}r9o-M3$l&{HY|hxL=f0dLvHRW+VmART2(r9!~ZVV5r=^>ot)CyH+X_Nr`+_ z#(U$@It)KdL`~;waeKmP0Si+1uR>$MHN9t~yyuLDhUBwj042U-LUy)#5bK_z{CQp> z%iHMa`Q_3Q;ClWxi@~AHn>QTIB>*qw0E@JAGKS(pMDQU*Szd>Vs&i0Z6Tp=FDPO$y zOiNq+u_j>1lOpfs#f2T-MS{H!iw@4_;|qUXl!n3${1Boxx|5!n^g9jpR*xrSX-Ny8 zFp3!;Y#dwK*oaV2^%j!knh$Pb;heBIXU8q1Zh;<7e?Pd$lS3$r0bYEs@=}`nuPn}) zuWb;sDZ}mEG=eS9^ArjQlafVwg*aaFr~!tQOGf&_Z48ZvkZTi+O3^QSCsMGQ4k2L* zv>e>d1~zI4J)s)ZTXe?&N(_}{4EkvirJuSLvher#;3ZjmSh10@BQBH_&=_5GdT!opX-xKItG96c>cG)4*&a8)BieEp@&&MUzvT;qh}6XyEs=vP(f5F zSGC!97fC zk+?NHAFueq_w7OHd*vzyd{(nR19u9x^h_S6x=B#Z6q*o+kybu+%+b@izz9QX)7z&k zNMN+RFHPl7N&$ao+!#wOf3G1$(v%ACex&*+NS|@*aR7XlM&;jG zG%P0u3P^*$Raa6B^h1K!&P=}-8hpXg|M73G4|RjGA3i`0UUuXdb)y$gs7)jPW~kS` zZoV4I_`pY##$%&Or|1D!<4<=7Kdq4R)fCgi3Q{DlWTqj>tG!ks2eIo zzDkw68YtdpUP#GD39J~=(JPv_pp>_JD+jnTa;-y>4k6cTug9-RD6hjbatn31aBT>`GDJ8^We9FY-*b#2w<26?t9I979mtaBE zvdMDDlh#?En~b3~vW7W)XQNh+`lyHb+@Qlr+4ijiwGyqz9@I+dT1a^MF24`=~6nrGp56RMK$X{#A!O&Je~$P2d{Wt$SRn=gFJ#524=3FGrG7(75O?Or7Y$%P7Tmy}RhgEx4Br0(^E{ixJ?HPId3%ymd1lt&^F1=X8Ed1o4WW$2AravI z)X|4jZ5^kZ_V$%HVMeQ6Q~M`#Ex-WMwDR`lpKZ892Vs04(s>*5;+~C6>V=2=^#YNT zIsC5nGG*muf7I(?pq(tPGaMzEBpJiZ={(;?+d~<8|FrZ|Hyob zf@Gv4Fgg985=q6tb=-X{(y&T}_>7gu=u@Q^b?X^HWly6YbZ*^MLa;ORCX}siC|Yp& zwe?_q6q+caH7=ie1C#*RhTs^0J0NTY`cE-6LL-#!IyP|Xv;eGZ7>U7S_Gz2(<7&0S zlfSb!J*wgJye%j{|1U>VY@uBdc-h3-9OB_FiEDeAv2Zz#w&-26#9*V^!PBfg$@HW> z(*Vvq_|kc2Bga(Ud}CL3aW$)zO{94>yfeL9LPfrO`pf2dYhYgYAendWMNM>AFnLA! zf|&b34fE2>()XdppUX?tr+N|WNh;LJ!fCNV9W%KYVjgRD;f(L%W>jY;KjFE;l{--k z!GTHOUchGc76>Awg;9pZ!dVn5nS=W+rXW`IQ>*9Wg^8Q-`>a#YgZM}D{out-*(vwT zeRz3|4Bnzr@M35rAkNwbfA`jbAlYGXpCiKfE|?92JzLwFO+2Y4RvlFWNXy64ksF0C zYV#3v5$!w-v@iyikRG|w!VObeSPF0LQ(O^DspW%b zzrpS3bQGHj-Xz3LfWost3p4HR{x6Caeg`=A%;A&zb0eI;F4U-~ta~3a+xM@M^gO@b z%^pNRq`{Yo(N&tUiJ)Q2k{*Gdu0bxQD{`se&bi znnMplw?~A>^v#`EFEWm?qq^qS6A*?hV*nvewkV3QTdPgP?baS-(Idp98n$71clkVq zM$usDyr`}3sMn@ldMeKfM#Gds%A1t6nnp9V?j??jL-QC1fEG;W@Nddvgtz2C+qt7$XW(qS9!qY6y`JUA7le|5XAHhQTjb5HA)7DE`fb6KIo;8i1FfZ6(}o z0eZiJsylcWYrI00$Pz?Q!RS(;0QvvDlq>PVj$Mv&fY|!S;j^18D}m^#u9f}7EUHhs zyGiM>kVlK(qk9bzT-3eQ^6*)@l5p!^ZJ}^i5Bp(HpSc>D$g+mEyCHuD{M~K^kA|3l(OleN`*rMmOOf$ zuymfaDcCZCLw2i(V)@*Dgf#xz0JsxkkS!WiLl}+MY%&*8Hc43fkGQ9py{^Bk4K(8e zo8}%k>RsU9i*(uiOjrduZjo^|x-)cdCiJQFrmRn*ePfJbN6=B`0#=+~4&1!+yZv&T zgY0oDvG~*S(S|&59%f<4 zA?VW&>OKXm*j}XkeU-$uye^xKG;f!na@WNoIStDu{K_;@*Yk;qcoRa*!9yGWxCAnC6!<0@FytU!)8)h^$R7AJzdm|o1k(U!n6*P{eBxP_+nbvJ#)p_f8yN$<@3A+g~wyS ze{`eWY42*77rgzWJC5mqb_W8C1Id?XEYSOsV_Jn#vX+gN^IlRI;g#jaFxF4q?REGX z7a|0cuV!B6P+F*6D6#F;C~GlLqtGHApo^SPBG|*H|72Ny4avn!4c1h@69ItXou^NK8XH*v*bW_;nXX!B#!~@q^gP($#Ie>7PIIvfG(RbdwV_jCJ_fMjE{vD_PJh3Ncsf4Kl7|# zakRB0;Qgl+WJWCgM{nn&64|Gidu`7+rskrxtKnqk%0kjppz!)v$?xZzJB6WYXLmw~ zCG+4$w}-~jyXZsws<{u_;@9Ed`A&^0e(g5#sJ~RFuQe!kWx}_6Yi-BgWP4%9{fB-! z8q->A4;Q1SWeHqVTWnsEfmVnOPQrqqm=ueQ7kb9K`vn+qmPOqVtsZ6{NqHp~2Q3Xi z6pMjKTcQrYSJejZ&ERSBUW9PCFgLi(K)r%y`%Wv=md@FQmd-7w2u}#)y9Wf|h=V-6 zA!Gzb+C}^TXO9&x8J)2c>%Ol}@*zChT)BThAPgEQ#xySs}f&6N;qxI|MzDa?7E#duqK&pXJP zpSsQnGCv1|F!Oh=J!j9gdJ%%87NAOIhVN*PT*-A)Pi7stQ~d?;{MmWJ``^TIxsBpI zUeXtZE;sJ0*N>MM%R%)*JfPEn#iLY_2d<_yqZAd_RNa*yRQKQeQ7bW2*$3VTT~tO* zSXK$(h{$+}klnc9VWi?0nP&FoG(HuxZaKA0n{lA=-fb~EdnK7=Nb%E>Y-MNpUos(W z18jd%-cxUpV6KOaYT4W8{((M4ZTf$&-@}Bz%t+n0f)OP%(aeWjgb9O>yah1eVhL^- zdh8c!Z%oHJriReZ(*(R(hNc+Q;|^$HgvlBV^1aQ5G5aX)9iT7#9*W^Cble9reusL& za2qD^9%wZ>)={#ue_0J=vzQ18zKUl18mn9Bnq; z779z(LUgh|r+OGXLpMQpy)=frH7eY}A;DYh`x>@%KJTA3T%H64T3Tt2n%DCV%wNT- zV&8fAy)3b2)|N1R6Cg*ax}f_A%bXz#ySph?f23O7z5HPpe^T@G zQO0zjY^8ZjJM^X??@aWu&z2Vk{xyY&SINx%5Viu>YS@xWLYR{9HcyZjQ1WO($k`Vl zAhMax#x$^t)fm@TmgWQ^wF8$&)b3Xhnr4^X2+)_4>4YQV_%jE#(k3EZlWO_0I2Ks(?DxEU~HV-N9wUj+mbkmDB zL~8M|OX!wx7KVQxoJ4!=d)!BOm}WisDiIv!kLZwA6-F6buuw#pJZHQe6~8yLS@5$V zXzQ^!rQQeOi{|^>$lfMLNYryq2g{?M~{$@#sThiAe(HakrUMqKLkHVbFj}au{VUczHxuN#_XH@m=+;DAB;ZT zhO_jJ#+41a_&lR4SLu+LgE5{dz37w&MRI5pFd1kkLnUKs>ZbYS?R!Ind_%YGDg)fM zz=aUU?3P$|zPMG~os<<6Xi7tG;gRHCECOP+MJl9KM}6)kv@jweWqYZ1ay9Hn&2*-Y zpvTtvq8fg@DCH|(4LN8=3ApKvuQ?N7lv>>+oha|IDZEa`UD16w=vZx0^Guw#$&||S z*DQA3>o)-%YhF$6_~)okJ?>_Ppd7a01n*V1ndQTMl%~GE5Mkn(Ijnf~7@lx< zG4I1&-fG&D0FG!1{@07}PuW5z%-8sLF`Zsn=3zCPWFuyzfRSKN(LGa^r4&$v*u;!q z{mln1b=N`7T@*F&Lw+^#XvQDQR%{1`_tonHZUyT|J8J=iA2}m%i5DY*=KvjFIqqp<%}3Ut+~!R1-Amc0a1#J zyto?R_!{n?wb~x!RL`IBed>4drjSjOci3cgJ9@8a3#kMR*9E{Fy29G?ec*NnYSDK^ zmXIF5fXzaCP2Fn9hhHDOn?du5Cc&I5mQo9FjS^fJ6$q0$&66&MXJNnQ58!P;#^wbC z$^U{5zt%#40+0IiBu+{7CKU;(VpKp`5r6d;2{>wP%S;8XJ()V?WbzPxpWNWbY6BN) z!1{6JqWM0_*Cdo=A3?sIY7cY#c~Y~I%eGM1r{~j=r$up~+o`Me6j}{e^tVccmFFPn z>4ET(Cg0jhfEyJow=Z}m`Q|haB^la283#ntQ_F~LnBJrxRwJkdc=z)CDYhmHr8@LS{l(@yKpk*k9EpegiPcW>b^rSVE0sbms8^lDac$?)9#4yTns|V=NriLQai+5)6Fc)-zFi+1?dP1hRHr~0OiOK=d_!;ibunO9#?=CGmLeZ4x73NDuir< z{qXIyq`5Xsr35Itce}a{dMiqE9Y)2_DS{kU=TJY-^Hh*h$P}QueQfeA=jVwRV7?cu zw8ggJfB?7g-m;~4NwM(YkeaWwus8~91gED;F!nh+a=Yo~U5YMM zl)N`3mAEH){BB;7J&n19h;j*f1d5eZ1gg5^uPyQkM;}tcC`0hSzNd#1`+n_Zu9=+b zz^dSU8-ok=ckrffxGBr{tb&9pIH=}zjvF|{{IEbspX4CQRS%*HP@1w@GrNr25;wGCu;eDOAH-P%CI_LZ>4= zqa&@S4ad4kxZ)5?FkRkeo1XZD#p{ z`wIVCq}2k6UJ|(2=XfiH3v({U%Zs$`nZ8VfkIy+rH|Om^(DRl3mAnmMy~5z~IfV^Yr8Vr3FwZ$93AO+adE(kM^6lNk>SJYpz5E%t&@fCC6?XL5_ed}j z*R4!~>Ac5}+mB)p@!|vm6+xe`5KVS0TYGO2bxZP|^GKOnu9+i(Y#~sHif)-gx|nv= z_?&r!G~e$rAzM`~zR6pFD5-p99LVMj#Ig%hIa4BbF@0pJhIvD;Lvyzy=CavxXqRcF0h!= zbk#yijtI}13?&5FFvqVb_`J^D33+#|!BePf#wIhGANtst-eE;Co&(}SqRlI)UNbh| z(vinuq>c!mn(K1k-dYGR3bJHALUhN7Tre+9wXmfIur=1CX!D;z#(?WO=Ws)88zY@Y zyK@LjW;O5@*_HRpm3Yvq{%tM(GN1DJz$ruog;^X6=uiojc$HI$qWJYO$T2JJ98aZU zGbccwTjy;giIiAvCT0I=5tDuG+jAqSEK1OX#r#)y)Y-x-U2g>QHQ&CbjiR;i-hjnq z>(OI>6)wtoFsI-WZB)55y1C?oMR2ZxQI=xQn=3W}538&ibezmx%GP^|{gwV*P2BD| zXkapZX`q3MN-hQkUwzetaj`|g$!OOfy%zPUTdU_YTg(n_-avs2TY{CV&J&L03qiU1 zZUofXvS&$kKd(|3MRCJX^&FV(h1?A;V|KKCx_+YNjkL5xus?cev1}|0j5H6rlLL+O z!^SbEtW{nMJEx@N$`SgqNcb{-Mhc<=kzah6sI?7rkjpCuvEI4=1`EUBiG;apMQP)k z{>=npe$b11FNKn`p96@M?}z-Bdn(1HHj!7Z3*rJi;zK+NT|}M?G( zbhP`4{-{dqiK5y?dVCl2U68pfJ98GAnaysKa8Fh1r2m6bm0zL+Le$2G%|O! zFJArd+GCbiGK$Qz+i;||<=-%N<8Ie@F1wnu{=O47wCS-AudCJ=U!6Z1We(U^2O)Mb3Vd{Fm$1K~9Trd39J&B|^k=2rX0eFwcNv6V)zvoha zZns%}KO1LU=G8rmtGn2KMmK{5IL@3ew6Q0@^r#qd`%kVwWyF`yRuN06nLO^ZfVpCX z;GSfS7K#&S$2w;m|8gG~13&MGox!y|8T9%IPBFeYs6%aa9Xt;xEbS23fju$Ls=0t$ z*OGqsB2>@T7m+AyoCsgSWY^TT9-gpDAsO}2w-1^;|4H;C4R#*X`eQ^xwsE5>z}>xU zt1HawsL*JmiQ(vV#3a*G57hUzyVIq|N91AxM=lh-I@8MZn9M5?0c3}Fvb%9U(FB@8 z-Uj5q2F;;PBS2`zA@rqrB(GTo(;w{JKaE|?A7?E}g+0SgRa>?G_{DH0fD#U5MO#=n z|1j38YLT zHqiSV2-^*2UxciQTW&(TU>VHSz$~@Uw|+(VM>$m%?iqXYTVLCl_AQQ~&Ur6%o-gy? zj~Z-96ukdLUwUOaVkA4&qcA{{6uH`*VS%LUi5E9Ubz4o5P1Kp20T_vwX8p ziZYp+hW}W!FwWR*HZl244zXweLb&@4;jJoCKt(e9_;n}D_oK_ZS7#3f$N{P|$yjdR z9E9a7nA}?kUG*0sV8nT7W9nWg^ym3nAsiVg&}8T6Esq9j%BNP|ADm+fc_CVWEOB6Q z=1#exGh)tdk6APtHvS8^@7tlQQy*40ckd#ti1{#IqcveA0(Qzm{n7Y|eS!NimO=6b zu$%`-`9|Bwepx^aw#F_|_-NJUR5qo<4g}WgXDt_08O)Y^y6Rf)CkEu`+8I@4EPM)S7QvgeIS{KZU@~e9biwj*Vopf~NK*Xo+B){4Z26EALQxJaFVJF$DyX>>m|* zTk<$IjXr<`Z}?EKQ3&Lc`A2-3-CC;&QOLqLQkbc(FgGpEb(pfvn+!L6Z%I^cRPPEU zW6?=cM@$U?w!dman@v7*0-<(P#JJH?KZSwa!N`tN#fAuUVJ;MXCv!z-_+D>k&4>K8 z;$A{Wgt<4O|LEEDL6w918N{k*fw@iZwHxFisGIWilEN_VE~w*5GzIA;Aks@H(#xnQi1dz9 zf+8Z)dnX`BF9Om_fDn31fD|&BIq`kZS?fFN`!{RW&dOxZ?0aAL@4EJB1+C+PJ?)?s z{GV>AIUFuob!akjq*trA_SW9DRI-dD)DimQEyEfSw9&Lbw4U?wWXSiNbe$qIqa1S! zKRpvBrStp$fOkUu@2HLB6eLa!Ce9gpHUG)L((5|LcoAz(bGP z{%^>L)I(%<3p$&=6%}%2(jJb?EHrc)BC@b7dV|#3&wLB7%zpSr<8hZNKDi_~lS6ks zxmri+-PAPt3-9MDkC^s|e!hye*(2xOsnk~=Z2Ypxmr4!pXC3cI`7TV$*}RlgnMfRAJyQ@9#( z6}r&-Gad+GJ$vpo^O`$n-s%VaW25}MSU(E3U?=KU#wg8eKF)u4dpjeL8ys)a;04Ul zfN;%iYyh-AGl#MeeBg)a-0QzEly851!dTzn!q<@UfxY?OC?x5ABX=a*BZTm_-k~pN zc`_kswy0-CEA3Ya=I_xBlg)pxr(~I%URG}$e&0qSH(2Xg7?g``aE!KM6kdD=gTv3T zl3tE5*t@%*f@-k$k(bg-SD+QYSV%^i`LgwHq!u-pA(-3hE3LnO3Oqh^me1oHEG+?7 zOSNNmwA+N>s${rd8H5r2!2R(@zb0%Ypx)3QJN zBd!W_(+~v5N%-3TN>j+zH*;1JkV(fu)RB@10In-|C^*kd*;&VRVQ*FiNYG4uxs=#+Ti<5jiFUU2Jss>@9bW8Jnp z2gwj8wDAxQ@1MnE)u4A%Ta20RuIWv$s?z>OdO`zbrtXsG-)DphV0+b_zELHABBu^o zLsSeIRCLkX&fFSLKN=uh6ikX8zxC2qL+)e6lUtvY*Ba6c?B17Wq5Z$rnl0G2Sjf^x z_=nv1MJFgXrJb|TYgK4gH!116iuAh$e9{tk2X@@ezZ|s(rgt#ZVEW7WSoQwCvW-4u zlZDc;lz+g~YnEZFN)wQmvY$%iB3hH@#mQ3pERE%k=Z<^E~7Nt^_2c)kBGqmkB&P6~ww z`MXY;Z-b;Vr2lJDk=R%C{#}%3`DCxXn_Du8l#Dgyi(9`DGr3xkvBb_p1dN^%Na+K4 zYLloK+T`+1a%2QxGS1b0a6%!;ACg*w(2NHW$*~&E@Q~7l1S(npvQ(LS!VMpM`}Ixy zrHe1uoxYCtAB3F`NT+Q&adN}Wv?*W#aUAI9drKlpmn+D9ie}b-i0wqjYCnu?kHoy$SHY0!)tp}d<1&JW>r#DKrKzHS#rd&`P2Iv%wFyt}Kl~!5?8XHDdds~(Zoyibq!0M$fF#$Tl%P&HzIEjeuPj;zY@oHJcHx4yg9%#RwI1VF*mD7f zOos7hz>>UY!(gLR9HV(J`ar%l2$E9G?+$5;b9IJN8CU9K;ga*f{%wp1+OIpc8&glv4)BmjX9xkcSIlndOd6FAcE{)~ z%WzkInYG@VNO!libNI%|D`IIfW--)$^-n^zJgZ#%&{x+pyHZ!Y_3}3i(p_{Rn zZ<`pe6=3Y!hE^;K*cdFbH;AxGMI+i?^t`zFl2hLn$X_dpoRKe;qkH^DMrrFlke5IS zK^#2D?}y>?mWnNoGWMRqe_B}<$4^nMNmvZ>|F4^lcqk$};tShz^)K$r945m1JcYNN z-no4Hd|Ny4pjm5k0?);vSe=66iAk55N1ncltIGP#yU)X%zdl~m;8@z*$Gr`>{leC< zEK$e;lT{HrcHn^)J6PG~w=omuN|Z&yF445Z+gJL{gZ8yV^M>CQToV!4(96^6S*5N&W}zrjkqId_+CZ*k*?u? zzYTA(?5?ZeT)uq0gZ_KYG0VP2Y`wLL=iA87LwDb5yXL-j#P7R}dAsK@QY5(!S+h(D z3Nu@aM{i3{!rvGUS7}R@?v2db{pyU2iCi?_%V6XO0!I{<3w0To)MMMV#RgJD+5MUZ zK4X%pNb;WdFKT?tvHgW5pm05Pi2633812DD%(}_;B*$|D84r7K!V(l&_;Ta#;A5=7 z_o_B}(E_|qLVd3ZxtqGBwgWdEX1~r{7%(qhvP&!nCXt=2((nIlNItl2NuS;fV!z=~ z!Bn1yAS-0orkb1@-GSTZ8{?jZ2QlYx4Z^oa13U~Z0EK|cF}IUujxoAli03#p55D+& zdbxK9O0}AvqCOA{bN+os0{@t<#BnV0&8;R;e$(HD1WdhEd*L&&D`fd|aTED}rG1{w zVy5nXc3TjPl(Fo*S10@}Fj-}F#UyF>odf!1B3VA(tW6_3F#F@{fnw_eIjty(Cs55=WIO!0 zWEV@@L)N!xEisjce%}$hN$WNq0NqHi**Hagg?Yf9IJ&Vuq9S#>GK_5N$=C>)vTh59 zRhUPxYZqc0n;yP6&1rKS?ZG&D{9bGNT?+Q&`>&FR*ZK|T5yyt!9qf^gT7XA(0Lpys zG_1tJVBHqW{&3gza%>{t#3c1x7xD@MCsO=J{=^;6dl+7mAD4jGDQS((Rn1=}53IK?M#3Pv9X`jT=?rr;nj96^8Dhl{3SFA=eLMX^yrs2@N8tHcMGIvz-uhD36AiGBgAItabtnB zK$>POzCGz)&_g8YDev`TfM|Mrq4D5 zxBMOJo$7KJ!|}3R`P<^7IA1(q-8VsHpP3syR`wekft`F0M7-`qx;FX|PxoLIBy$;a z3SW}libo);1gI6%h^e1|KEDFE1h@c=9Dmy_O zelygymB50mmwuW534p&~#o_%T*ugpu_$?wXUbm@(ys7#-5P%w=hy~m~is5A9jb0TQ zoMy`F;Mz7W6TYKy-e2NCndMWDJ4I#=TGY>xh(E21nT~(Nt5k9=xh&4)+wchgxrH7v z-?;qALGZcWh2rQudA*B&{>!0`J>K0M3kopHax7m_o?w?az1oV+dP3@_mYjfcW>O)U zQK(8+dWer0EcAp^?kxJpr|5xNiA5R!!9GFb)}vnOS{kd<$257(o6yKoGafBr#(cnlNtaYUDYY_a~hcI_4$oir${NV8- z0}`iTGo`}JX(;tH(ziyrW)e7xI(0%e0paxW7royU7W{y94J|qF-vIv-lE!}Z8(fF1 z4n4yzTCgP9hH`!-2x`kU6Z;;3@Gq`~2O#?=+-8}#k@?#u6gQ+5Ld_{K#QvOYB z$A9)eX0I`ar))Z6vnlJf=S{w_pZr9a$C!pzm_FYt;QI_vW{5xEuiJQ)af>7voPv z$UdAT-Z*&uu@QersoC}MH>v#6-$k_kxLRLx3jZIv^oyO^M&319?6JRMsqzovf%``${|0Gy65Xi=U6U?LNRI{;_mf z^vUPB&;JX34mDAkET>{+rk?0FqBpcO+%8?yYmcK8=lNp;HG#J2cyujI?7L$M9072u z#!hiAD+v$uyKBN(ciMC1ZD{uFn<$X8|?MlpV-EgXK=K)Fe9M2Qlnw{8+DNGB~W zvHKf}po2U_Y#Xu?WT28Co%l>cu4XjYCL^<2czgY+YY(4T( zQg3J?I%p>q5(~`zSE3t`r@c`_1Y?n!qJTZoAcl@CH8c3(vAoa0Mw)ID^?Us@ocDqU zSOUs2Z~Vu7xVZ6M(n%(0byu8da>?y}LxSe4T%!HkTxq+CwUVOb(AYCyg(|KwZ8yrc zq$qn*|5-x}%&Y^Y`Z`)gVtM-6B;V}KRvt07Y{AyOLFz-uhqs*IT~rOo+F~3w+>hmEn20w?p~#6|ga>he8dEyQ_B#_dg%Bi$4~z zRXFPUk_c!;Fx*SO$x?BQ=f& zRgGzPF02>HLID(iug)yQ4Pe>8oHOFhG~uQ`IUeEo(<*Q;uX}J3XMVo0=6k{^aWVC; z*gcx9M2wuqv!Azwm_?AIa~t-20{0?T>*ejTD^zJY{x zkXTe75)K%6og46xymc^-V-)-J#LUg8$1}~Fvp3i)TrSeU|xG9X08+27Fw~7jJ>o0 zjI3IKYS7sV=1cN7*Y4Dd({vXrZbjJ8J7g3S*#omEZNDMa!w@d}nPEBE4O-FmPVi3S z$NsH)QLv^x5ns4`fu`IPj{#qbq~^oX9t^^3!S~4Z+yc3O1c=p2oQ^|{T;k-tB? zwIk5}Eg-gg^PK%R4^!{vth(u5V|Wp(MvE#fwgYau&Y6?$Bb7Sf!Q`$(O4L!SwS9!E zQCt;!wWB>--YK>u=L>-SW8$d(WQhG0Jd)T+MR`lSR3)LG5JOS#m^n(q)y@=zZ5nZe zRNc;tDy-SR9?48x^_e@Su%|^5*!6DMF(btI&kU`8-iF%IejYU&m6PS5EqNwKffO74 zyym9Z?sjJ@;9Jr7&Zr`+P+f~7zQA_*!zG=UW*&-p`3$l`#nnlBL*Y?5`;Z$qY!Nc{ z>LkbZit(Z(%~Eq+>C`xr_kndb1^BAGRoRUy!)s8K3Tcatfs47g0fcYr7kKMsROsZD zR8q_00xP@9E5^DDR3;F+K)`mfDTtARM$JO8z8s2w#dF6+L*E4(J9&ha?T!15^;~x- z-yShRS9G!F10zj#P`sYTbjr100q4dQ`!)>gZ^YNG@_D3i#YKEG;)V@)JI&-3ODtmK z0y|`v?DZ+Ca@yhZr2zp3+J#Y+K0KZZ>wND2alF z8PJs=Lh5GvyEpO+El1?9!$Gvz=)tS&S$*=9OR>&2(h%6J`ACO;7-i@JZ45-({ZO{E z$Ce)zDNV|cD!9e!@jyARq@H0P4baDzOnaFB%a3^G+PMPUYmTazfr)ZVU%`jgCqzt| z$=Hd_-m@^9V$Z&R^ZV?Mt*hDq%C#rYt zpteDG>Rjk4@uaT%EWCyp+wX)D`hy@3f((ZIyl#u5)bQHbZwu-kTMApIaX;&qy-~vA z&BUtNha&GgTM@L&m{6X-@2;tuZ5$7a9$egE^FE$8O!#s(*0!=zP+-u>xn{)h%Vhg| zsuYghv7OXC*?wo^C=+Gq{T8S?WT^?hQ6)v^XVwj@9EcjVua#w%h`2~QubxYhT5Umn z*nYRB1|?85iqH1Wc*i~muRC912EfvZs~HEdjF%@!jZFW3=-gLB$bO)y_BnD)=F(h^@r z@?C!n<}k%J!h{qjeFca&glz;M%vC^lyF1x*FrdUO_mUvuX4*EDqwD4=vU*c(J6;ji z@!j;m=$_{ypl_cv5T|t+EQG#=ql#~ZFrv$wpC*6?6311WPQ8AK`C`KQ-|AeA-a0aK zrZ2YwXZYpE%IQKR$^I;PZPE*z41_$u$w}BC1qeS?dei7>f=A>zeH+@RFd*9B$!J2) zJfGfzNFW&i)(tbBRXSR7og>X$*dJnZT<_UsRT`fOVf#YcO4w)kc8bJVJ~!{#9)-q_ zI4{tmd&E2C^VZNzC6|gjy(3NmJgpr^D>g0wN99p~34yCnwf=K#ht^chM5avzl=bHD1WPzNx6aO@4?rczVP*Q4vt zo4ok0O3Mo0Q+J#!B{ogW+l7AeO)og;w+cD%+?tYp! zG(?G=*W~5HL31-n|8MJXt?_<$?t6y^AfzVbqU@Lc?q^ilYQ|lpJDrKumzXwzQNx8 zOOHHZ5!s8~>(Pl)Fn*+jxhKc~s5_y0bK+wad6|dS)@DX#0DW91(Q0I};#bM1VTN`a z5O~XxsI4fC$MCCoGtv4Dp;2rlYe`A^#9bm## zfhIODwnsY-LOzR#$AyVNwIER&Y(EP$eOtgy%p)aC%HS2wV8{0ap^w;#ONcF@f&F%=)?$dp!3x zCm<$~sm}M`Rei&toAXF|#J#N0-3I=tdtMSy=bdVJ&NA60i`e~iDG-{Z#rwAY7lN_R z;BojnoU4E5r(@gddh$o%CiBOf`JIq4%1kfY@l{jM&XxVu^}1fS@%9wF<&;AkhOG9n zgjtedK$8)8AtVnHt_Tx9{~!+F3zt3z48e;%a9cgJ2=;G-;Z}9%mZ3H2>v)VxquV-D z!MxoAd?G9$e-yGX5ZL@dI39i*Zh7$ahv>Hz6LjsGAg?X>J9Dj^|GgIsCT9ec0v}ML z2!$iM>jbWoS=-o2v#pn7VPnogz0S8gh0h^D%yD1^yVFe$lkt zLJLIg8|MtP#Sjr3!iES*oIfRYwq$jEE_IJpd z>>E+-jpcFVqrFT6v5X@m#3)%l>P_KiV8a65hP8?xQEn;^g^r|WMQ(?v~wR0-bymrAQ%Dd0Yy(TX{yXXj(ES;VcvonwC9B22Jo9&DS~u_=`*{1#ml!(Y(Lo{PX`z=9 zmhNnic@}Ka0Ckcr&qhix4NDBsfy~Xn00WN(37p&5y%8N~1+#$5sf>gt&BvBpTcH*q zSv<;uqi^+3&q0{}L5$IM;j_nS`=EJkGwXA(rU~Fll-6>hFtlTC0;jZ?!p$L5u!UwTawpRBH-g6V$4&9`!G;k}m@sNAl+Rth z#PXSN^Q966Ass`DKn~&HO)M^GwP?9hyRfb=fGg%AcbAsgk88m>r}?SgzkN*JgH=Oe|{%xhiz{HMzb!|gYcr{k#KqP}b zlSwE2dNI6-`P;Cn)qd^$&vDLJcXN3)hyjrHK_O(e7-JF5R)8H*m(4O^-d~aEz*+e%K3IPjPT7uyq8W(mgYa3n za*fe{qiV`HW7kpp`WC&3-rbijAHi|<@-8F9x}B}g;ORvXt0y#qs?rmIsiHLhzM4jY zl#*}Tor2C@bKtxuO`m?PBH>}Q)x+H#m&30!R`=XT)WwEOx2$5_gIh^d=UAg1onFHe ztAmZR-&_woWvS4K!20admD3~9S?*pR1-CTetR!s{KD04!_KL`h4%L$0aE-Y`MeXSk1RK~LVox(%WH zE&O>K&aDj<6W(*UhZ}zVxDcLwd+B6InBV!I-{8n0_P=77}(7GYE6fRceAQMZ`NPlC?d4 zpiI<^CD^cbKl_r;=4coh7EsWy1}NcVz+ka<>>Z&<$1XzgjhwMZyVS8h6!5d z(6x12^3{~v_q7iW>Q0`&X<9BP?&9c{PDsvu_x2J`srdN`m+Zingr!!*<7;pNdvM>} zwQ&U)A0kG?ut6`O{cRMf_rLu{ClI3yHmv^4>ng)A(nzx`iiS|@lo5dgLw!h#V*Y_S z$cBcg0DL4<9Q2L8T^EG__{nZqQZ-2(`?t}d{jYR1JRJ1O?wy1ror40uA)-0xtD;K% z(hGYu#2+3e{*XG0t2LZwmx=?r!WMwFi&vKRtzz{L-rqiU+0RJ|((;CxavM9_C{VN> zrbH%2)2ahr-O`VE^%umFIE%y#%mOu+aWF)oXkIS4&$)178o=+B))s4ZDutwy1N#mh zi_-iIr;`r!mGm*94@v4$P|}ptyRcO_w(V&~VqwM3mB^2Pe$4!7;F>>^#qyhe*Uvx#<~SGQW`|N zWo)*p&LP%mXfz&N7j$kaecV&2w>wdS8f>;m%u1rKoJO;5X**Zeo`7W^Q z;sqwkUp%$F@9Y=77tCuki+g&I#v(eY(3mOVdJp&6xsfqAZZxjsR5Ss-c77@2fN=g6 zi1Ac2JtUSVcMgsuA}$NTDooZd>(n)VS_;VjWGx(<+J#IXNyNk7Ck)zl$wY4x4Al{< zKf}DYiM;Xz>%B(umps#B-s?@GKKuAsEOYGpE8|NU9EltRoM1lo!g0xa+oK}k1JflE z1H|a%zL5%`y@cJBX;z9(SwH(o7UGVAVO`EM$KY;q?!-O&nXZA;`k|tfSH-sI?1KN{-Ik7>sD;!0dCq;SJ z^S@$CM%}yQw?!9|PYc^XsxTc4328|i)`@zySY%oj*)SE+3d)KHlsdY~tFwww> z&)UBW`Hh6Ok5=qDG_suBkRJr#52{MPcHW?t>DmmbjJ3fwRP$7#ssz*JeU5+O(+<2m z&U?x+638;1D#ef9!&cyN4vUE6PVH1mws@f0N)&Mmi-QO)OMoFn>T#Pp^l|zm28rDL z!XDoHtLi4~WjcB?uNkq=^BZSgP;{eW@zF@AYS*ih`LpKdl0&5B{VM7cAG4b-!at%# z^VT<>%K#s+sjN=1x{UHhc6Kt8Z^nmRzIGI7(NNB5(zDWq&3Pc;3whGOv6KdBA zMaPi5?vw{z9MB|3*WM^D^Xyz5j^X0Fucj>?9ni|(#q5vb+{v<#=<$LZgsGhwqw}$-hvWuLPHd3rDnck4+`wM+p*+~oJ7S{ zG>tgQIP|Tyl|UUn=nhrUAX{<<^0vM<2~y3v&&t*)o%t8vdVU%XTnQ)NS881KB&loP zq|`4d9uAsEkKMg{n2xQR*(>}4Y*av%JPBvy)I{@Bc>wPUE|zN9SFe)-NfTGsz6fUq z|5+1Fbu4&0tRbnQxNau%6C1~Tj&9Sw0v>%~7#3DRInA=O+qwP@lj7^SrTgS3QJ%u?vXW>-;Wkh(6_kVW zpyX?xI|%&E>OM3w3Bb`hH*1TT;$dY1hW1G(Xvtg3W|e_)jo7TehP-DOV{kB;1)f4M$lMmH?>%`t@Q>^GVrLe*-elO$6~_^ z4{MoZx-;CAoqp3W*^I4~9E|yu zNjPBGUhH!a*sL#wM^uf=iAQ{qR!d4vXJ|7=#yiYo1FTxCUKYZ)K~P(;95Yw&3w!Zp z9KggT76*3M+pt+tVy9GjEoa;s11vWUFi$@!6vbd2vjPR{mgOJNF*dQ2gJ8Ti&z|~L zuaC1%*i$o6%b%1^K zi!i*dtf}d_v*E2W)To&-8$&((aRFg=^}`os{E+?^!)N7=@9tlhs^td(0l0-!H}ybS z`cdjjF%rHMvx8)VPJN7^q)9D$e(#V_;~99~Zi!t&m{+v8q=AFU9ii0FyYDa3bzV@k zTgdD#sEVCnM~3k=r^J?+`ly_$A}$R#&>o>Xe5OFaOt;<$G`Rx}9*hM14r*5fCpzcn z349mu<%|Q~oPO_{0`Na69RNd+H{b>6-4Cf?nXhSqym7h@7uxy6;D}5PZuQpmu>?- z0&?sTkG}=tgX2Pv#Fd|oyxu}b*A2EGGzyHuYm~9=OU#UNd}EqL1|_tRQ1!dCH84%5Mdv8 z*LdxfCQ*ZiYNbvzcS-Ay#4R8sK5@cB8DrPnK33u(b}L*69!VavvYwrfy+zgr7Ki$R zSKFx|_ul7`7w=b@&6gWormkNcGG{;VZ=y%lPkh-uq~;KIhu@d;qS?>-iK1ED^8Qj}0keca z0i$-P;jnxAxbMB#?f!WLBSb{-GEvA;h82)uQt|p}RA%MD+>sd|zjaaNSwSIAz}5|} zYv=Rta3sU`;O4_FOn{%Xfd=ZRnJsWnuBT_obvp~5W;OnW?4NrKvq=J0$CE=yD)5Ihlo4iZ&g`-TDj0Sd!l zd_Vh{c@H>rIvZRTun?4pwuot(@Az>K60Ujcb;ZtY*)d9p zIl=-jGsr8e@&uTNV*N=BuD-1a%CG}FuUScmn}V8ufbw3~x-;ctsxNHt_4ny^IVD(x zxweo^NlDFSE#@7kf$ntSpRD6_U}aRig|NRhL#eF-!dQ=T7{l(2;gxYVD4MFO90y#I z8Hxb-?g$b9zS47mXG>>0wv{QKdC9S@z$T;ck|kU>YVM!-ucb|BW>=~V`Vjr+^3eS4 ztV5&Xkr(^SCLbAHBC0zGUSs~G+ncw{cwlaYF`ritPkqJayF6*P!tt<`U6D$X!^d$i zQLB;}kEW5_@6wQuE?cnQ5BGr55oU4V;Y$nbc($R6-v}0IJFd6-*WgQ-o8w2IPQ~)y zA+o;^0LB2y?;SjhFzr(EPtG>;7PYRL?)sI!`OSWOuh05E=_o;%4ldcaGBhF4L-(1t zEkhNJ{l#bBi5?4UNHd!lOSkAn@IDq8oS~&PDC!hVZY8? zi>CK>R@3FZ6F9ULN|>Mt&e#RJAzRY|T*=<`cnxtA;ZVLH!JKVgvo{$Ho-pqe!QkZ$8qtoOht9kU> zkw&?c=bsxCBxI{upT2)=p^z6EZ z!hCpR=Ftv>+Yu;)H>!6ITj6Xwe|K{)S^6Jvpy>fEbq4X^@4{kM4m&ck&;WBz7DaqK zs?2z-wWuA!)|ZAH9yu-Lkh?fxjVc!B7Z0z^nsuqhjv-aYi%C8WfocR!vr&%ak?ri_2|^}njqnx4CI+@&z{cvoj^ zgrJY&NkI-^!!c&PX<=iiG}|6KFZqd9M7}HlaWZ|*#1m(Kmj0SgR4p)yfVBY?5wt>s zSioMKPGMc6SO<@5mo@?Qt;#Xz1Z@&6u0D!F3Dxtge49txHs1H^A$y**o(Z__d5o}d zn>+o-ZABZp!xd()LZxB>Gb_Olh1O)^ZvPe|?B@teElZeR_ZPs6`~@!uO;K)FI1G3n zZP3vQzOn7MtbGoi_Qo~TQyR8zP={Ra7RGxL6AaRHHr+ZDyEuaO!U+3o=s%4Bz0QC+ zcjQVGNN1ykDU2G7s4XTiDygh!OM#-`9E`X58Tq1|;{p=LYa#vn$9RHA=5#_>QbSx% z-Ll;0j?QC?nvdR|*L*ga6#xAh?quYRnj_eVBW@bFKGAWs$5CH@YmjN%hB(L-;<3VlvjDdjuGM2e)L(9^1aXHMUP*lJnnRu$Do!wg+W3|^1L>yWNI2M^D|@5K|c*#pO@wYbJ?1xige*i|%E!Gf#! zkESa__$?mivxF>`-zYELUT!}!-%$K}+kkD3pN&60BtSOuG~;CJc^+)kiL74Y;g4y+ z;VU%s-P1^MO<2aIV_Z4pz_7*JDM2M*%Sp2Xa%<-m0#0s9aer!M0WvAU2vZX!cZhbO zAR&5_WDJYk zz&aJ4>&vQ3XSe^H51Jd`yCbAxt)r>RNof9Vr^>RQry*XxQTy4G@P=GI?r`?W9T_Lw z{~A=EW`Evu{9zlCUu*nMr#G9?O!GdYdabqMPzLo>^7Ttm+-|qN3cyLxNRY06(9*MK z0e9xmf?nnw9-;x?LrSU$*+UKlW`p@mHwB9(XcPdeV*aeYOhMDUVh0H?M|X7!VZ_is z<3|T&+r>(#uodb01!C0MAZ(!up>-dBgjuj@bqKp4j-2mBfUe}6NUo$<#zkjUu4uB>GiA72u&Z zzSnB`^kJWV|23C(MLXiTex!wA^k*Iz^8X461vAe{0-TuL&L%5JGWX+e@+EkMB%Mm1 z-U=e@75N9&kba8ZNRpEgb9?>yWxpatEGJ{1PN1Ont~+tAU;M_;KL2s#x&QhaIv_@y z!=}yeLHr(h(ov7QbhvoSI1e{w;h1kp*|@my%8u#8d&pTa5rVS0g?jb3ye5L-P%yAD zVO$ib6tq2)9r#JOSQjaxo4F@GF&u5oDzAoBW3eP$c>p6Jm+rmkSwkBK9{< zht^yb-8VPEoP|S|fZOxz%;}=!JKT4IDe8N5u>44Ioc#Z?0BEBqZ!_6{N)iuD)g*B7 zxij4kiLVB8w*rX`lSol#iMAjrZj>R8jNPQb_9< zu#0M+&HSWLL&I>ad+Z#H?@mq>5A50Ap6{>U?!PQ077w)L zM>#Poqc%XHew}?SyO;Z`6A2Y;_xj~C%b*dA(FB|kw9nIx0Xt0=^xpYc7kOyJpS>mZ zZ7c|!-E;3vZUn8FzBd5{jf~lS@nbm-sACyqS$gXWa0hMjk_uGZK|>fiyM{Roy7@>u zhG|xq1&8(Tp&LiBy$eVn!X}Ly0J%lsM(_7ArhfZBR3}5JnV=cDc#Iss_UqtzbH=L9 z3`Ex19%(uyo`51CpS*H<_jKYjksaUCahgqj^6?Nn1Xn4+iTj<*3=}{O^MzZ*TEyZ}A!CGvvzKyQ5SbSOvsq z56%>K>$8n*y>=h#=hf#Usz-|z`S#{@-)BJx>v^(fo11AmhS^6}vfm(AJNGUO&xvaX zvk3kvosrVmer9I%>ycB|+LdE9hn0QH53_dq@ZtQbf^+pGR2!--ZZN&ZM`boxE;0`e zi4LmJ1KH=Cg`p$(+mLG`YaBNLTgX+$2}k>{yAkw!hPA{VXPeT4VKD@)L-lYM0$xxh z6v2x{-AhUoU)mWI1eqR|Hm~rQa*K8h&z~91(%fTaLpeuPHbeY)glWE;PG?Q{36@)s zjD^NcsG9OG@@)Zix8*w(m){EC?pG+QXYO2f*omoV zH^YDxwU835Tcf%j2fY4Z_xxO}W;_L1rl3fF^69`lTN}16z>a-7U4H(gZ$Tz;D-{>x z9+0cMDw*a#t$yzQzB(%D^w}!6TLwHO?#o7IcCFcE2w39Su89spYag)`n|rY3lH19q zML)=Tb+3cAR4)hzubjKs^v~mLINOlYe73sB8N4r4deF|mmorrke>AukO8P7*W?+fp z311V=cVQ%=7Rqi{KYiKn*C*f6_V;<(t4-e;)9Pj1`X9$EQ~R%|pi+0ono8|Q;`Zw% zw%a<5%lcy>5_VX}Fz$iFv-_e4Oz~X9(z$n?a1O4)rL{!w8=|RnX$3oHu3x!3}?fx5h7Ggb)DnR2c?~>rX}oFnGEW4TAd5BLb;n4a)EzS=pb} zPstOG{Umx$#?PXOPv_yb2592B1|MJ{-%Hc}cRo%3JWomt9amd*&uLaNm4ZkSw=62k zT3*X9ghw8E{Z;HMTLIF35g6zRI&LdqjYKL(S~ zwJd3YVMO}$7A(7@R2iymnWWC_@|txWMo{l1R-BC$RDvfsx$l#-mn`S09&JKzutzqt z1tiXe{y#LGbyyT{yv0Q%q#F@%2?>z~0jU*1y1S&iLrP*rkdTs=Mp8;bQer9T6zN#H zk%rx6cjxZ!KKK5&f6eU7GtbQXp6}eywXc_&6b(W>$v`P<9|?5zSjlW=E} z_Oqe}uXYr6tM*|1PY)J#YF{ZGf)VoWbOdCS!a}W0;1~3xRAI46SuhAR7qANZFHrCm z%RDH7ec1v1Hf@sph(BO?5?t6_@4j0g|5g^Wb&VhoS)Zn2Z~zcVU8XRL+0PtZ{y{h( z^ZiG6;XK#FkeZbE-JSOWd$gvkCugLxhHLo=QlpD4PqN6EH8fh8j|%KW1y0^0T90e} zP|mRXa+@$Oe_SkmkE7HuoNG4uZ%>(b;1wSlyhR?U&0xD?e24tm+Nzy@yRw(=I8^07 z@x|yQ1dd8d8wE~sE?O*i28>k**wyR-^p1g4G_xtz=%AElGzu$_!czEVTNC3=%bCQb zL1YTLGVxdM#ird=6fggGMt=`Jy9Is*`~>szrv7+abqRiUcaGz*L?8AP6AvB+6Vt~( zI(iEcJ*;NOJjX2Ig<*v2&J70 zE%f+*fkHQ3vEusCgsX;icaV5H+ewgEFZk}W96FWeXc6MxkNLO)RmXCOBVDZ|Q@iym zmt0$X!^ENOJhxz?H~+xtk7uyMz{y5WZzIk=^S@R)d-!5l^ZvT9v}$$^9I|4&ljGF@ zyXzrWML>Ycv_0u|-l1s3PCvC;&~3X~&~zMjK0s=Pc=P)A&R5$?-Yu*NL#u6lV?>Ex zk7+Oi1SOwKxpAY|hHqT90ZSHlTD)|_5Ed)UWUX;6b89$-Hx{=WL;Moh2rtYEM#VO(mq5VTtJ;?b9Z&{r7VCAl7XC`eC(cpBrmj zf*kT7ygNaPBwR5k%Y^?zz7$yKQy`;ApdY36ItIKOE(#w-Z@DybIIzfC*pC5|y2?|8d`0QIpXNKOcm7nJN2mV4a?jfbzQ8Qhvf*n9| z3yrmB@$&b0NWI)JTp%*b1Eq2|^BZ&ymQ5VH16da2+-cC}bOet1+!ly*VumQnI)Tpp zkHofs?|L{vhE~{N>Q4Txw8EqP=X28MTK^8t6pM9~%qo4>fCDCCl3nQs$}&qY_6Q?Os4IlS-J%7Xb zsmq#snlrcGcec&(@V(etDk^CFVX@rQrE}MqdV@FD4<1soxRU1#dW&M3KBdmF*`qPv8Q@X~p5t6A_;NV#@3mit7oRG1dAlLB$BqvBp%NFA} zB73hJoX5DuKEbW;rr+q3reivTQ8PVgC&AlZ#x{TMd*kceIHYEj~^l<2*joA!6RWd;lw6*_kXaD zUXu%s9Uwi)&yk32x53;!6aXBD5_FrtnZ0RK`Oexng;)BCO*cJ}LIH>P`}e{RL@`!! z&kVIojGd6yI4EAgu`dee0iHtTN0y>j+L2Rnq6adK$AI+SSe8_~=W;0QL@Ko-0GS)+ za5K^A`x8SzE3T-26t^3w^GT>?<3%k*vK`}$4^<@laj3wZ{Nz^3L(ZSTU ze5(PNJ|@3;n=R4m;lZ{Hv4Zq6@i|KZ%!xcG2?VGham#Awk@x=eHi*&S*5IA7K2LP> zu_}^wP-hp4f8dEkEwhANbnT!9Tkc`sRc{hyO}AHkBMQ}4onN9JcM5-$x(D9Z99*w5 zXzP}G1&Q)x(*A@!_azju7C2fmphen?Qqt4*EwsOqLKoIPtrxkOOb8}V<Wd{RB{Pgg+;< z1<;!-+by4zbeo0$_M88_;@>``2HzexT^IxJ(5wy#U;@L3%E27Jw}%g-4Y;yb)c>L60OF=qI>o6TSr{}4Z_R+-f^VOl zvH^`|V@)BL`r_+Y;yj0Ov-vUV0;;?v5~;^>NT1StGb{Vqyea$$62!l7VIk=rK_Zl3 z(Hnyy`W}#5_3WCvwFPf0}H(! z!Mz1@U-AeXMY@5b1DndPNBZ8yJWN{3;st}R^-cVmW=Z7C&AtJ$tuwXJJW>#8vVt?-B`;6EVz~*i#tp5bd zM;sNzq~6R@ZkCqAPmLXsjHZACqKZrSWcX&!u*7@$xFAHUe_gqev`cD@7vdN?x_!!f zgMg}@SuoT&eL@ASIXfaI0?r0Ny$6nCDYeX(v_Ze0!wp$pGp;ihzG~`KqXG8?+&HZe zA*4QN9#~=@yCKwkspQw>6R_mkF|Z=zWL^ufFMogY_!s(ia&#e967jSGwGWiEBIuGG z3;g=u?Do}!qQWu~#OV+tnWYz%>4Tv_r)qkoF#eBVv{ss(8MY1zNJV?zKzro=!eif~ zJ-w{xntlWLv3`qaNBdW1`&y?^BX3Z=vm4N|jDme6=BKzKq}-8G=Z>q%cabV7rDxmC z@H;_6$lABBI9^W{!dY#Y<*Tlt(x#x-ap7qtaFznYL4WIwW0?x7_eq^@6naYoj42H3 z%G#{1srMgog|OwlOQ_2lI?H<)BLe)yTz8_-mne8>7hjwdIHb$vnLe35;QOex>|Sw4wT|(e{i> z%~afrE9)E%pf&{Y+y^zElv$o1!t;@hL~|g~BaM0l1!smWIF`HCvIrHo&75&141fvHMZuEB!lsr)KT>@sJ6Q*NyO4*x z-dM)RAD^uZJRUX zv`C!-hZ@L>e_cT_9)AHSAa4f?Rrmp26Wfp2T~xz+1%J~);d9B9{1Pt3e49)rrM~T3 z(1#CQ)p|^bF7BLGplm?Dq^5BE?_tS@oG<>n=ugZy1<=d_=(k>!7RX==yVQizg=7JA zR%UVgr9Hlybo>DTzZ;Ynj>%vv;K$chnkIaPa z+^QnJvHe{%mVNKEb5C@)R0w{a_K^fE*vMI#4CXNPV-&iCU5Oll;8W&zdg7{R;vW5) z(d$z)F4dii@{)!3j7`+P5#e|Ba)^g+n^F*E+}=ef^##`t?3#cyn@;F7Ro|7 z?M8k<0-A7Gs^&`*GT_g~h|Ne(Nl%uB$aKE@;}+pmrsb%)edFb>MWoidc#~}1!0~*D zR1YK_I6jIG4?eN)*P)PRJ2T;J{wm9G5CAo^n_pGn?^M{{17{LFiDbw{a8>8OU%@!D z(>RiF?>C|y;a_5jOM9+?^##kR1^z~GXb0B_d7yAj)BthnHGnz%3M9dC_x5%_=ia!s zvv!W~u)!y)It~p~)lN|2%E4I58_|x);R6DgH=7hxc=x= zT!Tl64}k=JX=j+Y9NeRC<@V`#SBp95``Q7!$8^^VoS%$Y7C7X*eBS5bzhZGxws&06 z%ROuE$89li09k}R>8fFP+&g&*DcA>psvbD@4!8pqsfGN+_!$E%kw0#MBryNS)ctK7 z>6It1?N|Luja~Vk!P{~MLDX%R{K<>+ancupbK3CL5Q%@zW5FssmXf-w&N8>Kb{n)f zC6KhoWRP)4(16x6xIn>dp%Bg@pz@aI-J{oY01tXlVf1d&0B7+>79@!0AnY!-y7?C& zICGOVU-eKx^xJ)|Vgm=Xq9sz~ge>7G1Ke>R-})!OyCK>XXugHj;GPvH~zz!Ntgz6 zzgbuyiGa_)xAI%}lo*>AFmCKH98PFf<9GAvdq2-yN0zDPw+ZM{yCkgA@oVQ7gjN0o zj@}1RJG73EcXsPW0zJU-Pf0FF8>~a>y!*kux>_r|McMS-_9EClN14 zK>LYw_UK|-R*8s_r(7$d#{LVv-G58oIF)s!JAzSjLr?$6m<>pSvI-BLU~~ z#9DKd3~t(VD}^iW;uYpTu{2QlC--fo2h+pevp9kER)JDC2A@{t!Be^B~Y=6DW z9}fR_r1{=}R@SbQo8|37Tyl~zk;XtiPWp!9KcgclY8r!jAvh{*BVj6MMZ<8iRs5qG zBdTJBgVJowV|aC=VK@ir66EI|EO87X&{^V1&FyMv+|zn@U(7eq1uskFlC+2Oi|n!X z@(}c7u3LQ1g5tB|i)Qtd<5dK~b_L*z{G#=pxPP!$JFAfpz4Q-YDkZVBPeXucdeJOb zY~K}?P~h-e$&kF1tIdJkdY!WQS9bIJusDG@x3;VyQXpef3EYco+MbO%pDK2%|1<>k zB&23@%97N0AkOzr9`+xdYe2t@z5z6|xm*;<32Xh^M;j0ffjW%Uu`a zp3w?^5PtyJSEq}?m~M2I5~$9%Qys+Xus${Qh*cDR0m~8&BN9h>egQItyE#jNUXvK% zd&a^pT~%){JskMu!01K-CWEgRuXfS`#>?Gs#OMi8>T@un&}J^R36xnCs_xl&PlM+% zV6-!3zV_hR()?$pfLxkocSpq42muWGY34**Kuem{XY5?}_&m&wY!M47^#D3a>f}ge zjG}7#FRs7okF{s{ohe5M&Rh(!7o`_R*GpS5PMei}Fffg*Wa`rl#$ zbYenf0omKJpMjubnTuXxI5VDuThor=#T^W-DSob01VN_E?4T|uGd0AQ^>UH(b0Y}jmcGwk3a#Cz?7SwJ|h3GizTgW@mb@v&JH zVxV5N1{MmdJK!H*9`0m{=LU+BiF0Qyi!ByPUtQ{2v0Hky^X;>f{2i`nGT^&aWgPjC zW}koYHNs_S@IvC1t-I8=XW)IO==DKA)en&0v_?Z6ne`PE+NSPUl>d8iV#-_p#W!^3 zRz22d3i9@Ly^CM%dOy+!jp=Ggtv-pq+1ddQ9wQI|L>W(?*Kq${y^YEB%NWAIxBG=! zDZRm(KMIRNHa`q;m{15?FaDfNvrirtvhGlfYbbevGJ4AP21w}uZb&- zOPhj5Y(GvlUaoG%{DyBR*$Pk#yus3wZuk5O7Xr!8;?1|mGvSQ~lUboo= z<{LRgE$e~ph&}Sir|g4v>hYw|nX4p%F!~zEY1p8O>F_g}IcKkoN zq6<53PtyXrF7v(x_a*pEPB_mR2R!~#ooTDvL&?=Bj}=GF zPo)!k4=ItcwNyp)eX^Xkz1e|!uAf@!zj*zUA@h@Ahipn|BAaq%PQ1&&UWLFlcoejI z`ulJ&qsj|zZIbUf=@jZFYX}J6yqp|tL`&8!M z_Z_-$Nr5c8DdB<73b85(x`NE)XLRs)T?7J*q_WFj3KiFttb~NZ;EAqLwI7vB#2ZlT z%eHKnFuzV6GdE*4`Q(Ga-vS&>4FoA%B&X_i#A^z`0x;oqx| zaGZheEg{ONa?7eHI851J{oOaXOZWS?xgHUW3+W=~f2H>J+LODEg)2AiEf);8%wYD+V2H%y z`aObo?3Q>`{C`E~`;Gtc41d(c4i7z37ktBJsO2p;+g_&OWt<@){0Hs_3W07zZNf&6v(;JcGXep z5PiI}V%9s`V1ro&OWQxxGJ=4AFGd}awzb+11esf=#Xb7nz=3h>xZ1j+#s3{!kd`x$ zKrr7VdSE{V74M=ffA!CnoZsA+w4F-^IgfEiC!b8d_~tqfECSy(E^Mm3Vokh_5CtD9 z;;J3@pb`h~z5&4x1yVP9oJdD&tt8JkFLe>D5OVhQOMBWi#@{lBL|PoB;)4|>kBbfi zZQKB*d-kr0n=cs~EU!-+&dg%;yN=$TreKEz%^r7VkO)%y-Lb6whiDuqHv0wa^)=FO zqOTqSGN4=?|ItyK=w#3BlVe*^zKW?!PCC%M`@|}G;V}2R${jv6Vf8mA_tPAY{`_`o z#e+V$GJ6kK3T-ZQfYthj7ch#@$^~D&@ned=RIvDCsp7E&ut?vl>K<{k|dXdxPDtCwYOam726?p zC~jLYM*|U`xdU+%c-_^VKR&SXR+h=IoTHbRa-=3%8G`Hx>=m2xruR=@ zH8xPCaY$F~+3C(dsd`zt(Px)x=eW5RLJVqjy4NW42f5=5M?P2)7l<>qsB&6v6Q$tjmKv~nM z;8Cr%Eqik`K*{)KM9`HlkM9>>|}Km2%>hgeF`d#d6z(^SO)xB{ONbnLr2W3 zEGu?%=Zm~Lm?KzeKbmM4mAMiY4!Qg^J`HYv{$%P|vBJ8Jy=tFr`!_Ah*0{vSRFbWq z_xG(u8*xIP#R`>w!jX+`R3dBqDOYE~-scF4us?sWek90|!63_!g3#i(L$lNKetp|I z{cSAuamxT^_h2z5F9OVufFMS7`$>-p$7~1S%9M{LezdWgDdBMb;1CU2=8INnzaw2>SxAveHImZ@3v=w7r7Ta|K zF~=AD$++ZHZHGJ+Zc^0goJk|ly%_0>m~|o5iaJ@Uyr*XS5K~pK6K~r~eZY7=6;qp* z|GB<|gjVhcU4kaqEFcA72-}6+9)da|0Ea_bpyjYq^jUHoKUEM}&KIo0mTsFUlsVDz z*^k-IB&1t2$1tDY)BI~~W0k?-i+4`r$e%M8r}zK_i=&TRPHt&X*5#1f3e@*g`Sep- zrZ57b#nN#ayXE^GjFWWpP`&UahObZ!^6l1A|Z`{W`ti57v( zCG&|CLx-SQsKLob^vD|t(@T>Ot<(aLnUiK_rPlX)--ha{%f6mKTjgzc}=QjbKGpCoe z{&hqRVBZN&BFPVPHe0H7RC`Y^;=JB}9e+O5j2EBOd6k8Puf z2 znL)`_Jm$dI8I` zhPLw%X?U5nVj!F{>(>>KUfUSvhU%8jn1Dj=gH#d0ayO7tcWy0nKovizQWkCN3KG66 z=$3L+o`d?=LgO#rjse9<{8}8HNBpyyHeCpXuE|WySqj6v*n6IXVF!qrJM%jt9Sh)* zz<@E}-Y`7iHLSW zOP4bJVE!j=i1me^KX~O-88bNy7PYI*uC1ruNxQ~@%$O9DB!#)ZRXF^);Z;7Op=Rut zeBW8D#}&fdf`jdX#Z~|lNab$4I37Y788E0}HCs7P%`M16i4KhT0=NtHY zc4=usdS7ZC$m^5=18+l{h9by$q?DR~ObZ%BFTFw+ayC30c(%7M@bS;iwU%?JOfB%# zp6Tc`1vl>;%+Tdq*oP|r_s$CAeaP4heKDW5xvpgrFxHY@e&#RKN4Bhn%~0e^aNPmI zo?a~2x-h;8RD(VL31GQQk0jJg)AWthF$<%6qt2@$nKQl5Ts(k^d|1^-{79X7(8K%G zp9&`SBn@RSkc1cY){wf5VZXzf($%5-M#f?}D{;d^Gba8H7Tngjm^)&RHvvnSHAA$JPN0!=ogDf-; zUiGu86RNnmg^`M{U}wg+eHRB>p-b&W&a64^iP4p|-^s&9fLILX>+7t{@Z;}K8gfY< zHdod=D{(i(Xh|;T*!-d^5;H0K#U;M?s^g~OaRa{W(hK~8w1r0+;YszvA@tfZ*eKyi z(3thu`j3Ngu0uEC zHz*Myps(%juO5WIc6Gn$J?f|_5NlgGX-*~nPlFubqBJFi`H9c&4D1$gPD1mtft#h} zkvmZPil0JSEqaPTH~*hpF2E=#n$WsgfhD%+I`y zbwl~VG1aY`AMT(NwGhkOrG%Y$k%>sQ7~VEG)#G;U_ZmQqfWKE-)Za|eW{gchPTH~} zSo77?^<>*#?AgNhxVNthkaYNv-kh|O9e!D=%NwtSm1Hnh;@q`+7-ILtI2%&*xUv}Umj~Oa=OPaqx3DYK3q8bvRv4#ibz4+*(_A*(=g3F6R#@vv*gh*_9_sOB z*`t&1Y`Bc_A`BaQoyl=-@{%~=68ugC6Z!>LIAD!pcU>3NXR1B%AjzWzc0qQH8l6!_cp zA5i1TUqR@Lk3$!7`%Avr@pjpseXI6kpK5!#$^+5yHlv?@rb zN_Uj_g_<+$}^t3%?BlMWU5w>%%AM^V^Bd_8OgR^}3(5Xr-lx*Rmjxm=~5 zaZaW`;HHV~XyfOzN=Otob0-Y#O4FZ>(*j^0byy@db=TRazQ1pBKg|?(o%cU%*tZd* z&q>;+L~o(U8SDLWAFE9XufxVMo^zgOQ>Q;%sDPG8p>)vh5ZW#P5c+mZU!=q}s7#Zr_*-Do=Qe2Sz2vl-|`pau)x)8E5 z+(Qnl@CJ%My0}>z&$WaB4S99B%6Boe@{4^@ygQQ(=gTmw1F-vepy5U1=t+`z)j7Tf z9_%{9y(Rgz_~pDXB$zk9<8CwaN{_8lthVpgtPAMmS@rf~>mx>tK!tN=U&b>}HCFWR9#A;HsMKf*N6;+6MmjC@H z=TeH%5S{t;UpFg()M$^i^G203cfsOL>Z>Fi%UQAz@t_62FZ<#rzIVe{sUQ=~cCXa} z_?i8q2~5*RE?OPd|8CXRuN1?d6O!-j(8C(aMS=iI*Mypjb}>ozxn3h_Ks(Q;_bO@x zT3A}5^ox`npvdUXT{USe@6owKCmoFZ(m`GzCxNYmun$Mj3$nI#i{7ZyAA(*o%=s`+ zLkuw93$W{-J&tF7;doZ!u1=F7XM7FWjIMjM~&!4OR$eRL3tVo1(foj@uM@~S#MX{b*hG4Q6WyW3*x*8X?$0k7OJCiMAzjxsY!Yjk za9ZmLHnDRK+;IvK(e*ff&1hEchO0kcpTn&}fKkX3xIA5{@YB@^qD;pdK+dxJJ9eU~ z{grctWkzhFptn&?=@{IX#PpRA2=WP>%Am+8c527Kf--5KNZP-Ts&rBRUch_g+ah#7 z$JPY%_Xoz+f9+p*Y%ph`wteC$?t131W@$S~atfd}nwn?Gm(WfJ=k1((RL?ep=7*-z}Y#N1Ax{YRuP$0F6eSRKOB9kOud;X%0 zW_E$ygZY=~JLIwcu3yNN-eaP`mE>+u0m&y8vId&LQM3h})CCJMYAyV~pb6+@$!XNGMAVtG?2rLT{w z<3qN!-H;$idE2xl3XA7dp4lQMnr>NIS7C#LJ-!Rpn5Z*7 z{$kw_|GJI7#T$+8x>X4WlSdhnhSXjKE+CT9(f17pibw;DTgs0Eh*?Y)(zudf2ZxB_5huKU3dI#^w0{6uq<-pGsv4VjU3KeV`q*aYEAvM;DS)QK2mZ0 zu)5}%+k!IgE%lK1KsgQDj@_7nHQvVGk)cIT8)nQ%v|^gJp|41<<%c2~iX7+m zsor}g;x#|xXp^)VmNW=u2b8CYzOdml$eJW0YKe5JnB>$NFgo)0eIH}XoV@JrigE4t zS3h*bTyUnP8c!-S4S&Af^IQ$S?; zO#JZAW_Q$3uEQT-9Zf#8Cwti@*gUS`FPd|($Tq_Qntw6)f$*MO0*#I|A=o*kx(j)@ zc3-^StQ1I0@MgVVtSYD&3U3ePD103k|4X z3a+L}lHt6_Xjzm`sIt)$=6+WLj9vtgz4J%kQ9bGS3L>9o_8BVQhHoX#1RY{0bn~A0 z9@sUx4oBZy%nqyi@9b3OYg+MI&aIjlttWwnygjyvEsdvQ;-5U?P(joQA1ZU57&oYW zRJnB+;xMcA6}~I=Y)QQy@G?#izG3_H#CZxD{L;BWpCd0iCu6`Zjc4X$i5n=6>|J2} z98VrQ20X^-(T!9|{sY3JD~w%C+z{HPW?1hd(-6`Kia8f$In2S+`(6!MhY@l*{ASn3 z=l53yHmPv;4G_RFqyaEUAHSUJ{sCG4dyVlepNHtt8wyku1laIs%Ia&|tq0mNqaTHp zm&l=o{T%H>IHq`B{3*oM*AsHWt0yWn4on@Nas311! z$>+gEvZp(lJK)<@u1OhjBa3>JPXnRYg{yqaR8sLZF3(86sVHCVh&+zoTuk+#pgTgF zU{K}tOE=_|-XALxc;s7g)VqZ*`P?C7sT&b-*F)e-2q8fOrVaY20G;FpVSoRPhv6Lu z0B#f)yDZ<5f=Rqawe8WsZ1Uz~a)s3FNTqIgIf^{d;rj|%9tK#6iKVZ()%fgTTKQ7* zbD-Lw1e%{#TpNMiY4d%h%Oij_g%e%~?o1!1_Oq6vQfsY+!B$q+HAK~_tiWyPf{R+8 zHOl<)ECi!?IYk7dlAKy8#vgrzF4*WYg~SYHlF_#va3+(AQS@|Fk-J%MvtIe~)d@U_ z%AKL?w9i$Dm3u?C+s8GTR=A=KEufSU7X zT86ciolqT)aH{0hhxU<*^%HYQ8Kno8xuf=w&R?g*;OsH)6yPCe8!Rn3FH2X?!u6=} z*kr|UdU-BOgc%0C!<;M^NW+5rtzHp~06g^*A8<-wD)SOET4E$=7)Jyb)Au>Zukm$B z+Z;-?r<;(^M_YzS2N>{}!NF8y|4it}9#i$&4qeQ7Rxa=Ny%{)^9JRlTrg!c`&s%+$ zSC|sUR284rp$=r0e!6dq=!0_oI@z2Fl)g+N7t}<+LPp0W@9&7xYDrkc*d~cB?KjD~ zRNH=JQ4Ml22x8zu`P?fRz{c+&=bpheUv-{YbEVIOxf2`QaL9LhssjGYi^~pf7n(m) z>v-)1lAD~lBn(F)lEOQx=m`XNe&Y|ZKK4U@WA-|i`${6%OGs8P=^+%dDL5&sszW;O z%NM03nD>7)uo03?JUGd3m@?!;JAbo3jJAou{Z}uL7m)A1NiDaD{;S7^EpJHGqm?2} zzn;&u8Dy?HwBekZ2+^9VjZr-2OP+veK8Q)W>I2HDY7SR9kp&?a!w=l6z zMm>+Yo}*v%CU`DyV6*@}2a%(X#myh7VOE+rHX>A^M+#*S!T+Yz7PfYG^LuTF=g>aH zlZ=ENh%lhE18rzxKt(4BY|5L3N&wG`quQ{vtop&xwPi^IXYvk+LAU`m{XjOV`cJ7N zmJB4L?Y`E(7h)T#tcYep5VVc}&;U z;z|`lmRUsOiyaYwnYEgHT5sfD*7R1%B$A^34L?Fo-pq!|lTCLYK`|8U;HZaryTEVy z^}Dr#q=x7AU%3-Fy@^HoFLngxDL#H47fU7*yIAah4IKufzu^(7DdOSP@_&BW==uo* zM!}-f#}#bvm|@j9@Mo$uQDAH2TOG+7;JO#O|N2grCQJJ1hez?$^CA|&PAyAm(rr*I zA(TAbV|$*sfJL{gyw?yfj*7hQHLgO<6qJS*S5P`+=OLx?IbYki5VdEFe|RUsc1f7) zBGj{6P3%Apwu%zq)R9xYow~k(cHiIajBz4)H_-=&yfBADo^VJ+)=jldqM=I#M6I8# zlYLK+==!TaYNJ#VP5VM&RICprUp#XW)4KG}vQ{mROPd7Cwy|4Sf1<72sUZ9Di+fj4 zK@9~0MP5xkgxgV>^lOu&5D}}!Nf(E1__cfoM;zOKJt%&@E`H#YQcWDT)D@ZN{l7O= z!yJ=`%?-Y8Gn;<~&npA>f!xrO7a$$XmBC}|!y?a=bK68zdn55T|TzxNDnZu|*{fdkpW(iOYPLB|rMg!lrUmxX$I zv#&&C9dfBFMIUI~|K@h=ZktLYJLB!}Y{osp5_i>m_-r^<4ErgAs^0y?WGQ_^SBU3r zKZ!yJw`!d)RSsv5A6|pqOvxC1ZI^)FxbMXQqN758zgaW&vzqi+MOi*t%c<+>)f*%L zbrQ~OH0NP{ZERsrTq*R$gY8&olp;4uRGbqyiqi7hLA9KhF^*?4{<74)@F|wo<4u2KU<{>*u(eu#u&H(Yn19z$6nMqk{zXV}zM60V|y>smB{$|aX5l^b!#rErr z?5;Gv=8n&}tZ+f+9|lL@)s>T{AymtRsDN&d{PhaL<46kR5h@xcF(RdJun6r!>)k(Z z6Yk}!+WeBO?8CPODZYx%N#F>}@X1aKqcNDC$^^lb#d{YL~3ZoB6jl^m`lY zzCY1;ieEPmU)&YcmE{bf)CTX_kKTZI$KDnV%AMd3V?1(wi=HjfSwUsLcK7 zRbLAlXrAF<6b38paovSEflwa5GY_I4{?MCf{{{uCat1H&-;VGwFAfR{W-v}*&K1K5 z^JAVfBo1{GmDIFwj)%ESU*0Ixh^28vab0A`*uft&)Hhk-iijkBYWTIz_`}*o^~)nr z2xIra*jZ0>_vhptPf}rEI9fCfp?C#TBvhY7T9!io)>~^GeW=}U#dO|wB3BuGeP-Vm z$|kVQGS1=5v)H(rAi?M2v&dN>d_C|gD@S<~NC2J804K24&XA3%z9ILQN5`;{`2qCWmXY=Ihl>yG&xVYvCbP@USveY)wrMj_E5Pg@SlYPY=4QYUvdS~NVRXrO1GkOz*M8( ze6e~0QOj{-^od%8g#Eyh`Wn{bbSzy#9TrkpdMvIFysX1>#mg2=9y55F`YjdZNgo2w z`JCS+Sa_vZPGExhk6fa>D!66er;V38{K6|^hn~ikAaJ_~PHJ{UBOw9*I zoKo1ZQD}3#v&sz+go%*to_}Mx!6{651lVhaQq?{=buD>UxU$*~scn@rEVz@m_o1=}6w^QXqm!uT)7AY_w2 zNVfh$pMADJu4X?7+!-?5^@+dta7L1yHs(VlGw7yry@LKp(&T$1ff^-WM)H>@hw4@W zpm6W3AH=flgRRg_#muslgT?wRP6ltE`9WLN9TgCzT1 z7KT!3#fs4kllcBKRL4A&SrvQ~4Yxd@c@%lIa5ilySANzB9mJt(FioM+qUm~KM&a@O z({wry2mAS!sx$HWkz4xMD`mp}&jOf&ZYVO29J-Eu!M30oJbAxwy=a?sJ9H_KhLuw^r|}FwrsnUR-rg=wkWACZCYNW*AA}6h z$+1CCJD_AtJp+++gE!fiap~_N(MR4(u6gg>P8B z?&<0KJ?}YacN5*dCBjMj85oO`z8wZYs8`!{@};eZJbFc@=sPGVHsqf#8yFq?JZm64 z)xiG8Z<#8HXuT=2c=5Nw@_vD+a|-He5S)(K`^oM*@pA7LeuZ4N|K~fIjww>T&zb&X z0|Fk0g5h05-=aT1kYXqTz(si>Py8=_?n77q$^28jE605Js!UGx16*mUb7K2C&Ng_w zp8N=VnOLIqRs=l|r4B}X*#TX#d>vys0z5@HZ?Yb&KGK$Q&Lmky%YSUnx+oSghz-SR|uVSC5ff5FJB^Rm?f{Zs24FEq&o zL`{T64*0k>mTeU(fBkZ+ zWnZ@uWZIxfnRgx)%z!mC3(wBE+@sVJxLIH+wB(+m^s1omvUU1$lD!c@K1(uz3Xmd< z(jL!Gv8(Z2HXM{0z(w>B z^Wf6%dcOBz6C^8RYat@8pdaXM2)FpoC!Zev>k;t5A5jDp{zpyN9)s9kmnX~EN|ch5 z%LN86cM_IQSv(i~8M}MNzF&gYo(`IAe})uAhScXS4ySS*-M2db-B*v2V@)?~mQ$KH zxvw%q{SCT!Su1>x{`WG+ixJeR|6cvI(y8JBFe~up<}Sw1GW117x8`Pc|3;r?r2ofk zn}Nf+)Ny9@_(4*ba}&;z7GQj);#VY^eiWedYT8yy)mJU`7uv^@QwRdAaXE~5X+}OKlvY~ z&O4~7F6{Oq(gjqIB2|#yK{^uY9i&TF0g$jBArUR7nf7*24qSK|{p8d_JY6P=qU5-E5#y^O3 zDG#tzot*hI`j3T67|@9UZQBXI7hK5*Lk*_R4EY(0JN;z9KcB{%54PpMIpX}2-P}a0 ztmO9)d3)8^MY?dh3f0DG;ZJ{0)40^+(9dHo0YqEJ@Vk(acb;G^PzCcIl%r$ z?l3vLnQG=P-x$K&Z2AB%U{TRYWtZI?;DUC|ln=^4`X4;{Vy< zVlnHO^Lk9p0Bd>&s5#U6<3F{(I}A&=j@O|bf#1H5GAaFf9w|N9D(UGs8v4?Y`DDt7 zn9t{(IdNlScRic%D%5SW><-<&6X;gWl`jh=4o+AHL#{3Plj$=%)1oDUGwmNa;heCp zomj*2od}ZIj~@wB_-!@)QgC@*wHqd{5b4y%5_hZMlK7gyAfr?CdoRU(`T6f#+}zG< zc3>2naQ3g=xV|~{0B%R25+D-~MnOyn{ z^a^cI=G%+DUT}i^qhTi(j)Vsnkr$aL{-8CEOR0}yQTcK_Ps#IGs2$#u9pk#fJMN&@ zM<;~A>OG3C+*l?+I;i(v^m_2mS-bkvpc-cBEb@H`Z=ZBB`FgkUk+X*I%?3!aCNF~~ zeumFl_Y<&*rkkeNNviNaEjL-+DE#0qG1f$(NNY;3+n`xb5h7$g?u4W~1REScM%twQ zB2L>ed^G0#B{W&0b3Le1arU+934+c>R@d6Sn$Xi`p7yZ>4*nWXS_+QQp2Dx z`$+1^?}Lcbir8aUl9Hy=WPoSBnOAJA@LkM5c)|FdIfwipaF-45nLYi6O)3H3i)$ru z-^>)Q0W#nF;?D)Q`Cc{OoOIg}td|*&r7FH2*fYdL0Cp#_ZSL(M4Y(IQ>K}bonh_oQ4m-Io+z1k%VJFIKv=YFMKLV z7T%Ry7N<+eS!@`D9EYrG{dP*JXXlr@&!>__7+R2!AvTs5UN9T_SJA_wtW2EgmcRV* zgUHE$oVOi4cro+uG5(jdg0TcVG)MB;(4T)xD z_kfT<+c3@p(=_N@%GZ`#LJ{BO_4{4b^@>M zbn`D(e|hwXt)eA5z3N6KVZeJVt&l_e6}@9Vckr@IYd2qAK~0lNBP+80{(QofCAF_N z%TXi*@}jLpW}b2*-Avq{gyl|K4U;)}U?H3^k)NjH$J0{h((*g=7RY%F$89b-0*e8f zg+L^%?t)}e0$cX=*950dEha3$gvmF{wG%k52a@1g4%LI;k;)(bf5>C2-%9}O6$uK< z!4kf#a8U)RGt=tQwT@@G!yH!gd`V&JfPd-`Hn84(&1HCW^pe)2Ei zn%mKE;1WWEI2U11))67nQ&m)26`L*G*KuyUtpR;uYql#?PfFidaRucieeBGnEvFCN zcr7ELg+_gBBN=EG{BTvDgmcWWAckz&yg_FAMDwQH`LY%0-zl%KT@?{ZTl9(u*R1~~ z9pRFe$PY}6qo)m^ji;BW%xcEJ4N>f|#qUUpCZN+FHvvwWj28Z8sq~2BH85Fa13ZAH zbr%B>FKs}>gmF1!OhP5*Tn;)7W;jO0mRyvlgki$TWg##??b7fKMW4KZIaiytF^Cag zBFBMuq&S@N9p8Yda#DbvNUlK2Us#_o0LoOlKL3EqC|X!VQ-ENPIoUEg)36OWmpoP zSr_EiM{Dh~BE-!{K4aqsZ2q?+|91(aKHMV%25#vB-kHVW%NkHX=1ohRD0i&f0yL{^3$dzTwm1K|0r` z=v}+uy)1$Y&q#wh@o634O`K3-HIv)DkP$lZ9BRXX33lM$co1F==D2^n=~`fWPa~cT zOGf&lMorfYh%b5lWY6CyMumIpvQkOt{H&tKdsW!?mUVfYka}eGCgMvYW5`ydIOn+? z3O@~x2RB#iLHZ&p7=vk~a6MS~(c4kcTrju_#XF_h(fV}S6W>xmHK&`;`kK*OT_gch zsKTHNPxD~clascS6cX0p7p$05LIk@#R$z)+p{tf;A`(m;yW0_e%oe!!c){yj$h69r zAmJDe#fk!mAF(rMV?n=eWMzb1>QO*Dh{4W07HFOta z2RVb)^Pde7mVmQD0@DS7Jp|tAEWPqB*P0%NwDLfS{*Z@nUBlFwVk?>-92okoG!Tvf z9&o^TCkM9bR-)h%MrY9?q?0_$0Ix-+Rx}qPpk9?w*^SPTTY*)~efOPdfB_Xi2J1Nu zijeicU@EuS$O=L9wuq6UA=?n`;Ga0F7@IQ25+*~H!rG@kII)6npt+BPaJvY-z+hZ8 zUc)RkEM0>)GW*3T#Eh{Aue5BH})w$^$v`j^vrwoxtgW*>{{BHJujM2HcSnw zB|e3}^d-!#pdQ>xEJx&h_k=Rfz;;y(fNowz!upirD#+(>i27ld74G8vlr( z0OH(^f%4@h3B8m1aQsYEuol#Klzh|yXC-)bu1Y-${{H!?O#g7Xc#)57)+8{Y0?gze zwe{a!_I?|URS*-$O2FZ{d81Y-OaXG)37-Z0t=_s#Bpa+-d6&O;PQwO&PFcr0>2n)4 zRep0$K4$X5AJ_c_2tCZxKUH+e-$pw`i9HRNCu%MLE$)9SFqZA47NzZcEHREQ{;bkU z7c!%nue5a5)AU%=;M0Y?W>KB0OfMS*BLS4DSODs^P_3u+4BAY`mN!cdG<^Poe}{Oq zh+u7e$gQV|BQ>`wfUwhz;B5g8D7F&SPWgVbHEFwAwLeR(SgHW!RETTpdeSghk1ruw z+GNH4+KKsG>-6mHhewOs|1H73qW~hZH&@muH<-MWqT#m*m7;3wJD#;8Fs{fKzq~*F znRoFSOK4#wdU$h7>AbwYu?St=MJQM7l2ic%VAVpi$;H>Tw)31`tO(!uB{M4iSRZAO z!A;o~5Ok;lg|BI@Bj`C-Inj{JiOBUo^v8<2KfK{b)l-MCrla3Q%;B4hczcDKQcHTR zMD?H@++V@KfAE+Mc%P3q|7zfjl>VK~+b8|O^_0}kn5}mYu#bDiOO1i`*QPVkVaC;I$WJ zFE|;AlS3w~V}++Y14Wn}4!!t13O*1VF z+evy7W+n`^j&*OmZ{hgxesG4lzMRF@*+5ap&=;8vEL`MI`^M@|Yz+-wE5K&x%mU0= zx(ab+Wy-GAMH25pO~C`T=8ttmI^yeu=UXTTT;(xrQV0T4<@y*~rCL&Ls@Sxj&$9}g z%x7q&-sm_yO^Q(Fy=FCY(vKIaROqPv=zz2ie6L9%)oe?cYl)1E>U}Np^bSjiu}FmV zhKx8yeSaL@?Iqf{<-P-+h0ZRpt$bdPxumkB$0*Moz|5iiURMiJWNhHvLFvv!X=k%! z^29%Ute;HyudyVsV}kqj1w?eTsL@g_h8R`Rjf+Qn_52P`VT2a~Eb2+d#c2e<+fk5> z2tYRlJ3`G}jBq(>mzwJk�=O>rI2^wLpnP75~CiUN&; z9mE54J4t|%A09b@Hq>zNGiJjma5@;q{%6K~Q*%Bc>KY{TsL=}`Tr`^v@Oe@H2u-31 z@LoMecyeN^Egc^Dx&TUX*8Kj_HXk-mK^Qi1z_THp z7Tn%7+!PcxxW#A092KNTdT_3_+^Ni#=)T~rPJV<#nmt_(?i&H<+KO^zRV1 zd9Gz)r%M73$n$8`aEILFCmDY_?J(Gqd^rwAOI%q9xr*!K&aVx2L%=@D0G`Hz3AybD zATkBcwl{Yj2|`NlzOF!lPbj>%+2(AE-1%~RWr@)WTbFET`e%U( zuzM}2`Cbqw+2TI=uMatEgnRFCo+hYs{}BtM?}Ek^P0OX!k=h%H2QXqGhB7JQkJLVH$}8f995a$|0SEU|H56;uYUO9=EKle0R0p8 z9Q<8u%5yk%n~k}Qah|;z*atR!9jcK)!uyNs>4+cxI~FNxWoX}~uWARv3zgd+knt$bx$F22S&=437XACq?y z-pKbKko=F+aBbDbpQqr=M}b)=Ln_ZBO8UvkjBfKqBTX)DXXl5(`KsYGaa$Ji zV@t24gZDl(*%W^7n z=k2bipqOf<5T;#+kTg2xT+Y!foQ>JaCBh&b$3Pucw2#U<4P8YELp zP$-|b_cxm|0CH0?s6rjre%fKAo8?9xTi$v@sK}pAsAzWBue>dTN|JS-xB#sTLJ#r& zTcJ8AW^VwL*nXud_ecC$HKMj1z_KOp?*2`HKC|r|&vEYkfMDCy4+laz86w4s8_N?P zX}9xp@vMz<9KX2A%PcH{)=v;SDpq05s#xQMA|p zivGn=>;C+QpMWv)F2z_wf}npmRN4h4K!?In!yPW7RHb0b4*hrg{0}_FpA~yy&N%#5 zLr>R_SU;6;p(lVn^5IhUoX1BlwF%C87mu;*b$7=Kpk?RPoO5KjUVVO`Kj&?I>ZdEQ zo0o*XwM0kw=$7K|d9Tj6+B&FTX0$Ug;JtM?gR$nt481XJMi*ASioQQqD?C(13B#R) zoM7=7-N6fanzV;u!r*hD+sm>A=Td?b3MHX8P%yk~`63;)jSY$wtodd#+haxMS{0G{ zGNUi>9d*h8E0;-%NQfbM24yWS^@_C^+_G(+Y~laiXVRexOD?;c>kXHM4e%{Y?uIpf z|MOVs&B*r_?#Xd`soi4Hsd|pWdz`nkH;Yqoep>TmK=9Q2F0F^@cqBQ$e{;>tRNpXY z@jL0ZI0f{#0#t;Ewxisz$p;R;$FG+Y6;6rJ14H1y_f-N!WiF_T^Z1kZMOt1R+;>-^ zna&b_=aGxfMA?n$Y`R?UK~^h?V{LcxeRN*LbJbg`-s=KV{j1rE<*l}KYLKsO;rmfX4vNB%uXFXVdwD8kS zARrSBQ(u#JU>dsBSI?bI4JJO}8UI!jIoW5$1;Xtc;5kjcsS9(qrFW=E0Q5rLS;d8i z_@OjZ%=Y^~Bbk-Am!{^%Qa-UE@|JceVs(NtlP1$*opPa%dv4ez zk1z5c2se(ErgOK+G;;ll>=&DoJL{LaWhzniShC*Jz3oGHlZ1J$0%7`k5-(uln+44H ze!Wb>yqmyj)Pd$7>(%faLp-xT`P)6)h)0&|JQmXYAvOII(7U+`$G-l>3%AVr2bzxZ z`+~#AEf9Okm%mkFEq;DiBQ<58nJD9quE`dtEbFOrtzi4TPOCK1A1qg}3)I|x^}l~< zRT+zHiA$#Pp_TD}fr3_hmKUtb@<>RuJcCc%TH6%6JNcWgJp;zPeQ7RUn{?;)N4jwr zBIhvwQP<3U$V#mQI2pZPbq!|g=Tp007WgxFW_IR1zAm)6`y(3mo#G^IwEu1KD;4ob z;Z1N~`OH<+8Cl`dMYSz@?hn&liKcwMOzS`~s**qpVeFJ+?YX{uF@GF{;(Kyu|4dfc z9K6Be;!uR9N`sR?v*~{4Dm~ej9Jx<7?6glX<>^LMfr)K!aEg61$CuGQV`#!s_k~ltHPl*|QFqu0 z5u2}_t9v^^;0uEu z41B~6d;9c-$72yM+XyHy=3KfDJG8x)%c?;i)NW`C?1yw={kzv-c+=mUZQM9Z_$vtt zst7FXG}HwASgcI<+ky+)htn=#JvGCAh#~skf)R_0gd1%J)$qtx=6h3~dZmy-KazMk zCNM3Mn9JI=UvHg6x8k*4TTEB=qhI{?-_tZK^WT@3*hGRtQ06tY{HZ9=Ao}!I^09%9 zq)EXwh8jW#$n=6JlO^pF+9l|}*kTzqC077pn%s(3HtATg7SmYA8> zSoDjM7>0nN{O=&tdbNMjYjCI$jUV+SbfSmipziCBOT-8+zMB7NYopfxF1q9F`!Hw{ zJZ)$+;n8^w?e>5#2+t?{=l*tiE1iBRy{4#6>cKhOq5vcd1mGgZKPGLsOwMDWzhX+? z{tUT*hUz>!1dl!5x-9+W@*7;DBfa2=9?v91n$wKN)&16;d0Labcp!&oFlKp~_J7~| z-LFa;PAqvWXAks^Dm$&of7B$#Fr4Bv9txC;hf1mLw9*jF)sNt}OLe$^q|OrEGCf~j zdapF%Zf2@gI}s1dUfH+E9M5soRZG`GqvhlK8DRs50D`e(=wKg>JODX|#6vwd-Rxr{0O&xu)&3X9oBdj`OY}>egxN#1_Ofm-`?Xwdc(?G0FI{xx@cybR$@yl&V)vaOX4aN zRkkWU>v9$-Myqk zXMLQ=kI#=?{xtr~lnJFBWm^#>fboQlLDyb?II7_Qn$=^MK6sa!E+@H_rDW=j2bbzhTJ!z@#)g{M**#9n~vSXB6>G zr_JX(ARpmF$P(cZo44T*wFtH1kO21)ICzIgR6JV49Gwxs0M-QN<-i z;bI2actX{9RJ;}`K``J~c8dfsGnMn2>z8aT;55bB2o;_IW3l6xYNXHp0Q7bl!(r5# zq0%xRMiW|^o@*D{_%nE^l#POlf#W3AQv8nNg%sYhow*3fW^tr*D87%_zhKstAeF{; zq@IGGR0u3=9gU(r709kYWnbIX^7vo&^d%m!t&)oT2bg@3`$Lk1x)IoUnqv3H7*G$+ z&NSGu4#^s{6HeVoB`F-+RZCRxLSBNOA$CvMs8^Jso0P#uXEND4%RC-H6th>fV2nZH zr+v7{5E$nz63O#Cj@Ilem+MQsa+6}1fFxKqaZ6>|vR`KxV^BC03mFSa%o2I0z;>UC zj6ap&tf}wPkmVXROm@(`3TRv-R(V1lxRf znYa`22p#jubm}hbTXFDOu^9fPLqhb+=vM`J>vLd|Z9VjK-Mvn7Hf>|;RD1iEfEsHc zsh=dID-y@Tu?rDNHwaPQ!poz4h%OqcMLh8m`giT60&czoYbF}q$ z=`|0Swji(K(xonyw?Qcj+n89%^$*PTqh z1bOw!JPZNLLT}&bmVI+R#`K=C)N`1LU_gL{LunHQt2>(wz!qY%MY*(bO4%IqLAEP> zA{SSyIwXmE=SV;0`SRD9z%TtiUR)_2Vfy`2$?a)-P^RCAO46%irXE-6V0V=Z*Cy^^ zJDAz+^TTa$-TIP0WzWDasY7X=+Uli8Z_tyQpgX?3BzBEKVJ7dSNeMxKvY*N9aKih2 zNtCo9kWQF|Tdl=cpx<@89oe)7BYA%T&C_B)c87oOT9XC>8p|vqWC?zTi8k@)phz4Q zc~AS>Ek5e8^oofC4Myw=IwqA)J_Safze0i)c7Hx_kpxU@r=#EQu|!i+ti1HuI2UX+UhrwTunm3br3&2c+9LDry?N1%q1xj`E zIm#gSm6Bkz2=1=CO8gy5IPoCY(ENNW zht>J!V4JI3){}i6`E>ijNlObuwRM_Tx1VbD_Y;IHFcIBVe2M>%NCq`T>wG3w>@`5W z>KVj{TpC@A4c(h_{*j7N@AP9(Zv7rDc5bYH z%6tCv*7eKiS&sRml~bzyw?wUD)y}iLvU07 za|{RR3Sv=2m5kaS^}WMC^940DNyT0~!+p>2*V|A`Eyq+QT>t}9m-O0O;@$as6_D#H z{<~q^LdtcvSKyW3xKGHGsO?NBrno9|@I$TXrAUdpT2E~Y()Wgt8fSje7v_&YO$k%Y zk$KA>%zGEjx&I(+z7v%W%5fp0AzhW>PW@7E8&@f;WwsB3P3W}MGj&eh-nu)27if}F z=|Ex65v$tTvh!ZQbBbR}Q$G_V^1Mylw(p3y_0o6=Wzqf)b&}M#WFNx2N2`xM*dS!! z5k4KofUrxd*xB5gMS@W3lRAve37!+$Un~gZ{{!ehi0rSNe%&wtdsRxWs}ZtuJ%07? z74@wR_fO$UaIp5LHy2u7?0eEm<@K(qSs{3Vh>ENUsEbc%7z#3Vx-Bhu-2Uco*27dS zCClnRW|>dqe}Q93PPfHn^mv~Nw!Os7yJblH{SE51CF3o5FECd_{V;z9Pc{`_2kG~3 zlbQakH?z@&j>JBZs};+l{c&mO=A$KE03&|+zJLgy6NZY%-5&*?b)hPe?Jx?wpHTqn z^3Y;GU0^6pYt5VT+XEB52q*jydXZvc;4mo!3%l547h9zB=%t$2gKP%><@oF>nF5p(_j}%Sq~F)_%M``?AsYsad*ox7%_KpUsG4L*-VB;2Rt^7-6)Jq&q1hF>w{^I#8!M%hS_zxU2wky_`a z5T8ePI5xpEn)$<>gFJnm43=agaUDy}8b^+juV!XDc$pw`GZijqiL5KhSoc3NWZ-8g zVpw1VpX0u#>q9Q2pOrbS@E!>BUvPG9asE3tJVlcTQhIA&6o5z@s!8{|{Pr;0xAmER zV7wD<^V=se;8$Ka6BZx3Ceup_BM#4#flTWse#r~A+M)~2-e9te%42q3j?+zyshR;S zv2hzD{Y66%w`ec3|NpKq7cv~8a+vw}n7eF&S?{H|@vJ{KmqIzFjh|9geynxMDzngk zN3a&!m!`$w1N!Nji?28W^FpRxQ3MnqU=~6$E%iCIq2O@Nu&)-Aj}j_%`%ET>hw+5Q z#```KX9&=30UEIx6Qs>=92OsDDgf!LNnB=TcXwvjaUGdN@7;Uh4}VNDLOxQRY(mWm zcCxLks9zQ!eBo}KfyYnYo~%!jyt6=se<$c2*C^Osv6XOz7w-NvX4 z_^%&653ITWMx6Vuu*=BT*GSk~xT3CZBMRGMG1K))+hPJwpFRh+&DSVz zKo(70XL(y4*K`)UlE^zNZ+PoMTe!DWP{cHj{>kOea3A&#nSDoCX)%BE`CEL< zOOuR?NUTuXQEd;1k`Iw$pzmqKm@4*<4qcOyG)BQYbs742F>kbKCrXd+q*-O_s7|e_({kzQ1~_O6CmMCfTa4|n z(O&T2Ejsg$)faj@1W%T%H&Y90CuciFgDoWquMG2=wQy2Hp)vD&7<%I00i}OYrCH~v z{%g`|04}DfpWzh`Vqv^L0N*U!cH!}#-%gY-zAev(%NO)0I`A(v#SMgze(V8~sei;4 z(7+!G3-_`2R@(wnYQTun;6daO^k@-^W(!bW{j7UkEy(r}9n=p%6m_<+Y~Sw`BPJDS zmY!W}Iv$uP(5q1I64Y^fs)7LK4WssSpV@zkVFLg0E+D;DSNx#Va0*@fobr|pEr&81 z^>=i!Fhqv*$k_l3+f5+!<>&B>g6bKIgwh;t;8v+UJgkkC2De|@>#^$D-sP9A2@K4=p7tH+w@V{jq zk}!5-2jNbK;vRZ9+FqW*95YgGm!_CqEgQ$`Nr}B^Z(WLssX3+=+*`OgEkXfgXV5L) zg(3`JgLDPii&Y^wk0GKlM)_&V)ZnD$EpZd+7Y=d{7A+i2GLyGS9gU2gSFN+?>>1dc zpCBEKh@2P?Nqu^&3G5;a?BVV|{%b##%Zi5Jr&=l@9++PTXem`19sin@A1@Y?dk2VY zGDC7aB_U1Ns{@Ss1=tg|kPP``0~>)bQFEfe_R80X0AfUhr|xZ@a4Pp}xj(^p6*V-I zC%@}@S+d>=F!EPrAM<+w5Q^yTrYOC6i3O7l;MQ+c9{GOGi{AlOalS<-i>{T}-*c=* zc3BTvo(f`r{KU&UhD%5;s;O|Pv~&DP>FuvYoa1!C({4`$S8j0Nmo)=Dz9SGgM`ntL zG)B{3GyWEUC)7qTb$jn>pS_X!6v1?bslPYUka#(Mz^$l-V0wedzt-&b?BMuOMDkhE zaIp}4RLz6|F7~G!3nJ*g+>M-*234(w-;oV}t0UOieDoVj<;AL~(`SEGy0@=pCVO^6 zExA!WnS96iGsV4Lz3jg|N%OpqBI68=>TsX-pe{^AQhO+O-I){e5Ub`f1?eW;&2@Up zMbb<^=s;)6sPbo0+G`AYDVllzS6|ey2`>Ndxj{96d!&UhQ~Zh{u>mBoy=Z8FV8f#+ z$RmKMW9P+j?bk)uxXUDHX3P|MHN#sPNOL_yh)|0PA}`GIFzIi%w3!34SR0ue_uyxR zg+)I29HT=?w)j1zSkKdP8*D@0W&*6D&6rJnlLzgpKa&igSq1svE4~ZZPaM1X0)IaF z`(H1?s#g1?BWJ7!u0R-}m;uImyV4MFgB|Yv8#~N@0V8_hCAawfjJ=FQjqdZhK(h)x zHNyD?q}5(ry$t<8o~>kjVJN6h=&{Do_nuNV;Y-SXvZDagETXrU+CAqEOnn4yqnicT z?}KMSGwD+EFooO%srRqIe=G z=8Mk2IhDmSI}F`XQy&l$Fx9HA`!!P~F-LXODe80Y38OxW>ShZ*R1h_H``tOn1at7@ zF$wi9D5~Vq2(ueFHDFS_71L|kIRyo^y~eg3R<#YyT0vP$?>$Q-8A0}Ia@<>G*(lL0 z1joI?w)Kz;41Ng?63v=R;X9Y>+dF5o^2-KT!tiG-j^8=tiu)#*m7|i|ZG4LLehn+~ zKbZDVXzFblP9Y6RF?+N4}GY|Hi;YAs&d*|BC)9 zp7_8Ee=75)Q}0w&ybi^biG@|*>3a9KJ)wMS5=9!k9pULdS-A1##rGYXQVO?{#>M`A zTc;B1>qSD9Qb1QwH>_b`4SNUv2UE|lfM-R(2U~uz#Qt(M!vC)G3fQ(BKZaox(s&}* zb|nvP-+io&1~p?>DZ&1QWM_*9=xxfzrPhn)ts}lG{gUai^XRcGFBLMM2F^7 zR}vm%ipcQXx*kXecW9q9V|^buhaQ44W|FNPq6vm1x2ibZq{%V^kb1=($&JRj6=UUd z7Ur{d)R4L}*uA5PTL7ikcjKb|OA$QBu}yw0#Va=X7W-%N_bgT-1}>(A3*C!-k4}5o zFqV5nv|q8!wcQKp0ck-_r$$oGhxw<<&7{df(w>lv!3mqvdufx!ig$iB*h(wmUz#~@ znG9@}Br{Rtsd8URj~Q?5{GKs`eX|CWz}$F}&zI+vH+ulmcPy(kr?LY*qck5~3+x91 zy-iaJ)*%bq_!_#WakrX)Z>GgAM*26-6YEoqlsMWd1wC9K^6hz14m@m+xh4OPzae#S zds6!964Tp8OBavFHo_A4n*UA9Z(Wkmh7sE<#AdGxj^KEB6C@mvFt(0kaM^7eK`{Er zyxKi5a5cavV8rUj%Qs~AgDs<951tyhmihSdIZA)SRi7W)b!tI3Vg>f?I2fMTNU=s) zjt4R?T^3;*27xNdwsp3EC>qMGv2G82j2Xvmr{8}}aga*l6=3tRW?S`E@GzPl3?>$O zv;i*r8xx6I?%@zHa5+bhG*6D5SH}t*LS3s5L9!g}P18iz&_heGM|(_1I;sguX2Aay zk2KBdbeM%=Zb$+ZMll9$Y!@id6x;IZ8XA0gj=l+_f>O+ZkqYG=8>fxsgW@Cni9_T9 zi&6p=3S}(2Xr0h-o$*ny&jfVfq-`V*V=eZj>P8Ql&309&IEj?A4DtYd$m&Y<_iR5? zx3Eu;bZh>AfCevzZ~W=PJt5E_s!;85dlj)R_Xn&~8ul(#H2PWAE0r`G*>^L)6?H`^ zrx(~&l~hcnKF(C$h`y7{Z)jk%did;smyt6*v(d?rfqPbpurqOlgglp?_x&%4;N9>V zg>&dMrC!zpl1~+4QF51{LpByiwgs4(RkSreFsE*O^0e11Q9(I&#?0Ku5d6inf?}8- zuP(rhsxw{DV^73gF7a}B^VZ6jo#tCWU#h_{tg3bRh~=ljjAzE|B<->iFRLq#DG57u ze9;apywO=2jLI{)npcS}N_=4P>C!7PDvwr!mCvi2NW0RFR9@_tkip@;J>Ysklg9f( zLGvPY3!mwSp#D{DI19c+_dmc!XmRrS{1|45+%Ovj5rp<*PF-y||Afk@zn?MHwm^GU zg;)YDGOYCuH#+ujCDf1$_X2q zLV6RYIxL@EYd_!w7#EK}+XNT-5P8R;Y#Tvt`-kcf*_bK_faELu&z?b|$o}VOfn&*? z2cMa|aj8T~81?+u=2T0~^M-r`4%gpsfXzqsoAr$G(EL=T?=b=NGPy-c=ou#)=^NDG=?+`e6)W0=B0 zP9FUrSo`_!^`H!B?9=qmMEhgpof`E1YH5XU3bY?8rO@3YC{__oR;Ijj+W~;WuJcYu zvpTrig04^guFCJO=E-6YVX_iVJ9neL3pFlI_7{f6gRuutpVP47eQ?|WEOI>+LpKj= zf)dK3(igS1dnnXG&b$@UTfdr^eb!UBZpY&Swh2&P$o0RGvt8Gn@@ilkc(;>%!|Vw; zu={aF9uRm6DCrJ?sV<9gp@eERAlvwZ#t+Zmbw(T=@P}(iomztM z1XQ4kcAcZsm4N>IOh;I6dce7b(u*VKwbLMV$nO`M)fU07ST=0%d|+|^7$-FUGHMb? z!!=QyfF|)s98cCCcvG#yh^$tlXC=T0zr40^(%xQEQMCo@8OAB*Lv}wM8K#t1)1mI~ z;PI>c58Fo_y5GHXz|6f?Vxo`7(V<}JW|m4je{+iL2^^LAXI?#h(5&uAy5IbhZBo&# z_DiLb6+sj|brF*64&~?_T2j|81W&shHp!zB^2|rm^urOufWaC0AYWlWp*m-6MJa*R zgNWwswZUlLm@v-m-HBhG$Ztk`O$QO4m-FtfYen?Owc-DZcU)TCvO~aHj1T@V+L5Fw z2vkR|47-9suu5#Gz8XeePEw*d=-KA)=P}1!pgQPao*!f9xFY9_Q~nUW^gCj)W%h>EEn7M2CPd$0iFYlETa7_QSV} zPEQ)aG|#C~u2PIqklJbFFejl7gEm?Rm5GL+m>qVYPnkTw9<;$I$m0Tgb;aF)MD5EBO#7CpZA+cCmM{+{9E(=)eO!nGcjjOisuL!qvce^{XQCr{95^X7Fm}x zH+-J;Y$F-2sR#G*F>Io^KENVAjePZ;SbJLSXAXplruV`VVa^}N7ITa=ix+A!19qp6RZ z8Qb{!P9!}rBzexb^=57!)}Nz!d9@_QZ_SZ@SVbRzykDkpNxM2aV1-JZOFxi@ais%D z>J+Yw_uE~vmrpas?6a37F9b@HOY_~_`N&|C#BhOgre^_|-c4e^psY3}#t z&*WCAP~);I8V_7XYrlUHcNXhC4A3v%^F$$a?$9i=*m3gHT0{&oR!*fpKh?~+5_~JL z!@UWp&r4nT247Q41D9>!`qZr7sSnT06dS)jTpZec&0{~RUop-emw)R~l!0zc#)BVG z&-o|>f3zHjkE62wPSH~K7Usd_^}uA%Il=v@xBSBNCQi~ zw_BQ}djOm@Xqa-&jv8qydxG5Dwt`_wIE%>pNtpeDVTlaGV!L`h*lWOswd>RX+)8AZ zjZP43GpW)V{(mfhWz{t9P0*oHJ^!l7q~P!59jY`O@;&n{sURS1kY7Jiw~M8j`{}*& z{67_4%iQ?Qdk<26E8%Q|Rr6Z@?YS zRPo5dxT)8q{0B2+G#742Y8#-Al&~dnTl=T6tKefoh0+VqRydzF9;i#{A>a_lALAU$ zQYK3`Y3({maKq+>YkG#+{kyJYxo$0r1)bcn>MQ+~_fcFCf-o(M@1@XZ-FYUGzM0sc zXk8kkaPiv&4F8mJ8ZuY^c*btlTy3nr$w{nA{>vn_+6|<>|H=BeiV$`OZH;?nt`{Lt z)vny>)7RwC|4gEuA|us6L0HUJOZIS7!tqio^$W?28{xKKbr?W+2`!?<-br3%jy4eF zqIFgyaq>sJTj0mJO?vUQbelP*f5X;c#Mr@}oWS z`heGDumyW3xZ^8?2%ZAKX2NGo_~fUFM~2N?@CEhMRljF;5LF!w*S|c{ItHgfQUe`} zH~YdxpjUd(GU=H?>dBuPFQ&JUG_99a^`K|pM*=b9Ux@rECe-&UgzVj_^EmaFk)#mE zgDgJkw$skGT*Nco_IT!zDjG!rdT$KZSZppNUYJ=Z==TNnt&-zn-;K|NkKDf)yt|uM z2=zmjSew1YFbLwVsi5i&rcOQ{PJ>%MkRz3lXIDf2NcU@}Od017sPF4fj$6n~)n z@A$O3>9rU_TdlEH&_mq*!%Gmv>(~e1nsk$ASbr;7_063@=14T<`Uv07UBok~Xb)Z` zLAQiePB&qBeye==3C{R99aj9`>PG0rm}~@%@ZE-qPGJ zD-8GKcEN5lMc(iPKFV(Bu~eBYF38c(k%KjS-u~AhUPoyiBy{xlVG0&h6sO=td=z7LauI(rP9MDE31c+fLuxST@}RJ^Uau#fC2umTj3_W=}vp(!pBbGT_q!IKuDA zD+(8?3!1x{cQV)WdlApt!f;7H_1BzS#F<%|{D~)f&(S&&Kb0TlUyx8TG$KUy&8gB5 zr|STcxb>uwoOt1w`}`s081&z4f0z;+xL$D`zL_CZ_3zMmPD)IjvYk2Oep#NR$ej+u znehk9PEdp`O%WFuf^p;ha<;(xA1XZCSn2IQ`1o0;-#AfxT^rvt5ik(HM~8Lq(yIL7(% ztHw2r;IxsV-~8{ZZSvuhP14YrWoj;YS&`w(i^^utdGYewCD=#nO2Gs*OwcJ!_tY_7 zZuK&`kK?SZG37vuCyT41A<`tViwU-ySL(@@EXCO}vOkwa{7&m(UZ2n`PF^19*m)FO zlu|*m8l<1ce^?34e{&o~eh zAe>}VCbtcURvY6;@|h@0%*$SWDUppS@Qu2l_ANRpTw%{&prk{ks1Wlnw_2dBQvz~Sd5hy-1nr%gwrsbk(84Lt@T5q z|II%>K=A_RyK20dG;8|q27FHWiE~QqI8C$$N9>PCyvwk7Wk7GhPXPjmGWa>4 z{d`0G+CpJWU&mr-@^Mq6=!MawXLi*QyTp*>_RPdAz>YHlOqaW5tZ_1j*jZk5xQgPDx}R_%?V)0XYnFna4;ZcjNMtl9)oC@{#GG+O*Eh)}PT1+84L17&-h;)NZT{Z%dgvs`Rw>br^*W+BGHo#h>}pmrXUX^XnO z*q$}ENH(kK1@hXsW;J$m0SoTyFPL=4Yd!Tuiv=7wN%A#5lTRTQuN!BvPj|@aW9+c0 z#B9Cu6qqW-YIOP1Cvjg7@q9wZO=QWk!1csq5OajAnYjB3-Qw?xWI;=`r(LEnbgP3J z7;NRh0R&9}b=sWwfE&g)C=cKUU$R7bt*r?-SVQ_%iNJF>km(KO3$EMszE0q zZKLa!Pz)OeyS}#>FmjCC~N$G>ZK}%dP!@}$q@3Qx?p5kAAucV zpZ$pWU2!V++x7792;cPzTHY;^H~$xQX?xI|dDbuGs0S1(Crr8R?cR9lAy(Wxe9E4g zn%1i!OmROKpOQ0PIfE69Pzq||t1jOM`^6y1A9V_!w|S)T?=XQe8e_kw5|{)KZLYN+nc^#$*q2zBV089t`-l zUmYb?4s8vl*sUf)&Br8e%&UL0Cdytrmj6udomx$?s|gdEwB)I6>^{MeyhMQume{6_ z`Q-bR}>F=?DrR7y$mcyjRsy8fJkZM zrZ`9tnIf8DPF-anzlDLMw@pyt6Py%pI-@rK->9FKrZkrJ=W77TUO*+- z(BeyfCs+B8)REIxB$X$q$}G_Jar^ZiL!jmREl^&X@WX;r$q>Ft5p>0rrfwAv?g`M2bi>TuYZC?ENMUc;#D~e>-skLW<|UR(9EnQ z>s&SHUHIsSK)Zww)}N=fi*;^GP_)XJrbu9W6$Th9fBp0XiXs6pyEnA1-HTo8YJR5J z1Yb7$)zV@Ucfzr)c~^!ReBaFuc?@5Py)c1vcY%}O4JId15q$Tv(jL|KVJt^NaO}4K zBm|W5KV>5M3XANP++7Q^o8ff1H}Lq0XBe0MHx|gcTZS3YmVUQqAsgN6vNB)IfD}L1 zbYM&3OG1ZsuPF&Sg@bMoC2L=won|fW_Z-J<)p%USZ1|^n8%D%4(V=}OzE#2QN$>h3 zg^nSDhH-$eQ`7%fb=$zB7?0s_r?RirozI?pzL56tn};@ zdkvziBhsP-V1lPn7Z-4Xv1zX>lzT}3T&GmWT2;#<5|1MT698AS0E4Y?%T>k2?u;(Z zvlXG*K-iE-MH-o=O#yLK#o)NJ$q!DxNr;W-LdCySA1^s-aXax&doM zJss5Buye9XV#D_q#DyiA|8mTWzCNO)<7&5v-)bNCE| zddF%JTK_BWx+LLIBFh!!>wi$>r`0Od&Ac`@6SCT$zqV|y?3q(HMcVEGg7byjGf7iF zI z(d;|akFDy5&r}q^qgWCeW8G=+PzA}vx=CJcvci6eW%(zq^hVh{zenO&ZBwZaIzA5j zt-WJzy-iZj@aNyI?;{PQKJg3fT&ZG|b3C_(R8`~`fbUzF+%E0yNjTl`Um}%%)`DMQ zPmxbtb>fr*s+{z0PcV;ysSL#gDJ|Q7n#CH$O}&f44@AYKerdKwhI~8b-rJE&n6^^W z$n?{#>AqMc5?hjUplxIM&xkU(a(76SsyxuPlb2Bi-8|rCl69`$`eO4z;E{Ntbogz5 z>=N)Mx@>5G^sm;JtywiVvfjuN>dw8OhKQd6LwGmh=toO956P8aMayyN`xb#3+fcxa z+fuBx>|v|!^+lwm?my_zvDOl_OUdXo`IzV1QmYzJF$XGNRfShQGYoXk6>@vkWc1U( zbm5?n+ceqdAefp^L)RO(7p%FnCWd^7U+x4cj2_QJq56sLUZgF@yS}Nb;QeH&v&UP> zDlvzle&vh!-~qMXG7hOMDn;fpuT#I4oP~rN)v-ARXaI{_a}=zbDrL|xM{o?|2Bbw) zk-WSZDpokXpHZNVSOq9o8bNl-LngA@`Ef=5HXU0#=^_HCW$>}YU0aUNr*3tgNQRH` zRwu$paAj9}4OdZ9NqBa~<)k~HaYpAWbjfR)Y5Twul+1Jc7kmFH?We7XU6mSwu0ami zQ#Ug=8NOPD4}0UT*uQ$blGqlCICXkG$r^r$Nz;6?SRGq2G)mt#5m$8%=cWo(Aw$^P zmw$$d&eRz)1R*5y*@MWRAau@Q<(yaJuUNSwmC3RjA8ey#K|-u6#Z;X+L{LCm&a_VT^H0frPrxFoQ{8$f$#LMA6tDQ zJy|^bki^^zx|)x=+lHpLsu1b(SIr{Low;fF?EQH%%dnsrYejO8u^+{sevA?C*AGy5qf1c# zF$X)(m`CJ8wEEx26VP3g)Hs!vm!LWjhE)tS_X|Qc%M?p0ei0wfN2lZ6ys zR#6{Dw%kpRxN=XI#PTL$=P5CB*3fk}G5pMSk!KBaAv3dMB3xF4Thk{zVN{S$v0K~i zp_xYpYgFXTTyb}Q1lIXMOjGb4L=Qq%5+^!&F1WHpZphlys7AqO{5yO z4$TxOfp&A$ZGE%+`?h`i*~(bfGlNL$a%E60aB`Qa=mh^&=%YI%1t{}EMlpc zP}6IHbQuRG-2R_WEd$>2Bo^UDK&W`%fAsNBDrXc^G2VV}tiFEWvuiop`PUo0=|5Ep zzfg2d^SOlTM4ay_k$;@9t1c7cur$f#5-Qb3yf!nN#by%4dKgbc-|kZwoPLVs@aAbB zg^cnoPiyTXb`bXmIHa>F^7lv1=-=3ST9P4*N6R##U<7)>5fq$T5WClZW4fg+9*x?x zPX6$h%u}OHux|ceK359crotq+OE=5iwX*ex1__b&{$UPXCiPFu{m~SAb`AeZEwCL+5cv|!@|6=IbI8tl9Z!fiD_FuSq-w!5yDXhq>`nMiIpQ3PxnfEjy|suQLTY2k6Z1oW;)DMhB|A;{2q>^!>{~-%~YMj-AGd9F6j5?jUusx!2uy=j2nJ?LcHk4 z(JIeqY#H1zv^j@aRsiCnn8Vz?UXPs+Fn&N>zEChe2yd2+Z5pU*8(I)dB*C^Nq8UE? z6};(o5>ga=s^4-@1da$>iTmN%Pto?tp1u@7G(p0DS^2t{B?7trU9MCVk zf$dTK@bdY`C9I*49uVHc=QWG6*^Dm^ZIa;e+=*Taud&{IB}ejE_s?WDsEVibyI>1c zyKraxucR@P8j=$gDq~wPDr)vaq3PC?MsdKhx8C|SGU&OlQrjo^Sc{(j&KxW3F34|v z9NhqW`8xMPJHf_qnfOI6#2)6G$#a)7oZ!gqe z+a3)b3}9Tx%y$@{P^J!f3bUSsqpV|$$xBU(7+@%#51=9gR(smgCtIAtxkW{J_9$JhuV9a zjvS6h=JD6SIqcQ#`riDNq@>cd{-h@wDL((}_X|o+&BDN0U(o(JqJTO=&T$C=B= zU}%*uSYW%2f+~Ac#`dtiZC+V98XJ#Cf>JeBL1^zk&Q)wG@u^PJ-<7P0!51!Pfl?gC z_a&_l!W?_(;Mp}v7%n%*&`rCqIzb;KYCo0x@0N}Nd?nvy8?ne7G%^06-5P*LxXoLO zW8DHb6*c<5Bp4lUS3!z-8oP-Se zQVf6EEbZ{GB%m_F_Ltg;0HtdAkd6)A%_L>?Lkezr)V=d2V8trNV61}`g4Ocw7|`f> zH!#17_s1Yvg{)+-XA?7|>^TDQXd_M92}AYc0PNd%Z{Q0K$L zq<*F+AFG0}=~_1!6*HzFL!b2)F0Hip72olasE~^^FxoZ$lyAVU5dT(-7xYn2k! zIKuQAg$XqfoExS}-;l;B;&bSv+4WJ@2`Gn7pCT#xVUk}Sj;-#Rv5830Mm|}lF#del z3LDtY;K85cuV<9MR%yxQc*awdOw*s0G|!K$YQD}II4RS=Nxka2`9Ke72(&+i0F#M2 zQK}2SL?~iMoXcw>%{}(mg?8rW)yFd(a`>lf_)Z_8Lf+6FVW!D$R?wtL=6$9n^xeu$ z$F#xu+L_Evj7vL>3+vMEl51dRyXsf}4cB3(k{D@p{Xm+r>UO%>L!SG`_8ar3xGSA3 zoY*hwyTBK2emLK0mrfA#9=P{qCAF;U8A2ZA{Gu)(EeSefo`B)%bO~UhcHA3I>uT)$ z_LyYsjHkw%i1>eD%rgBN*lz9w?G4BG$$G@^uOHMn^14}Td2_A|SiTn(U`)T(rb!VX z%)NxuVeE)YQk#qtsL$QoIBlZmr)-HsLkNSebhG#D{fxf6K9NgPNjzgKc@8_ogit&49I0=HzRs8_o8@8Jg!fC z!X)&sk`#(+_unl;z(LZ#;Xo7(pLcBMQPf#w3N#d6*E?RMQo(Qg3+;_YiECS>!W{0J zi!dsoFtY0XHzR-b3S9i0i$|2s0gbT5Z_sgBo-|89@5memk>kB^=%85DPlqpac+A^P z9vBLWIe1SYnX4B2;NccKt*|uJG~?9p-<|~`uNf1$1R|fH)!%-v`qB9!6@Um_=mD90 zgj8r_Gdx==Js=y-u-GQcd_I||dX)hWO?*U6^&X{$D!F?Dp$q)vJD~H(-LVPG3_)1? zMe<4aP};W7U=k-jVJuofo7LUnt$qHxM(95gK^zZwBfE4e&Ikfhh86Ie7(IG@bJekn zj%tZ3swOfUyE_+memJp`Sf)|QmA5n8g{=nmyEybXlDsg{mMPeAd>;y|Nb#>{V0On6 zjsB?L*;ty7+qi@JFYa2`;5Bm5N~pCb-qKjLY1MLN0~ggYl$_wvs(aMMtFlvjQ7X>n zuT=!8i#TIkH^?h^M4=`A)0N)`5ZuuY+KRy*xjxvZDNs8I(%(HJ^ZmMGuVg2eLtq?M zb*qS$TD`5L55Ck@@A^ZG&T_fF9<>FAFYExGDmkwvOv37M^lL3!=PQ^h;Wfe;rD-ew z6&(k@x`4TD2*=UHA6b%gmavO$&f&1`PZ6MbOGJ3HVi`w&(tu5o$GhN|vFYjpDCt99V=dcR7>@|3~!8IMo2C zOflGw+6nCI@o4&f^`O><(5Q{mCpiUGk(5EKI{|*e@R+lv(36KlhQ>tBcO1?!Lf{*p z=IyORVL>CTvIqjIp|65ALRZsYiL|)2=#@8E;yuS?RP0ByB9y?il-$I9IIr;`3J;p)InPU8mxm_oKGmIC9XuG<17q@`BDjc=0F)akYBgS5{fY?@)CcE>M1fsb{L&kWvX;@zbx0FbsAVm^tiCh-d-sVYONVU=MjB56|q)J%@ly z3M2EE{QVkM@H_O$2Zs2loU4UIfgKu{p@hggo2U{6^}T;r;(Jwb)LihsZZEe@*I}6l zQZ+^k33?$@T>?FVkt^qqeH6bA^tsV3N31a%TWpYOFRpKxkevTq59JR)mn(m z_TZSF%=*neI^nie!u-Bk+tR(OC+_%`^GP3aG>7!qK}_kO_GErp*gZ1dXY%5*QL)Ot zz*pnUqHv(?SI;{?f%ovYF<+)a{NXh|fv7}&od?evUCJ1Pstv>zpR884DLr&!Gd#1; z{DxxW)md?ux%Q+h<20@@^s`fN;kbK5#R(OlXmiSmj2mIGf>Pso3qr_tJ>+rlj8PU6yUrL8;!72c=b#>FD?S*7A47=@+UK{>ggfRPM&ANod4yd%9RUl ziL?7v6;HoqNa7A@Tl}E=WsP317S4}NOq|B-tZ~YaZ+qB-EqX6z%of5%9siboW=oWT zK#MAOA|zb`C;bZVXdOyYEm+uaI>>YZb!8jS`L6U)GfkxOpzUW_ z0(w5SwG41^XlxH-GTu|)Y!yJ&?S5Z*0`hcM0q!kAp`z*R3OrRM9|?}2F_Pd>^Jncj zNlW3#lURll93gOLcj6t`f2-e9P1naH`*$G9a$uvbIzFT}7Q0TwikO`PN)juB**Q0? z&yG6vueypGQ!O_>7D757LnJ6<#k81Tbonh<*#C8o%wfSZ@uMa(x?%3i5S47=a$Ng&(V5Gr2=pc(Uw;s{wypr_J=%P& z2B#H!p*O^oGcfZ>J#;PL+vIbZZ#IMS>F>~}){4`YcX(4U0_*~)NV;8(@EZz&a6`tx zHXH@*#w75=AtxUCDd~&M0DD@hS}d&n5_k(Ip5#y~ccKiE`sLpw8n%Lih5A2)w=yx4 zhbwo-M>Z_5U`fk()_;z?8bSn}2Tt`Y$W0(Zi!J%m>Vs@?$C{}0yRFO8L^Xv<{%ls0 z-{qRRTC{;DYDSLwkf5-h!khH%)YHM(FJL8IG0>?@X5$&FR{Q8Zl%j^?y*>ua?)$Em z{%3z-8c_?XS?>0fEwV$fUv;}EMe+JB{5ON=jP9qVjvYCEorE|>jaAjVQ^tkLbHrv$ zm37WUk#1q{uF&&&!7@6bio6Evx%_DuOJU8*+TCJr`5JY}rAb>b0BxA9G5Oeme~j7P z^+1BHt7s#}pnGnISX`PcTS;LgX}X2Vk&+9V-(9Njn>lY_u)@_N?)B!3@l%~^@UGp) zGb&hmQiL0=F$Q^04?(x;4AnGMWS{8^%N;E^pD{+@$@(Xe^lgf0`xpKuVFZQ!!U$mE z1q3IckiDY|=hiLy94w>`5nX2CA25eU(}#9l^lmo-WlovNb-*ZMa2h^VsWjUCwS zhZ5G7d9exScYn#FNrgOvE3JVl)t+nQ+A-3|Uw9`}GQh)-L>7S_SRU7TpM05gWE)$1 zbcVW1Q_i|Im?>%&r&4#h{t_}kR8Rzq7W;j{sz`z&s*sJi$o2XW{-I za%oQd@5Acy~`K*HH9n>+w>j*s;y!Kvw}e-ZfEjU0!6 zJKG^xn91<(nC0{8_N18L>VR=#`SF_a*-a8g^z*|9@AC-nWzy*?ONZ;sUnMZzndU6~r6+Jc)c!uxsKTs^0Z5Dp-U77M^}Oh)t1~SLaA_ThhjT3#%@l zf&V*%yJ3_!dGv5-abcy;YY-6C5XgT5Cplt7XCdKEfu5E=ySPb&1gXhn8^5m?zDo(5 z6P>XfLRI1Iv?<~gbk1_KGpLpWD->|a*2xgTUvIO_*+~KI^4gCvtIJ61(AAR^->A^S(UF`y=7=5 z^p8h>vX7~pd78sRh*I7v!23z#1_yoa7Uc*KZQP+By&oGgMR}|Ix-C~FAOn5}$9q6j zcSha3LcdIe6Lc_((FO2VZ*~9E%>OEiXi?k59>E3fY}y|jKfCLv&M$v$-Q43QaXBPT&$mh>U+C=KA@0ja(@DS=m6m%=g=a=dhK{m^0;-!;%O?UsgoVA zdelVaSqbJa7l>(&W#ye|z*9Wgoz ziH99sKz{VU{LHODVU2HjC9c0i?c4w|(V|M)=d+Utcrj;IFMo}HPnGTm0 z+wR64lM2kC^q*N})B77}%oBG^JMlj1Lr9p-hiv)lduX(pD11oOLesG7{RMz%=YKgM)mo_Ifji-9alIgtf&kZm{G-30s= z|L><-_?8VNTkw5_GU!nwr1ni(RW(W+#qIYtwo@7(1A!mn9X;NI>h|up6k5x~9!%CX zkYWA`>z3K##P>_6cOqo*Jv{fY<)G4IRMhyVCJ^^t)h*sEnKqDPExx0>WD{R4G90Ws z!&#s(Hg~$^{Od8yM_S^tAP&?AphS`8*_zFl)R(-)kU+g&?~L-&l#m&N*uUOXW#PT3 zM{wt&ho<7^x|(_!)Xv<*dHcAPzlxc|+@vTlyw!uqM4yVUVEr&~xu;dR@f0b}4EWNZ z&v0I=LDDQSjuQo%jz^5|s*nR_zvh8Xev z_k?Q*f-QO3XSO*`uV$Aoc@?F_9YtTRmNvB5G_axkWB!RiI_ zBb0KAYi+o_;xD4@=Ztz^CKyj}4$D)z{c<)P!5>O5G8M0{iqNJeEO-%ng9W>{vAuh5 zou>mMNJEVu{)xc21na@Zc_#DEzmp%HtYLZp>{{p_I$oW=rg;CV>KHl~Mp>k88Skf; z*|rmWLekMqQzF#XT8wofvbLAwc*nya9f{CcV;F!DCk+ep3MS$kWr0nih#Yl-04nH$=;hcGJE45@j^yHcg7V z7=m~;4;Q+>1RNZF;;?<)Cv3O=#@9KsZ(7gZYIvo*TJ9R+c;3&ibZBN8?#aCBU6BKZ zXKv-6L3T|Bp`I4S07V}-0rqbXxK4si6>J{6oq8A$ry^s}{UGECDgeJ@~Sj5(C+n5N^1an^}n%#G4TtzO86O$4WTnCXeBeW-yk?BU|Q@w4>y5%E^t77Q036Ct7Ytnf$Lg?2{!wS^A2*>q?= z*BuY6y~stSTR^?SP#Va+#HcE?fEEz&o}z+n+u$>mHEPGP;B7cP494N{(1IVa3U=@4 zA9{*qdO71+XG6@{LIdd;)j@|GM?aJ|AXRrIO@9nY{R0n3EKY^cv!=H5T*1V&@6+N3 zB!oNqNWbtm!tM7#fEY(2?B8~`i4WC+yEjSNUl^p0Y3^}W9`w_*6B#l_R)zw|%WvT@ zK?VQGSYS`S-B7=(=*RoCH{Lq?#5@P%26+?>yw2JBxYLoeK$kC6x4HBn4yn1|rY#)SZ9iGROr><;*v(IH71AK`c; z_Boz&^NJdiw@^z|D6fR8xoVAG$|kJw(sjQOUcBi>eo8k77pQ?vWR^2bgNPg>Jy{^qcN+SMnxLOt=*jzuTT({J2G1||2th{nps zoK0P@Y%ZTmrGgx}v&-ok$(8u(0*;S7X?BRrA@HX$*!)u}Xk(i}5+pZg?)l2pJ1x!b z$qH|c@A9hy!s&_7ps*m7wU=!SzJFidMQ`T|^oedfbZ$r=h7qjBJ}>$Fe35Xq>^(Ig z*gvnpzXLoO$L)5t2vXw~SaH)-PcRI!tn=v0T%Vtk7O2R2=}O%$Hn7mB4`y9K#JbD3 zUn+;Cd&Qz!lfVdiXx1robl*}9T5&XqMbQtuu+VLcOfF3CH`SN+S^yA~vQdOk=^R8I*WCqlKi++?GXpRQRtQ=GT#<;%mB^px+>U6tbJ4K5f zS53k8%xAZ|vZ1r8AE|UEo{;c`o=mWgkcB9qFk-4r-iL&newkADZlcQptN5Zclf{LN zpXB!#S8&V993;0NQ3nj?27F4yk6P)kVbx#G3gN4BFxz}^uHg8nz4AvvB3a0)dNHpU zPH~I=In2IbM3;#7|FT){>L7^wEumct;J#nB3zZAWFt+-)GGW!$(c`dmvX315shZwG zYl%MzS<|3(*h4G_&|k8#e}-JBclQ)%8KA5u1E*OZzLDOU`P^=Ax$uuDJSR9a*rHy_ z5u*>h)|ihgjZ3W?m!(>b%w^}PYWt>{-8lYnjXXF~c@MK+vj{`e#9%eS**g}0R*4DY zr0CNqBk%K!LvizRoEL}{ffeU)w;wbgM%V@yWW0TK)^Yt}IOLibLbO!bQD4CKa~Q|@ z%;lAi4S1HGCaK9+|1QgQ9W^Xi`srkz=nh_^5tEaEjTC??ZGDSXWDGIkCXsoH2@PX) zu80sn<0VS0DR?GYWHzQjOMZp ziNsw_-E-X6>@?M3IiUbeSJr25dG%>7qN6SNYn+|J`)};tOZ|g9lk7O+>TcQXAC;B3 z;7t~Q8gf%jy8_Xxg-HAF<-Z#TY*AL?6b*EGgA;UMc6b%TwaX$gT3;_K(sDR5Ph_SS z{Ju`t#(e#6+p;T)IdG^Ffixb1oJ0l;RkK}FEjx_sgEA$i5__CaCvn|aQ5gJvTp6~LRIP?v4x4{bq=PNvzPR>qOH0gs#Ri{X%%90 zQU1!EA>DFX#drYDgw&4|@c@TY!z9V6ztjx<%@k@0*Y4dn#MnB1dX!31XoLBSdwO;P zLcHO?1udfOq9|ZoM$aGVbT_ipV|STl^6TILbBd^aTAAw{l+pYPdApiTOcfe}>0863 z^VGq@xPh&Anq{B)x9Gy&iGIq7ZHgF>Cs1R`+YUZpZO^!qFqrZWXZ%X~XtX8lBQo;D z{nC6|;DSDc^%$>fk@#T*vA^eKQqxQ zbPk2M++>H1hd#Yjg5drznuM5ax+QFxci|n{qn+77&-1EveW@+WE^T_3L9pjePPik{ z19} z>uH%4x0Y@DWG*_%>*l(5%HJ53VRm2lnO)UW8xC-gpDOgM6`3ihnWmMsexHwjb}){w zfO;tca&>M<;dkmt7A?C6{u`1)Ggt6bxM#i_nq)zz=WcxUHo+hfva(cL_Jc<1>V{If z!+U*PDM3>G2LxSaaQgVF1D)u}lY_Crw2RtlxHC8zk)i~cOzV<}H+pyIi!s>>o)MKMP9a|2tMptrbzr7;cxjGh`y9Bm;-NF@voN?a$o%3DrUuc z>AjOIQu+BctzjC#`oMud7>OuQ^uWbwkwHdTj>1)l>i#2K`%XEM(XvOn#;r& zG|F4AWJ?p>RX(Y2qsHy15e7)OrZpM6UfRnALyn(4Ukyiw)X@j~2Ldaa>!Z;m$NaVH zPcZv68AG+hF)nE0veVJ9J1~z5tWJmS?7+v*7Tcc7j!F3}eNcJRqpc~}r$5M$1`RKaWy`Eq;|?zbEP9|eP$wFbpspSAHHtPTw%vmR7L zts+^S-!@tDz3g0)(3|&#BnO_%wtYuK#Vx@k723!XG|ZMAJ10Lhg+=z>d7pY#qk+$s z4L?iuGl4NrhRA<&Wq+!1T6Wjs(Q(`hPrZJcNj=UajQdmXNfgvLX2`%Cw0x13~Q-h28TuxG_(dh&j(z13cg%^NWQ z$GccPYU=hq>hwDY;>A~cY65So+?=sre+d40dawuY!|G2@A%hKAd z{x7$#dt8Ynb+&DMgia5tx+bbVevr**egP5v3{^T#r{gWY0mhF9S0%7dRR>cBb!Tosqw9*R#d6h-s4~H-aMJw zlOb|iBu#|TFtxrRg7@;#tz$WT%iwFauudtBkIyE1yWCO%!u(GH>m@2s*-E>hlY+=8wpw;G?aSNBP~h5-;~2F11iy+|s|MS*rs36m9UN=@0nq^b7k^>O3_l78~JE}PkP%`pPA7>3t{>0htUBC^jRV_mBFVY-WBI) zCP3>O9~1Po43G}=+op4I+U~WGZ4aqF{xHKh96PAEt5)xSTNro+O|WQPmokTBoQ9Q> zN9tfpmpN!Wo&2{AJIMG*pGsw7fe+d4K3#(otI6^=OYSzWT|#fc=%rv|gZcePI(55Q z!}VRc5q<5yzVI`-+?)W~mAO^btyoL1Vq+GL8SH{h+syTx8NAzNY4AVqV@a;>qqfQ+ zJ2=oVl8FvB{ZNd7LV!UEQJsd7WtCsSpgj#%-({LJ%D?fymZhU{)tUUM#U8dw9C09z zwuY$SkE1bPSmXFSL!-qd#mQPWS}Q;%?bv0C0z2CFLCCv9N?)RV+B@6@!e}2e<#d`y zDYTyk^ngzvmU%(nc75ALUybcKGo|lGd>+f*Eh)*|pLpu>qDoL)3o9tNed`jMkWyk; z#JTzeD)HYX(Exs~N3xX7z*Ffcea&>Sh)3U~~%>op%WmJ>U@#d~$=f-e(^azQb%%#vDRy4K`R zTvy`=>db_q1NEWRfpl;sv7ZG`5E z`OECQ|E9-&B?D%fUm$@faGK~N%<4yt1khsqGjqh{^)$!^|A%C3$>gS5`0y>r%*@r; zcrk!YJYx;+J9NA-Eb6I!E26A^hh<#$mPRFJ9lG&YpBW9cS(l$b3TquC`h#)fdZZWt zd&{1I`~1Y7l6ng%30qZB4)*Bb!sL^%QRua00^BNeU%&5t;%#Q=XnRU23+bI==xatWorD&uVO< zssIWrp!KYeZuon)%d?wh->MN9gy{e;aCjRIH}4Y=Wn>+$AGHbjE-iW-u;W1KxZ8oy z2!FeK4~3VVEV;f)(u#?-bT0F`fHpcuoC~|(_0bh0_p~*(%Sws@1~AITNQsh_c*G`D z7!$0Pl?CNvO*B72(SImS0kJ+pmToPlNUl78UwD>5KVjw8L1IFUFZ`34DVCgGy&Sla ze@~B6Pu~%Qq^QPAJ%-fDDKK4vglqX7Oez92B4hZs zI$52H=<#5!A1IgP?kmraV~q-%$nj<;*1;E3Pr<>G*A`ACYU6^mY`SEaTZ_pFXuG^f znuf5AY#%#dNT93pXNec^QXj#MM<(~cn?Ab85wXQJ!iR7C-+c(a@o7mC(1yIf`fcHo zBpR<&nz;`PvHwUT_~av?DcSLZ37M7@UW>VP1JS%3+EC0lZvNeFzr5^szORIceAk|T z4N4be7GI1U9_^6s<-=rpkArPB@%8cT(lV}bSN9PQpN7pdp)@&(3;Yo=rPDFTIseE) z`Y;^J43uvLrp#Xmnp`_rl+3`}z$2|YETT?x97%vM6%b%>|7f156ojKN*vAxX!&AOO z-p`A$gUb32erU@aeQttuL(J|&F|+w@2m`Of}jJbU)zg#+k+jywSrCfDf(82u$`A7@ZQu4R{IsOhy{D!UhalD6*_ z6r-c~|5yP0DMRVMoPJw--|Llqc>VCw7}uo{F@U#wp6zaSBK=*r{P zR-@zEviI!vl#wZYSO8OO@pM$hJLgW9$kjBtZIb4NzqZYcZkeb_zPtL6LNc!BqjWBa z`?df7RQ(of3<7G}(T`|?d2XjtjB zpb+N&XKZ)6?dSLq#f8?@S4W!pgTg<8(0KuXp6p|fi+*pVK@Q)`Xw}E)w}LNCX|&oC{Ezeg4lz@Ate#()-=&&1z-q z9O|Uy;A@NcIi?k z^iLJYN=ns!&!}qRqb10iL&y=lR7{@_BAGh+^kXn!%8FR!y~K~(jqsIcObcj*^K(>& zUDwnwCfyr&DYp4E#_$2UsoTVKr#Qp%3qHBd4#}UPe{TIv%xp~AQEp$o5&$w;37zxR z%MQPSf1{2QP~zI)d3eLY8&KtA1%0gk@oV&g&FjM7*KS`^PlcmrPMNUTm{ImyC+1h4 zIE7;AbETgWzZqbn(|)Oh1dK`4U?o6o`r^Kj1X)p;I!|TQo^$^k`pT~CM8};shW(}h z9T!T9&^ZfSxyYN)MxRp99dY^qh`e@AOA$jf;MlYH@ z=qVnS(x(nMEIAT^vs&^cT)x6DoOmmAyAtz=l6yzN_}F+PJoP4yL>x%5SQW0;*aiE; z6rZ?}yK$$bHGgJmP~4<1%rsmdjTe2kUV+NgYj#i`4Sg!L%SZU?mdYBq!&#wS|JHB4 zifEx9KY78r9zltO<))4oNmglQAER)YL{1v_o{7dHxH#k>M>rS8 znYTp-4kHt|R-B*L_n~Q0%qya0oPU@kt9GBEvDowv)EoF?OGI;aI2rnoCG=c-0^dQ? zhn-tS=ZnI?%r@9BKryNwOsP9dulUwVOEra9S(E^Gsd{3|ug$iD)wcKsB2rAzMk>Yk z7+u8nxJb7MHS%S_O2D#6R2?K|mm=J9L+T7c>SFaUFil(m1JSknS1#-rz(D6;gH|Eg zlA3yDEEG)kjcrw8H0si&?C{beIrw^XQ|)m{9(UR>bo}dpF-fgw;^xNsK|>MJaH7!V zv$?$6@&Oo65nBQ$y&dm=ef)U{kEUPn3Cq#)+AQNVEztnv+MEe|?;IS#aBrXMKDE>c zNkPkJ%B-({{cStuujIyaq<4#drN1VB@-eWW`JYPq)6)i6QMxx>KsA4T#kR5C@sj`f zzicG`7Uem4uM< z9IrItARil%|23)#I{KyA!r|E5A7|BOg&tZ!FyqMB?){}?HDGv+$0GpG{c@%prNETX zme-04n4el+;Yv3hmBvXu4TC*4eVAC1^D;;K178H+(+rD%GG-ojF=sg=M_s27j?CWkdGs@|!VpCR@0&C*YI8xI7l7zrc)JbJ)*mq(sdI?V@%lpA6PKL7qYvxE zYUfE=C_PYS=YD!UzCWq(E`ntPRHOcfL{EbIKM8F6;#%z{zbW~x_!H1n5nTEssjP52 zx=FJ6snF+onZkFb+aL~|3uoYQa}>Sjh@jBBOUIO0Pt)}WJ`AUW{sdUl@Lx$Vj*LsJ zScAp;9RDM2j%%xXe~Kq6*Wx{U_CcvfSk-gLRm_g|cIE3b_Lr-z=SfyTy;I&(I^nzntS`Qk@3ZS&X5wk40@p(P2=;x1!%{Ufhs`h1}23+MpzPS5e$ zh_=_xe~-QrOWK*hUUz{AqrG{x^UBEG*n zSQj(bs+2)28VZYWzNVbdZo;vs@UDp!k100Eks(ao$7H0Km4I%=z#l9yLFz z{Mqx-Muah9MOU!)UioR%$tdUt!~P03ECMOXKOSGKegW19OH;hCUy5JNJOYI=zb7pW zp}wVD`lj_jbXs`#RiFj5;T3r5UiY@kEo9SL664+oXo6>hoI(dQ)1mPtitnEzS<{k_ zD+`!XMaICfaj0b-VYc7BSbm~>bWby=n zuS9Ek4xWP}Z(Z^yc@}RPwsW?&e-|2Cr+|PLV1Q$yv$uo`^bnpyGGJlH)H)63{kXli z90sUoVt;j*^?TBr9BGGSf^zUxN8w629J>YdSUl~w@)D7gLX(`XoX1EXeP4Lo$R8g} z$b8cxbF+fj)P^DxQU95^Y`n?m6>v|oLj(5^FQ#oySlhyXpFO0Bzp||s9J%~@^XoGG zN+^d)a$iwTW+P#S>FkgChqNoZFNg0-hVaWCy)w`&xJ-B5G%)%tUnh3A9du7(fsBCO zX@_H9D+jvT``v3E)@7d%pcIq#gA(%LIrstWD(`8Q8az+wCW3XojyS-FwF7xT1P5bk zJo3)O9|V?fC@cP5g}6_P21-7k|H;MQ!^s!3RDNM3U7MAx%!+>JVJL>&4BfIu94`xg{3Pu4yL!z-HHEirwse4|gdp_pFdx~U znFk8L{SZGC$&_zHY%DDFy~%ojFADW>N6?&X&;tphc{@)3wIk-(zk+YAfwKKRdG#kZ z)YIc;WlSpMX>x3b))T9tg-WAavtKcYVj_thvj@2`+xBkKb42X=8T}qn_8cZ!e00?$ zd06Ubon7~I+<_)g`utJu1%aD_U(uxVW4_(kHRb2?QMTX2B||FI2xh*NU{cS?t#Kpl2=vFB=F*} z!V5#ln#}h(E#b-ot+b?xPTqC@C5Mi3wj*y$lX-*Zl~xqa!EUZ z&53c0TitU6h^4b~VvJE2d|FJKisEAHp0Un22YB+=&A9HKsKYVj)86Ry&OOs+tBZzJ z92H=U;###gC3xp0%IplT(w8ti8g7d%e8c(9=?r`0gVaI{Q4j zjyD_lb%yH@%=|>fO5kz_ZGR=&S^V)=^1p9X!keV=;qzA;|dz?((or?xL#!7EPw+mEJ>fI1Ig7dF(QB+Dr~K$TDZW{Sm#uA=W$!)On#CbAjDtpp|p= zsW>KNJ}$e@>yJc8`o$A#?l}s!;kprQ12U9wWng}Xo2>vX^aljT>L4hzX8jQ2JFRZd>xnJPS*A}wNAHD z6f=f!>78uJIvXE1!1Qx6!=&VI77+DI5jc*Y(&FExD6T+i1T8}lE&AiPb|}%3~A9FP8+x09_Z6U)}W>5pKUvm zEIm?esIoBZhmhmpuc`&YtR8cJAQFIf!DQTyjVjGNCSwYo0EB0e*g=!6b@RSrjvpUL z4J-MD>b3J%_dKQ_o%_%ly-t@1LYQ&+IOEo}$BHRs0-X3V&wX;eao6EU?|mV<$p`F# ziGt;uC&@XF+g}TfZzTZM5ogcG)95Y9GyrvUP|l?dbj3IK7D$DEZ$0^@Z`m{4hn?4+ z(kNQ?@O>CrKfZNQHs80W+iN{Z&^KvcaPpso?ST3tBrLt!9oGKLz`^G3(QgL3ndjJ{ zZI{cx?8OIZAbu~kWH0sCqxW5?}+ex_-CNJODTBsrHb@Mk!tO8sIraxK7?x*>6(WWMM&!zfETn5;V z7)Sn&ixIqXot9c2?^vM>>cEqihdrDfC-y$sIp(_IB6wZKW!7e;-gGup@`w%46~KWv zGJ*bm7a8h{F?;+`_0xJ_wy_j5-8+x4TcDTU?`4XmkA1qZXUwOzO~bSHY!CJ~^vHTJ zOT}|Ki`N9Ew2nhKJ!L!>Pt*xv0rFckWKsm!&8z3U8rL?=ndfQ4BH$94H4YuqVHP&r z2wuPJGu;66w8I(QYX{7n->GM{VR^^gaYl-mYXxXAzCNA-BO}%tmySQg3=`-X$|a;0 zDkj`#zgi-`*&*$`g->c7H;AmDk=2lOf(vr%0xF|V41PAsF+X;(~5%8VHv8d%(>P_mLs_cJ278-vhN-{8%!xAVr+_~Q@MZ15wHCEE*+MD2UVa0HIv_q} z7diJl`>n0Exqx`D{Bf0ATS^t$eS*E8n|41_Odb6E(j;F*4zYXcB>!YIh--SrE`c%I zVF3bPd7w2veJ2S}EYm#K4oW}!`S7MSCry&lBdFo@lvl;%-h%fIEL3RFg-HN z&FC{nlYuSBi1VX%!55!@jZ8GY>=5(_7fOZ0+6*$cA*}Zxh~~pbsDZG@?QU-ycOjh6 z^M^0dk%E8@%zBK=_cx*a7yGSQFKd?rGbuoA#ZJd35Zb%>OYUPi+w}NbKK)cc2^+X^g@&MuMh1p@K_-F{%H_5A{FB0{Xi~nry#$^I?d{fYaXX z(@%#HvT(BOou96>{nXusJ*CcZ1xtyIX_V_Jc~qvap+w+S5Ak!@P#&y3#HZF9W0DG- zju#dAWX6=xsgQ}+9U*|Qq4`II*S-P2goNMDo;x)I_VuzZcUX-x=G_qK%t2GXzQ8wG zJ9D|?RuZh^jh`^2VjvzN`zA(e??u^{o!NHwOsDqrR^*uBe`B!IycnjMd@Po*`F{EA*qG(GpaT^eugIU^USv|ds=#C*yKYcLNxU)# z<7=w%AcCJ|*@o2_9wR%d?qzo}sDa9S@_G;bR(R)6c;?E&Mt0Wkft`3O>yS-D0gTFqU9bjqIc9A{wUjo(>@^W1^1w8RI>kEx6#O7N$Ne-UB8Ut*UNrN8iK@#P27u}9R0!XClRL?`J9W*T+D+;}0K05bGkw;QG6~1=JHXt9-EF)Ch|ch8 zDCvYvH)iMPXRwKJlJJJ9>iFf9X?Ul}izWLK-*M!%F4;O(D-i4{Hy+oJ0g zF)AQ70~r@n$#acP!W~8C&R3XB=Lii;2E?6jroLRzt&SV+JWIo)fWlqp2XZRBiF5^j z9%IDu5Nu$P8xSZ4mpr2lSWS5nyh=)Jx30%tQk!OjTM5sLHEfBm8A~8O+{#sXEAHH3 zJJUyg{qhQk8rT}_B8!)IiE4JlSj|aeqW3|}h02wzR^8D7=Tu|x{}lVU7vRd=Tw%=0 zreMHWGG09Ip5;jY9EJDO{3gdd;N3|0gHLx%`9kmW^Qyft&hWNrHv0qI^B0A7MSI|5 z_(Gd#!qHT7xviu^GDfvJVOa)!p5trz@D3@y5z62k$C2ZOBb~dZMfurzxU4?zylSp) zkez|w5_l%W28kOOpR+ynprk^9`~={Ko5r(U>$jIb?}Dx$v}u59DJ> z(Bf1kO4m~QDY6L-kK~c`=(yS@ub6fR;+K;;H|$g&#eDBBm6hBYR0EqRha$pzDTP8FAN~PV>(lr zxToP{l(Da0aW^+}j|RG=&=a}-`Rvd}KmSG-C*2g0KOFLLgO6HH%9O{CFjSaA&6QTb z?{mHJjY;-{g`8L49hulRs|<0_A{t~o_~KlR9TySANg*bP`(%!tK?S(ZZ36Y=({W${ z3_0P=mAujv)8#vd#IMSJ((al(GO_Ij9*fRw ztAmb7K7-##QlB{nPI#+Fi>T2#^DbpyXfAA-s2idUFqAm#XEGsHL_>`>Zes)nK;DHmqdKn z6EAE~?IH6Feu?2wDrvj2WB%GBH5lCq3$rlE=`!<+tW*`FM*Pb8R_3J=Q=9op=@QVE zy-mt;-J%m(U-q@BOEyZ$J3TC(HiF20ZxuwkpYzjRu5H7xIx+TbMA-z;biVZt{ek7tF3VfW@^!P3VF)u*NVY7y=9*HLWc1HH z`1y>Lzmj$td}sJmj+5Pvh20bzmSGwMzcRAtSP*GAiGmYtC?h4PN-kZ zAm@t>O`0ZzDN?0kW~)lNSh3LrWtKrp}e#{ogi`7)OTT6oUOE(CfU#E~c)G9fjQkLvKpUbRc zCwTHtu67ICs0FP+smZ}_iJxblLMz~yjFYm_5E4X(omXlCCxaHzg)M1O7$#kNG|BG7 zJC}kMip?mp@DnATeLd7Wb_$yX?B1*Iw0JYs4sp4SU}21VC1o=T81nVp*z5Ld|Fet9 zIo`rvPa3CPRsg{a!*RU99HdchR!s8@QL7%<{8O|S zD(0Y%ad(aFk*hGlg{SUAj{Zn9BdadnsAn4M;l4X(U#a=`b^=m?wJblfjyU2Nen>lE zS7aD0Rm@TqYT%Fr%w#bM)iTQ=bJZN5#~3- zznQ(0^vAm23n5-OCjji&vJyABgD1d4$_yC%`Z_&$YF9VyQt>jKUC8bscp>%gf9Ax8 z$B$QXKM+)|{s?-Q5TB=P@3=iiu*>N);u0JC#^{W%_ZLoxz!h!I`vm+1V>sjZ_OTU&OoH zT2&bsK8M5$sz;^pLvM%A%01gCFL7e^Tow|c738`zZmS_;ar}RB4KV#)K z>kn_o0TjPCSt88FMb~rBzHseaQ~iTd*mw`wtOY-``Eq<^*J5avOYU>KW4XTl9-$IDw1uSTjrBiyvY3V72l{~UU=3) zaM`G~^q~lyEWPvRXZa_Z4?0XKv@iJ2!1p1zFgE|c6=gaD9MrMwF&)FM&rx?k0Jwf1 z{$>}xe?%ul=dAvz?$WSYHYaJx?${qW1O0q(0r9^yXfC?xz$M3GZDoEgASyEbB>R(U9sEaxJuYZ{T8%(zPsx_ zM%A<%TVwoO$O}{cAV^RWoUXK-EWMprIP*uf+Rp=Jyi;;} zsX%7qTUM*~(`e@bA}{W&EI5VGO$T4gz}>2z>MIw@T)VPz(q|ud6@nvwGTv}tiCmZS zzo2x6O~wl=Gz`*S`| zg#X3Nm_hy1<@|jXR$7c5Ocwc&3;59a<^P7{@5O0dQc~>9^g>;!d#<*ClM!YsT+QIB#xz7=w9cXOCs+|{aFkJG)MIb-B88>1G;EVNtfaKGMrII z*$v>e;_tthm)CYH(^4j8O{6wvs0r-mTmhAq&e1P*42yn+OSEg**#KpD%m9?4kJJ5% zHxTT3B;!p&u7pG(lKus2CqGaR$`E`UDapNdm!S{{)Yzicb9|Y_<=?R|0(P>(>SFy6 z$DKxu!hs5{XwaC51tjH=~wr+P`ieZ@vQooNMnXm%NGfYYG_h)|sZcde*At&PVsvFKw z%Ej;M4tRan{Q%wSGNqZOsoBDtcT(73EB=W6m9D}_jx^iL;{^mJ#t$F_>vY?Kw;?X= zep|wKgvx)PeadQ6fh9!UHtzCKO-M@4K#fx@7w<}r8;7A`Y3M9>Sniz113Uxt3y4r>GGG& zIF9tud13E4NW$$b;ju&8zTjv%o9mw)XI#`}p+qYmoF9-6k z=8IUZPSb<5{gH`8$}r0x;h5Ck9S9KJ3CAgS3>b#o%lL)becImZ0FHp0VzGXfufrNeSESa>| z7;k1tH~mX&c`98S9eq$DcJ?3luy(qnZ;#5z!o;j#NQ#q`=jy6R0ZCO+Ve9qsqr*wH z2z7k%O^QMfcL8wZYr8S8ku4<}lx#57D(Su@A9h)V;~3O{(7zF#et+kGb?bHD+fx)j zoeBvs8E(Zwf!d6hjuTLpX*g1mrV-N-e5}g+fM1COr>aWT+ZIrCWbLx%o6SwkBssKuJ7#O>`ZnNg5LCc z-g+5vcd70IJpp$HG_B1Yk9Ie2ZmePrJQ4oa8TCYM#d&H{9g@eb=g->VnCS|)6W<-W z2g4W1D5f$g&w)-th9lX7j+H}|MVgD~mh%#-GwoaWs#E{2 z{Wsv26%Ya@62p@-1mFG3nXZYL-Ydjvf@UnGKR+gReO+>SkZ-4D2#2DN8`cf~LV}pl zzKr}riXR=GN~)f?f4v8MBjGEm63h;|8W3L4JSd~NyBl>5$Xl|rCX)lqWHHK^+Hin^ zxj$n{3;7N5k)tYzXF>~Or&BZcwnMx{QVw@QyObL9R|T}B0PF9V9nThSIJQKtK@q&~ z)p$|HIt+tcGbu%A=z$ezc-6n-KIG7(A~maYp|b&Q*}>sdC%sa5&Vp?E?36NQ;;;?l zZ#NdUBN=%?5BTo=y&q~H`axTeS{|U{T$ux9lqw(u@MqsKS0k5lp)sCE2PmimJM&#H zDjW`9OevW#dawEyHUL*WN7&homt*>&eh+7RSVvW^8^?&;3!X!uc#vRy8nHNd`$*thZx@m6z_joa})GJU!M@jLB(C@L*Z1m6E~Z9L<+e+ygY zU-4TRvOIDoV@EA>j%`1rMzvjKq-4f!Z7KK*V)WZj%g?^~l zw(5Gu$HsdJzKaEP7Zo4aQsq5`ullbntcmgG4e2cbmn3e!oxq{@O2MerpKc`RCZ9d*V z0?uaJTVr8S*DIh&{`b*&V%AI4kr4vz_DH=fjcx*)75~%1-7u6EKx5 zp>L96n^;q)Qn z+kMABI$c+LNKq!A-ofI~W!DIo#~3hj&Z?U1MCyHz_Jd2`Yr=-?o)Y+b3G?IBKQtjwia5Kash?S`m$c*!(Mp>&cmL93 zJ>_6k@l4a|fJ|EXQ<1Sw-ZO`(xx7B+md94d_*OF)DPx=mBP>=y+=#_ zhM9Ea{Z+5l81}Hroc*FE2m!q-5k})i${#0dKHYtDLH2A?DT!VtK~pqlZYeSY69st= z&&5N>$e?iuxP3Q^@)YInTH`P@ut-MFtN~nd+C?$5g_bv$u)w_<86?GX)o7#2-r8|_ zMj}6lr~pIw6Keq>T}tG5u&`M?3Ot6=MNgKlAl1+#M^9r!T)UCO<3&cNV{5#DsBNWh zrNM~D1+e1xt~<^YLxcX;CX$_}9u6!~LqV*uXZV|Vo=OE<4SZmm-z!&Xd6VP{U`bBU7zce&ca!Z2BE)E-&|GtlWJ$w#gR5!)2DTCwV zMJ%-T7Nh^kDq1TYC>!%JUj7ysuEbQG)UW^*VUu-J`8X3+bi=?{UQN0^M1hz&avtE6 zuO1mb2DRXnk5-Xx!=ql#b=YaBXdBbmyMggoa8`(XU_~uAIB>PAig@ zFZ-c+CZGlhgA@dn!J4MjYHF?p&LO{@O?sYl1iv%t<-f5fHg9-DfTl{e<>4&UhM*eO zSbZZh`oSfaB{>OT%Sy0gf6o=JKg}Q?kQTVl{-HRq{pnDSjHdIz%F}NzU(<JZ5`075qCNBRojC@c2ip5(&{uQ#Fya51IuuxA$kGMv%K4_wetv zzWz#|P{0FhOI*hH-DXDRHKr@`#osa1C}|Uf!n$da!2QH&-#Ktatkrcfz4QRszmGRzm*`M@;vV+heFEWLLOg+*v3_Gc^oUe`;sh`BR0M0XF;-W02&a2GowJ9lmsE z6Kkct7&|bB&JE$}E%BOgz^jo?cDxIs7UJ z$3Et$9R6~L5FdO6ZSNpbKX;iRCHp6hj+(UL$kH$&RMBYoh|to)_JvvbEmfh;b<-)x z8Ny&wkUQBEKFX3kWbFL!s?&ber#AYvC_;M~e<4Z6DFsrOUMizXNpdN=Cjlm{HtLw6 zeCX8J;~BU^t?3ZFkV-IR0;m?nl)9aFPD+P96y?y$PC6f>)L8tDOc#9g9z^Z$Ecc_P zErPTXdwMO&b)p3)v8B(H4TdHF{a*^{6s|omb6Le{x<`z_bIUy&Mx1*&^yai_J$3iU zuNuyaC>QgCV7!)OihA;oRyNivE+X0Qtr#Q(d>wZ6{#NySE!dbs-TNX|dP_pKeEFWc z1bursv;da5~$NK&}MD``Jm28G4Zzq@2tb3_`LjXQzlZ2ALA{NUUfm4nqtd}+zetrenDj| zoZ#yoO{<{{Yb^A!&MkYz+<#^&SeLvIQl$3h+eq==I9-hJb5+_t0<2l(D-Z0r>3X>< z^l*euI3)4q2*fGLQ$K)7O5!+lQqaficEQLHZ=!X+z3y4WaGsAgjI?DPVl_ zsSA$yvmMCis<*u)B+O&G&D!9ereqfsl9)R;6$o0)N* zV8-m41&otT9pU+?wsv_|&V3P)(oBI2iQ*wLJtUD;;*%?~!M ze+(SJp2zk%#E=>0NB*dJUL;*L#LZ3FDi;#@{>^#jAh0>0ZNfDGj3m0gODXhb+kkFo z=YC6c_-+ndu8-Zw4@jY%IZ&gh@Dd#4<`%dw@ArK)C{pZ?n`TO)vrFpqF9~|D{%j8( zK1Su1&>e=exR(aN$tfyBSImrT0G_|2_Z*W(A?2}1>eyad(-miWGl`$`6d?c-`!x6H zFxvXYiY^7_xXX@>shV_szGRybD^*g9RGoCkR`)*Zw~zR3_&3E9af}!ru?*+fbfjD& zVb%Ki&<%x;00ke#V%e|@MvuBg0m$gk`1jbBXb4J4-BySeaO*C9i=FVigxK<$5Y3Xj zBQJjV6^gl~*(v$Szkb#Y%vr^z@cxapu!r-3IEdrvhh^g+||Q2S}{6SdK5 zQ{dDz>6|}q5zbd7$QtE4^ibLuM61HDzCSa!DPm;3h&O0uF5x+;nE(=IPl8%gFGga_ z2B6ak7+Por{@}3n=}mW8C)Ki9mleD9m>QF@4Z;!(erA zMJ_Y;k8Z-DgXx}X>GU(OPPLEf->%=gP^>;pJ&|40g%LHoL+DsO`?1tl=9hNrs5nAj3Y#HkTl3~3xvJm=ROw=nU$nn!P*Z$CCcTn-2@BeO z3Az#20SvH5*XZ3mK8+}O(5 zR@u$yE$RBnmj3dDpkm$iKCAXL^6TzTatm)&AG72W4UKh;Vym^&%Xd#zA0-)v85~kg z8bwP{M-Qmn$R}w7MnVtP1-TWVEbI?0y?HZfokFPuv}G^zKMmg%2mPb+WCPYqp#$FA zuvlL)Zz#Uf;2?HRTnO@9W>Z=ha*fkddT^>a=#qa{qqiGMN~0WSM9}_hk?GiW)0R6g z5S{SV+Ii)D*pZ@$lxgh;=6Bc2ADuNe5TIR|>w`$Kc@pPd-z{vM2jYUKs&!RF6K6`Kjj zUxHi7f39=y!hTCNJ_+JXnqb2m5i)&OqVSar|JHEj98MB?HQ(r&=Xrd$`A30VRQs*; z(T&xY{<~p*t1jXrNR_~9Ov_Q+$MWXBM1yxAC@MUSr*T<++2@Yeey)fDb!Hl9?mjBo zT)7^>eGu@|q{enIl)>>EDt>c;ho<@HQTz+eCG&$j|1F;(e?(33 zC8Ez5uRHb#p(U^KvGvUWR9wLg@$NA!bSM2`8f_XqpGIStm{NcWM;_MJSIiPx{ZxS4K zz84T8N1429@wY_e!%9=}u$C`^tVucLLU!U$qo=fIYy5*rAyBp_$z$?K0v zxpEYwO|stVA(n7AfZUExa8bvnHTD%xZvW`Uj>s!=2)gX-^>R=Y&GQ$>i%KI7x4e@^ z(|DYFGrzg*Ww+kgB7wufirbwVE0)&-!y&H(6@21CpS-4BTiLvXt7s$CrP0{v8_G?~ zt<47Ks$KB3k1b>#33+@Wn#Z?!b=-h}a%|@?d2r)hy2ss3O=_jqNv%jf)=c?LKu*t- zirR;rO{mv*-1^o+VqG~rrf!}-&VCNyV2qY zG>!U)U1m)o-HpmC;l@SLWRmv&8#bbb6ebfd>S@JYdDVBpHuH?Tck_6XrNbE$?+XZq z3WoF7kGY^#I_V4Vh7;i`u?ian0`b}#Xyzjh4*A$*d%!)2lCx>|Z(fT@qDH*{QnUbI zH`eJHye=YYxM9fBN|(VOFAa6Bua34)ehuA+t&8ZU;e|9q-7n$l_g}h%IH${14jSlo}?HOD&Va~0pHX^bl4ui~qdcq{eO#`v!WRY7siLD_Uy z?z{G!H&38AU1kTu4q5SX(!vN$l)?sPNB53(T8?J$&x6UV$FgA4`@J-}x&zU4&;EeI zvsc$SqR4gMi)#|z!TV3&a+fPEHS;{Yz2SAQb4BW z8uxj6xuJb~gt=E|h?|Iqk z2kd?NixFy_l*W<8wAg$ShNaw{s^QlA!^uY$g7J_j6K@;5wXjvr2mZ*5_#kpQStg2fS#bmLIQ)u*(buGi8WMgFf8mDX?`*&v6?7be}>ls^ZxC^URRzs=A;1+$h;2 zG3_CLaxOYo?OR=nMNNKaen>P=AR@k~_3S=xStS%z)JdG}Q;Rfz1q& znQS;!0o;~orK(l&m;K_V^Lbq%$+k|xF89IvL2Xiv$@hu3w4S0$u+IrHRR4_ekS)jT zMJUA^-t{Wo^|q|i7fqsul(m)55p69SE*J51v%aE#XD^=HCjNFiggGQZpI-d?`Z=7N z6-?*_^4_4pw?Q}cANl_9uOZXwlH`uKc?%=ld{6pVi9D!}GO4%(qj&IDK3#hvqBzf_ z6b0Y7&}K4F0@%$TWqLc%C4*=B-%5PEMdx7P+t3XYZp$0nk1s9GZoyA}7AQmQhhcy0 z%xvr%pkUMqYx+3}GCr_ExGD{mQBv`W%btJ;>sRh&GFxpf+J-?bJrItuF4Y#6%Uc z->0{LPrkT9NoNJpYk|fI;aB0cn#xJ~J!1q79Gd z4YPd^^k;bI+Ijg~s|*^Zc80?qel_RuMPsL-FIbNHW>V*gsAc0w3D=o>_Zx*zv}v-w zG4Flu${GYaWlb7?4~g1{EGF;P0(o<_l%#OUCBbq#ofQ>cx=*D-DNoUW&^3(<>=Nfo zHL)N6C8}c3BoXbbE~3sYcWS154s{)4uc*=pYqk60#Wm^73efc@RsOnwedwcVDoujd z0D(y4c-BdGXgm0w){ihj5`&(zut`I=9d-MqK)njtkdiD6pESkP57aoyuG;+igl58V zVLJbg&oRGu=(47DPiBh6KZ5X*PP5L~#KgWJd1I{Q$hvYwA=`+?-JRgKrLFc5)RCvQ zk%57iV;%3*AhRJ3_n{O^hnZzrHAc-=aI?ZHA>FGd^{t5AiM!;pmmwEktI>>GC;|G| zkmS*_o@DZ2`EQA;GhtS+naT5kpSwj<)mT1B)LT{ioA}W^WT@U#agcC_(jfGg_?kTZqs;jcTgZP;!4|D*S{H0}J7l<-_ zIl*9#+jmA6#~IPVl-=V@p0(p*Re$e3hSUw2x$?bY+STvIx_n;o^A@f!aly=<7Le3F z%^^jPYZ?eG$oR`22Z|C_RJgB2F|Cr95&>I%n%^mn!={K>S#?I*oemYmek4CzB2Qop zz(G{sKW2D(=V7TZ z^D(GbYRHEZBP=;I1-{pS9ACf9B(*zzgNBQ&*sS7+_J+85KcY@?3=2m=vwB0m!MWPQ zn0+2l-qH~M7M8Ku!F|8hqb5;Ml^R8R`kRj#wHii8J7SLCPv}(S_!UilQ!`a=&|-b( zj6=`G2?wB4}cE2?ckt|(LV@b8}!IB## z^|XK9ZiD&b4U9HDWBpQ+VV<*cvC|{1k=`R8c(S@yjE1R8hI|@ZDk!vMR=GPl~qUp zbHh~%^v8l?OS1UxLu>LaTCP@(-)vseo2yI#e^spMk6g`NHBCyfrn)UOp-af*LU z0;1+-mscyDT!71!v07JFX$< zW&yY=<5b)txe(S{@m20~PXE2&RPd_7z1vEr4C_I2o=Nto*Akw5=~XNzc;zdq@95G_ z+eU`8T|0lJsLmDo39BR9gy;_0AHEX*@kk(@v6jJatBt1?s^a(FU!5KIj(yhLzs~I_ z_Ur;SJe`8YtEWe{Z@;Z_V{QorBX_L+(R+gvTi)NQWRTh2v2eOlW7P-nUrI&qgB@MRZv`O1<+ zJ0+&aaZ^;ywyt>@O*WoJ?jV6S8IugCMFdF5efcp!!-~H^RCUATOpjGS<~hf$>mJRu zCaXX)9{>Iixe&Xa@xtzWD^do%W$o?y=gDsKsDs;p(?>(~TeFAKv)*ib*!5AVgN<6i zf5{R7Hxcc?0dO*JLBHqMawu)1B0{qZT#8#JHz1h5$!!w)`15BdR0|@zWL85w)6aXB z&dv~C7Y4wtzd>#1oC=06bELfeir#x;Rb^wp0-lmwb?OwB?U7*S)NG>dZ1Y7hU`_E* zd@Spf_3DvmrvwuIC(Pj~>_@Ec_|o;a^;VhFoav$e`^nC_i zzFy@nSO8Kt#ksomqICl{fs<%1Mx)b=@kL*XTR);!CIwD#2QU;9`$qpB6s9<}p8l%F zu8oQpwMvWw67SaXV*{E-UP8`IYhR|z>A8koMsSzDkk)njR9zaKeVIHCn7)>m@}-&N zfM6$l-Tl!F9I57RL=#r^wJ+z*19oj9TzfcJH}EgsL7-^S3jiGFak%i7?}rtwibS05 zyR7oP6_xnDyI zKO#42yFWz<)=t*4sjv#zcDHmol5Y6hFVr*j$&2;&bslcYXT3Q%9lY#nhejr1df)w$#lcSlGYFO>iS2wW0CGhGV7QgnkyF zn#+TZa_YSZ2T#K-b2h_AItI*ab&sEv`8;oDMo6g+Jo!UH5^k{8Lq{B~Vr~5X`eu*n z%WHMpXMS7wnYTji^orqCKSQ2Au*;(tvo*=EZ7yM?_6m@E%uO%L{*AncMTC%0PjBb( zKkw~qYy1NO)P2w^L%cj!!q(x#;}_6Z5LXr>8B5(c=!XUC?{*#GxTSGW$kFP%xtQAu1bu;kxq<%cvS+eEmwt4&PPe4E7oj( zP$Z>ZAZV4wXDhN8IK|CR^H!9L&_T9+e+i)rBo9D!0r`5{ebJ4cL@hij)&UNCMag%E zQC8!!GSlEf#V3ZKZ}^Wtvbx>g!aVJ@bA;#`&xDdGPofH~&*v+@aVKPp&|P~ypZl_CiBX`xgEqs$pGuW%tCSt&nqlX>LapSD{KUMs2tNnA|J79_uBHZ#U>U)OiprrgIx@e6vSFo>tt%(LQWc5+R9ocP7cSLpCAdpTq9XxBh$KW)UzU@iZ_m%qZWt1SxsR{r1HO<}*2Qme zzyD(9eG85&;zO2uLizOpgE)ZZ^W(lKCT;}uI(siqA*`+|nJpcYRSqvn46wO?pb>~D z2V1?w7C=1C=>O9Lvkj%@ra-;DL9Z#5%i=)50eWE;J{r@=Q1m6sAjA(GP||P z{6&W`KKQA$$@|Os(ra@QdJ-u2I_UJX%lq+iA0>4TQh`=%H0O`bL@(u`xK4%7Lh(oX zdr6pNBTG7|BKO*(KQWIwu>R?0t+DIF1?~F7Y4v5Fecnx3UckuG8-ER>PRO9zIa+H^ z-2j0Hxb4@sZNQz;G%x9<-x>e$qA}tKQ@uNCsW`wYG3aGSq1eX0dz6i1FSuoWD>c?F zwztZ}8IG>^p>%(ZHQR~BVdbfBWWhaOhy1UX3SGYJ&m>1;6M4quwi(;HwrBCH0z>iL zA@77ROuJt`@_Qq9?KYf9zlwY^1_uyg*_N7An!YOISk{%vP zfAG!nn!1O*FOmD&-7Gm!fQbgp+KO|F{R7%lecQv{eQWkSUM4oK(2{LB`5Ea-dM)Wj zDFthd2ooIK)h~5mma!I~W?Up@y6Rw=ke#^4p=|O&Ia%ys4Eit_o=b+rG4G?q>RwJG zzGGh-c!>995ud)+%C1$Y2oO{EwTFFf=AJ;%98UA`VgtWN|(6jt3 zx4-NaXFqoXmBbnluF@0Bgri&)0OYrypR@hzO)!b-$^EHG?e2jj12Se~8Or~YEB{Fw3D4`nfplFK~Ze$vu z7-`h7gUZ8?b3vPt)_H7s zv2Lu*_n&?Ty*m2?#wf^*nG#TNYz!ZmcG@}NHDaDvJZIpgLnnP&+dq|ELR;t}RRXAk z3>)kt+dCH<|(B6WPQf1iA zHC&83MmR_ywzlJEUCk*I->>mrkyyF2qSi?1Y;Y$|_P88C>X@!9bz4hWUXC1I4cq6N z3YKtrL=up4+r{}m02o2%z6V&W_vvTy?6c2cqJoR{hU;}ro8qLo?YHZM-AfqkXB0;S zODezet%0Wh*It7@P)B@#4I#r(M{sgtBkBY9O>cvRKf7>}3)mRN_3rNp7#HE>)6cVi zmbUF}LFODxa`6hl{+#T=f=BMg>J0gA!O4|u>lM>a*mP@nldAD)*i z`{&t6V@&n{p5OgXFj4i=OPuV%L=|{%*QT8;O8RrMh~EdwKMeVZHQPoTZd5SANlwa3 z|H|l}y7u@g6{^Qt%+K&ma;c`Zh8H|4*8d{Wk7T1oDH^2$;&8yXUOma9?VsDDME-Md zpel$DM|8tdAb%tqEfv#8sK@tm=kFyVKO|xK(<%ro|1$4ac>-fl&%HfGmTT5tX8(XO z)AGijC|M=sKLnFDI+0JMfd)^uJMFa7EV6o)t$!Zt{}@8tv1=$}+ z4a3HTz99XrKX6Cv3udr6l&|4(U+><^R~_`TPRF9DFyO?N2}U=wouzjW=lS_rM^w+O2_>Cf-g zU3Z~>9%8b8-X-+Ui}gSAr(OxZ)_9vs2G!L=Y9(>U|=Go}4nHYTotKo;@A;`vv@kiP8yn(=o6A42S9 zSUkSxdw=uW$8iMajyhOO-}AySj(~3szqgKk>a?k-=Y~kHUbo28PdzR7-g}SKZ&14Z z&qe&azKVTdQYUTy{#qY6jJYIylKy(@hw%;iBT9McuS_AFi%RZedc)Mickv>Of$-Go zo3R*#{dXGDpNa`&_TPK;x>cUS7|6Z%-Yxa(H_-1udF8)4(q9=+4zl!fz5Ba7`i#c* zAK_^C^i!O{kG>{hO3CcmlN!k4J6(TQMBic}a(VQ-?SqEYXNn&Rm)s3E7(1S6EJP7# z`~SRigB(fwpYuR}#;*Q_)PKB+FXs6ucC;v^$Zw=eOV#ABi}r`=l`=ynF1mVo@K@4l z<7RUB?rmkQ)=f0n&{jYF$U5jpdB_V~(H+S~i$;s#5eY5k>+wC^{k>S}U&Z`~7QVAX z8-xOHmZ%x)CQ=a0xioNnI%F;EEK#A{LqiddtveXSfQQjgFMXuoDSgA`8FL}@frNY$ zYJD_Rx#LUqOn#%nVN?2fX@wm+ZbtK{YZ~$i_%++CNr!szVHg6QAU`M6S6`zQCh|up zz}RtPl|JJIaU%-jiuWxWvQ&O$kNz}>$S|ik5(Cd1G^b47LEEc<@N+ud>Q20Sk`xm_I*+DI|6{Hj`$nG?QR*!DN=m z6_;JEG(0^w=3VU3kiIcMtZq`f^N+8a8VjRu7BAKbiTu0{K_x|md01-YdiQsAr@!(O z=#z%a;QTvXe@AJ82jQ8AQvDOgHfPS@Wtc1r(e?2f_xDmhGa z$(S*MBNrrxRlkA>4a~6N0|)nU$D*ZCTLe2SdBlFT(g(7dlAj%5>S6%~Tb^WR>aEGL z=XR@w34QF)M_8(V!UnCHs_2Gi%xl4SY01ykt{3;b2opquapJ@z<6U_olX+d8yjF*` zaQ(-zdXfltAk$Qs{xz!q>izzzFnBB#{m-2zn9UNo^%lvG#a@o2SL@O>E90n z#pj-T9)c+sf>|FL0KDu=Pj60`FF+b`fY2H1i%EKoYiKQcuDRB{Bl6yRAIOk*2IHD2 zEc8-HG8bad3OL?_+a0F?48g#Sg#VDf*;Ae;ViC>H6D30TnzH4<6N({$GDH9VCblr^R8> zuhaGS!s!12i<~)8L-W~ZHfXgp^-o1ERr=6Fh4gXyi%gn~^yD{{smTZ-{lPoyKe{|W zs3rI*D0cMZuwdVy=wl|r!&xDHRS*fLF#>(Wb#H$g_6%O>;$bu_l+P4fRp=9z3Yb4H zFHfP|ynxA@J+@_kBfBrz6l4@4{UQx4Dt%H{{ECT!3nl@WB?2q@CJ~;dvY<}x#wo`C`d+>{lD99zg8I+!8{>o(2zkG?C&K#o_<>H#zaL*-zXV3p8hj`nk6S3k40~Np$|=H>swMj>f@=W zpV9~icTXmJ0wT|U!*JqHx*+rPGs1$$Fv&%5o>Ikc^O*?ifJrW=@A8v~>C~R89}>X7 zv1RR2!Z}@k(@$du55<6o(XddS1lrpdjbc*vLQI&Pi|aIRzPXo8m}sKr9)eSR zdfj4+M(@FdNdrun&;+p~eNMvbWDl>rB+*0#5djF@STEM^)1Qr34c2gWdReM zL4G14y0K%%V!`9n>aP===p*ueZW`CSztxirRG@#2XMp`@4`pgmiCz z8``SVkF)|`#!cWU(SHd3e@;J?KV%M_oMXztK5zc7(x6FmY0$Wt9_P$&LUWvUdRU(jBMe4h{92 z(0&-FX-sw%|HI0w?orS-kKG{4#7L7MJhd zb$9fG#yES5(a;vk-w$%XhXucbvAKY!3MeJd{{+R4`ucVL0yzk$Qw&94nMS>Xh4u!G zte#Zf@US!{fEmKLG2JMnc_GkT!O!V?1iU@ICyT#5fLh_zhI^q8JRs=yvWOxZeG3zJ>HP4Rf0f2u%P%;drit;lk~%AiOQoYGoh%k5`Kh8N(psJqc1!>fy~#R#Atwu z8xMVZaKYOeYYKrVmU$CDVk(b5+Ko=X309DH>eLCxec;z=dp)v*?(OM^9cbIbkE>tZ zjNybDjCF)XTDDxxS%z=<^y}C2&Jd1_z_tp#FHkXs2Y5=QRo08-PsCCBrW-sV|5!A- zwU7q=t-9K3<;!m>R>JkYlLo)}Xwdxp^Bnb8+<2@8`L@_{i>R;v{`bGyktEUg-+NDZ zOfK!UYuAz;J9kEd0}Xif!LGaM(=ovOuDa*q!}7{2ufT5ncESDM_0Sn{!;f_;Kf#oT zVDa>Fc-OzV^Vop760Dj2GZD>?eFYlz&jQUeK1tC6m3HJI0z^wjA3zabMxTJ?=Dz^X zR}vL27BTZFMV}`MxM4$7VfYM4jy?M57@v*4?Y7@u1`HS=lW;T=8D)g^NS-iU;%yV_u1zJ z%UeYrxZnH;6F77E=~G<)(yy#Z=ra&b`jj?{C%l-7k1suE*Ut(|`E6 zfWE_LyeuC53{PsSy8P3zlK1GNj+PBJSYIB)J!exUPh!$&X)w&j=Z`af#0IJNF`sa+1ex^CWRBJU$nJK6SPT`i~?FEG6jk7-^DV5PpL(p?} z43p2s@!QKlN0O8YczOCWYGup+|!`o>644)Fc5tQ z?xmWJ_QX#}SC*r*&pJ~F9{wB47Hl(wnQ7%5v{ zrm~s<)s_AypKNJw+Q`XrhjvpaN3MU>>QZk1{=rXi z18-k^_C=JV&{87(1Hc!^<8Ew0bCTC;GXFWzqBrXwa;v&t3iO#zE2NKdqxoOU1{Cp@mmYWq<2sDa>SLZ_L1IF_o!ON?E6jgkbpZ*afTB=At z=6}$DeJ+15#s$jT? zlRMlwxm&9L@AT*XGFQ$!|9lzv+N&Nl_P@Qn{N;$l^|T?K#3z(8!t@UY)_$0%IN~oy zxKGjB^#3K={F|Wd_7E18I(-60xu}=m!4sS|#R8Y_zO|Fm3J_}kF#5a}vO6YwY^N&x z@bCI!pSvL^xg4(sI;JE~5F&j4oIarxB4f@YhBqNCP~+gCZx2lz^Rf7W6E!#9coW)n z3p8FS`TO>L2s%7GroUnj^yd->P8rfui_|Y65NVsm0ZMK~l&gOvD$2U&=R|eLezo{xyE%So3myJ9SR zWuqJN-^u6;a#4+&4L96S9`FBX)*ryPpe0Vv@cch&MkfJ178^kxh)8DJd_)KU>r?ezkWmXfjKD}Y9hb{F!d{ZB4vbV4UQA1lWn_lb6p7NUuJkAVh3a3KD5Os&=)=dt>Ai++HeIosf8&H4#*rU7O)=2xS>669(nwVKy|okFrUbmsz-yf@$+ zE>i*&j8rig1w_H(gq}mDZ-eZN}73eX5 zl`Hky2*`a5*qQ7*sYDg%Ju2bl2KtxFe$gJ)(C#?u)3U{b4rFXV`! zhBu3ZHF_izi5Z}+yoBBP;k!t&>ZRb6UoCh5k+!VdZSR^$e<$hpk3&a=k`q z6FqYl!NCc97F>upbCpj2lH`F*abZ#hEiVPhqmh-{X=f3het{2&8Ngu|rQUspNr*hk zVh$4ksaY)gK_cQ)eyUr3;wys_J7yN-?6|Z4PQng-9J#++8(C#Vtd33^|0biS6I>9~ zP-PMmCUY%<{uAaDcNt!>?+*o|$00qmV_-ZvRREK?zZf@~ynO!U=NN?PCOlGY|3B{^ zL(JlGWdzlhIRm`I!MO zR6Z;xc}&FScoodaCzYVNU zP{qwg=3#J z_Y)SHoV<+1ckB$CxOr_;J%PHSVPj6To8djvJ7EFKmRMvtZtOVqWJmneQ%^6rW`|5A zO-aDP1Bf|s%>j&3#&l^zncCa6*}c+t}t?(coS90DC`Y&?#G4t=*i{dnZs9XPS(nyardn4q!8o8B~3K#w(i}X^H$RB(dkC2O#{sRZRqKhvB zO4%K!T=mC*M`N^coe8G>E<5ic9g+6q$B$F~Zbsd9DhAKeBW*LD%lZD9piU~hy)0#c zYRjBOaBzZ<1!oiql)1{GA21cZey4A`%oMJ+%&F?q_CFNt)#h9*!sgN8&kvfaNA4fm zrM0xct}>csq5Yek{FRrt2fr{cVbpN;N^bMotROVMESV`lQ31B%`ZNbS!Uc8#o4HtP`gkyL& z$6~zs^XBP-{WH%z1G}=nQc|PvUB2q-tK`ZnuhM*UJjO|yG{LtJ_+G-*hwtzThF~td z@B&@D$Tz!ATnWwOV5hIr`|MuuyIj`5uGFgD}B=KX&~8 z@(UJ4fc41}CMqnUy`IvT2xI%s&E8n>$ij{i2I5)WwO^2d@x1^DOA&tEn1q6efVKzV>Q-`>zamB05;WC!V_C z5tBW%(~lc`Hr`-E83J8>jH?7}%*4|PAOIxo^2$H!UpE@d5W82=+wmGk2CB_bFfQzL z`hrI~xM*~L=#XP<4`O%v(07rp@F#fx$NI~qqc78=vXyk8|1Vry#Uq{Fu#V=U(+$zi z{bu^t+CIG=$MkYx1EVxT|LV%Cu0-3cw(Ps#{>np&k35m0Y0+q_7kvUvXX<6#+w5jh z)Bhxvg=B(r15ET?gp(jR-pErQ-$B}+h3lfG@B~8`9-Ym0J+J#J-&COgC<9|-LB|TM zq!E6dPIxubg=&i7t_|8~2VSNJap0$R^CoaK%tLlo9<^w^5UjZWN0dJPsdm5^d(I?d zncwg9P0>;RN&O#`rn+KE=$8 z?!5a>^`uX~hx^G{ShSwPJ0ebIX7xi}4(V}7fHP=r#PYr%eRju2IgT5#uj=^NAJ}=< zU1UovbRIWuJfQM=>PprhVEMEB7nx$@V7Nd%?F$;-48ZfM`k07_nUghIx0Z`BiLd68 zoV@YI8|qIQyiTAs!p&qDHF_oHUk0L|Oxe{S`W~29;9_R)qtm}Fz6rZP?kze}?s2LQ z>jPUq$XR81$_(rbz^hnphat4r0`yb2c{)ghP+Y-xBqr4V@|VA8+13n$I7TYNcrg0F zUD*dNHp(pXpV1>xPPFqX1onYF{~dQc`oI`R@DuBP#R@6*fv?6%lj-+|8!~8!Z#8V{ z@=dhQT(@>zxdPW0(pVpOLyT{93fjs9E-(Efgz_Hv4ieNxyMfmk*cgaG zl6NjXdzoV(@KWA!9Rjbf$&6{fI@6!TY);Dbp)6(ui)^0qQ94ae*1v_u;ouAT4LT?y z0A{Ymq)!qVI}v;I!bysG{4#n5lDV0$GJY4JEw=z44{qF@k1>?_INfalHqPmLKH9|a z%=XZ zMvd0R*Z;FmJuUm-q#>6-q7M3qZU+edWr6`XXz(D}daJFoCJRDPR6clO}2-jv26hyY^bK7$WC2U{JJt zs72o?eQmu3`7DNEY1g)`S;F(H(0Daa!xX6=A$%UK11i zKZoP>+3|W6EBaW$Py)M;EiXJmoek-sL*G?IFFgOEe(ZUyK9i90*R4}Wj|ws#O#b%l zsKFpuA6>idi5+`q!IAO7a2nT`xzcB%{{TdK#K2hgPT|epCH4@5%jg#9UojEV!nJ}8 zF4CxM{wD@ih`#Yq!Rc^r%As!x(zwQ_#hEjEHg@|SYYsk(&&AkedMj!%*MM9O9iD4fh;&Z{ln!!Fhf-R}njn;Z1r<^G`bnE@YP6h!OCq zM*2%aey!6nY@`(fr6K=Tg7oNE?3j68kHBn+Gso)TZ0NawSwi}+g#0`f@Kp@>6d1_D z`xw_+Yi(HpJDT!nn0)>9G~6TkseFVTf-Ha9GC`zcaLoP|TXw|VlHTDlOq#Bhpl@(# z9M_)b5QM4KbnE4sF*L5~KY=w=7FNUb11Y=~lm2thE2Iz6DgYUM z8#!?KXX9JR-8?SCNhhCFFhF3qa&q!1f7gyKb;w$2=+L1$k?-^g=ih%9pcXpcefr!l z2m46|3XG!)>(|2o;^j6tL4+`NRH9>Z3=-N}Rx-Fl;EvmGE11Oh7|n6j&P5koD1SZb zD2+_Hp2oEZ#~ph#+?m2Gr$7RDQ6Cpzw_G-GGJ?|*j-_){Xrepm&-4|A`D^}{OGV7 zK~aFlEZlMX<(FSXCQm$(Y1{`zWPs`(GZmn!|JCsT{@@_YHSTFz@I=*9|*S~$q3?QaYkYH{-V4{tr$g8dCn z=x>7s#JUg$6Z%vvKLAN$JNCpY1XrwotT%+g1NgGIj*VevER0VJ^ZHLGbp~MbgqoZ# z^5Us9e)SRI+i{WCE<0gzV@nw~VVug_>y}%?BCnRxVXd`rtog-?;d>natP}Z43hjam zE+|@DL;B=6FL&N^k9_>`$Lb;-PME)l zRsYl{OzqlrPzT(L!T!S>h1f(zH+lgX2s7S1{fWbQGm3esgGqHx_V|Pe zbJnp~tg+AD(BZ(BP2V7KN0APR>hUL@;7d`hzFMn-#jL#R{%S0e<}vtea}lpj_H49? zY`fjIi8c&H;WKGds0ftP%3O7&|3H)jroUmv1Sk4`k72<=LvDHqdgC}N7d*0mq1TxL zSSR|J_SnH}#?wEOfH+;hd>7~+As%B6?o|I7zY#BOo~F8I6+6TUu)ul@jL z>{Xgk?h7Gz)(2)khzsY{0Jv6C|9btyZ=K2Q*bJOc<1+9DS9#^15NZFP8youQMhN0$ zAb!x7VY2$~=Q%$YP2@X$LMgaDxETC?^nvX}M8Jgn$CL9eyZmx|ANkz_P6TmRdx)*+ zGzW|AzWn@4rB8WJKI!k!U11-Rw0LTQPUHtZZG59sz-3$$NoCTX_}q~9KKJ*LgL@oI z76b3NUZJ4>PUxI&#PE3jCgti(f6xCh|2ZLjyZ$pz(`AwT7?b#B#xFL}jEQi}M`L1| z6W-`^adMiI(mMH#aRZ!?!%1tlDe2EsAUH{l7bdVdQ61pn!9{YI=f>o>;iDVDy1S+ELUWRjWd_E@N6^8VH8^9bZr=3?_&qw&L77*3O}Q0Xt@XIVMvZ;mpsh4Fa>k!k0St7`pbscFwFc`Yp$C;i7yoM;&Y@`p^V zZ|{FwVDbzsu>4sJEB|y)96vFc$WM3;(Fw1G*9wo8KQI3b@fZe2AA@EH>uIZ@L#P59 z2QEL7(@;bNjxj-PVG+V91Oi4>2|@#Q#5Q0f9)`m~Wzm~TP*nP`Fl990iScNmaaJ8( zy)?uGR4X11Pd5qKi!OCkK)+qPwgwh(@NUY#W5Cx0ElrZ{LVYxMWP2_{c_uhY3kHmc*tG})NkjIg( zouny_yEKEyfNS60_;>lojvH5w{H|kVCCEpeP*4wpolbh=!58JtjAwZIz~7@Q02~#h ze?~)*Yt#%uLf?r6vZeIzAcn;7DXt3XI~;0fS`|NbxnMOs?}MY^Jzy`r^pZUBA9(;wGaM|YkbdE^nOTzz+6Dfxj|D^QS7azThd&|JnxK`@xHFoy>-!Aki( z{mD4vMOZkhJ^iT&8W15huAc66{Hx}tKID}gfOy6Xd;m@TbC)MnZ1oQTXc9nm0*)D0 zGc9oyniM*<8%O2SCz+St9VVEPlw%w0C7+)3e!sstq`N2H&tUBaN^$&~cKZn)Pf`RENvSQ1PWX5!xgpHbx zLb;{l{1@Yq1}K>uPmiwJgQLRoXYs!M_8U0|JAxTi5y8T{22aOILi})JVBGq_qNn}z052uJ8#sJ8VJlnV_%LMX_M_2Hr zpZ_bH|5)r^8ahlGK?lb)Ov}m7C?_J_&VEb2ZM-rCmJOy(l79n!)}*N@Cp=P)`k?aQ zxy;KS_*6zctB`M$mQerj$bJ4!J?Iazf9F;zvpx`&xxGyK$NZoGiu55IKbdDQf(8}e zK?n6PyXuM7ydIUtqsTd^N5hHZA7{>x<(t@DRpgY&{=j>wsaK^fas!S4WxK$SMb7xi z=o6aoybn43{_qG+i;Frh@~xnBCKk`*#zjWIP<|d+&kj?FEbZI2Cz4=Huf*G;`KtCU zAR_2_DtEcT1nerGibxN&h}?Mnb=f3-0!(>%4Cv|@+@3snGDBp_lqq;3GUEoMyx)D; z2iScYzQ6nJ_h;;ESd-u|2AI-6;J^druhM67{&ELDk2^F>#92sR z5mlo8V@x7Z%p|9l;K>(_`ce00YM~QnCQT1S)+}dG%+A+eM;#ul?FIY(x;%ND#IOl; zxO&v#WYr<#5+*0(G;Kp3hIe}|>Ve7fo4K0v?sPYR^R!i_gZw!1rXc-YNau!vk+PCj z%i^U!cPS5N+7l}T-*3MIbk~csY1BNAiyU~sK{kLzh7e`+=rPKQF%)72W{c&gn?~D3 z$7SE2Wug63+4|qU)6^AyOtT^b-yo%H&H+6C1;r(RzXkbCQj^*1f0Lhi!-U~Z<0mmO z`SLqoN$vWLW!^7ghl8rX<);JK68Y6ph4OC-b_AX`le7rd4j9Dl&|xjz6-lc1-+#Z% z!Jy{3=bj_H)+8nrnF;iV5C2#$yX;auf>Xm0)U4Ucz!U0gCmRJQpMI(QZ>%kX{K#*O z*mgP&Z71fv|7ZD+>8D21>9;1o@tfrbq(46wOhcIY*gC|kDqbFn-IynCoYnstN*K>$ zu$nWmcv#5vj1#P+m-T-I6X-*6;?YVLH{u~U$|m)S{tu7Ced7(}v4G_K^UpJQJnPr5 zi%Ff^!=#QQF#4;k(j51v`{V`}x6ML3?O81F!=#QHEC0b^a;Px$0j?6||FX+34dA4O zo7^6_U$0#>&y4<>t=Ev(Uw=bJU}3qPoZ%3K^4o$p4EA%eo<|30eN5V1e%WPl*B*5? zO2?gu#{gZs=bv}JaSabnaNLd)j*1t(QFYB$X(kuo)B-MgBMx2gh)J&Fj-fsPK(zL- zqD(u!zvrKK0VeTJVqh?Z^fihyWB$aY&hXSoie3q5`hWAy^swNOp@g4BmOc6raaZlHRx=kVAp?cH*{PWLM^kkua159vY z!J{qMq<0_+8y7rYpc9-}@Ce}9C=1U#>kA$Otjj?swUx+UR{x1~0aW9dA_C=%zO$0m zH%23K19ma!8-7RNy!}pJlai01ih}%y{92%Yum6!j_T8bC9I|U0X|XbPxl*laT%V?A zcV+2Tq<>6Abw|B+?X_HdWaaPSM;?_6F?npp=uetBNl#EwJb3VAipD&dHlX|Em-+I+ z2jRpF@Vq}3*0)Ce8jX_;5e__%{eh!nu-2QU?>rf6{UK zw3pO>Lcf&!^_OcQM_=|1CaEzQT^AG7V7M-3q-$gHniJi1&|cF?Z%k;{rk#`2oZP0H zae`g_F?n4Jesz6P8wILCeZpe$yLLUBRIi6PpkeVb*{%3MM`v^H(@$h7cc#h z*7cCcYK#ZxMO~!f-kBx%6_$T`F5k3?LPcH~H?j;z73v?0=Lo!RjL0J1RDMX}03l*< zIvO&WMliwcsb$@x15Yh5OK6Q7{fPn0z-|79Im7RO!6!KBGT!VE6^Am)_AI7P7^9Ea zNrZ*b&nCJjplT=CE0bj!aOn}#8;MC7q zjDU$yrYo*IRznczmVABt-M84yW)m5u#tYPudv&n zgX=tIpN4maZL)D{*Ia$2EVtb9syCC|>-ryr>C zRZ9Psav2c0svLdASRnu4!9xrYWNF%T1zm-_%{JR)$b-r<53{Lv8TsMM{09(*n~m@e zScHtu0ImGfS!6Q(6GvJ7*GT#&JpBCqWO={e5s8mkizJC@Y5wUmjanEh%XOJ8g;slE zlo(v>t!_**<$KSEmhy7frGJwAAJ;X_zk)_SHX1*9<#sqPiBpv@{ifmmZ`^UT%8K&e z+jrEo@>DW_@GK@&j<{m1e%kC3DR181G} z54jrkRaRCk$e*OY_10U?U}( z-{F?G0;8@=d#BF;pf86qzf9|yErl)-B?cE_3h9B+lyWhQhhjEX1my%-Meh374kGKW zWBLo_}MQ% z+#g5%9rafmB-Q+j5bk#3B&sd`6Zr^7c=B%7bI&~w_vIZ?xVk=;f6M>eEyH%!SR5fs zeufYKD8xM%agPkE%Mq!Z{2v1tP6RNi{Hw0oQlm1pXeLdbq;Edb@;7n9IQ3C{(6OHy z6Z4<*t!Ru`hd-=MG-Zs_Uk&v(C-Mo+$zw)M>z`3$#_Ec0A|MQRX?*<22-t%l46s;g zGbmb({^&8uAVz@5LT#lxcfz?Trk~1Re*05nWu)QcVuO=T`kU!5@Zv9}517t7?HI@w ze*YCtL#+M#2k6lFGCKBDEDE*SY8=F{!HcO=rzYCSOLOEq*9GxTnq+C~`d9h7^r!#; zKmbWZK~w|?n25!-F>}DCr%|a7ctbqWti}6W96KwS$Y`)r9g`aP2BhsGk8khvnc{OO=f+EY z_1AyYYaun#EcD5!X10`DwHHQ-!NuP0ws-oBQDAW>^UJ7awiH_Jg;8Q~vA4VJojxtf z_^;4z`sZLckb)smwXdkThW%gW6`p#Q!_MNCdNon!L*A}|{txLJPtG&-;yED-7%bfJ z!8@wiPxLOTlP`-jQ$UcVz$x=iZKrUB_AQ&Q@y@+IV>J5BkrQ44)Cs+p~T2pxq{ zVsNpyyX~DmW0W$;GOC#^g;u*GE($32c6UnI{j#WqJ{i@_mcg2B0{R<=pugdn)`!y- zQ%nF3%GQ#-wr(jsc5j2-nDw=cWdn*180ZIQ#!dP3HUB9qj08BeU4QuDdt)v>EzF+0 z8gHnE5dS*#ZO5hbSIqTx6wu$#fYWkPbcvSSlIv1W5MI~I*D&|A}Y|$;U4=!XmqwLb{a5yw+%CB*ST1 z3)+e7WZ{|qHDdBGs`N0MJsC4KMxT9N6Q0OsVz{+kY)q$@f}dPOS+WI>K@vl&u$8L+Ep@7v{#hb|j+ejR>8o0n zl>Rk^jaXX$*TZDk%gJ~)G7n1&(jDZ+lYpwl<6s@tMiZ*ZZK1CNH~Lce_d zQ-h`D|7GSBdFtvTBbO=DS*g!#E>CERaJ5eEu_pxg zChL`>dJI4swQfBO#AK8BN})*q#qGb0#{ftVn#5_=j1vRMo~801L(8MjYK-GAOb<4s|1Og1^Vm01j9M4|}CgQUVm>m7;G|puq$pMk>&;l!k+DnDBzP zWd`-B1bt4D?z(FiyXzEq*r+_?v@_(s`|eFXibj;rVeb!OECxueu5ns55#YC)gKr$3Lgp<#Ocsnee@313&;e01D=Q)%A3Id=PF z@&vG45t|uXc~~qzkLTF4Ygc*VsV7zbAFx~X3f%X88CGXIPsCAP`Wui#2HofjekjHK zh^p{G{$lzTR=NC#Du&3^Q2s^9|4QF5;A#9a-CY8Er%$72&%)eJ`mFfW0M+D46NR0Z zgSvN@b=F=-8aHW#m7a}tQh;x5yr~|TI&R<*^1pLoToJ}?0D;xZA7fH}^>c+PI>K8l zuRu-z3=FsoB+{?`^e>|Cd?{Trtt>|R>Jj88sOcqrDuM#}kz3Srk<%a14Fl1DA*YYV zb#A@&Pn+d1SAg2Y92Eh;`_?M&~FAyJg6Az8ypaVbRMn+~isE1XBUAMy#m%Cwg^okAS-UnWeQXD4+1^(6NJ$_ekqUe~`f=+MGwUa_RW%CLUwhwa|>4*tIl`N?cx zThoYni=FqJY`y+USee>Z)@|EV0SwZFQ#;^w6io$%j|RbEATj+v{BSr136Q5zH?aI2 zbie^Ps>&vOGaZ>ud-OOIlQIu0Jys<=Uf(A29nz5BdFS1-Gdiw$Hpt<{$ye_D;Q+#6 zhaQHb-A=*o$__3WkZ1Yt9<)t%+zFjtMg+4K_ar$FyRdi<-QQ0+S=Pc1PnzQT*Omim zG=^FxSJr?F|IL~;la;aX@*AA0!N6N@?Ip+K{rh+cr-j-{uLtNZiSNuxR}Y%CeJ`e(J2jh zkm=xWbm*Y5un+`y%7p@`9ILF7?WUhUA36*&5}s+hO0$(@`ZwPyx?6kwTaG{Bc%&_l z76fKRo-*8gUhpt~G9?9V*wEYE)6DJSo} z_n!WgMXei*TXpNh+=m~2I8Le9Klo(z?~+b`*T3_(r0IM8`~NflBgY_*q~G7l=%1em z%m2trjIZT@xsL(Z(Zq!>Tj45-J=#hOPT;#Cc#>NeyZ#HfEJu@w(Km9=n}x3En=lYK z1`G7;6d{Y#2;()J)Y0OC=B~aG#8Od9XL2}3p+e(@U5YxX1H44-z(keMFK_%|^eB9n z60`g6U4f=*<BeE_kFdTzbW3 z(2@HxY+TWT$9C=5UN0_x?t>uW2=XW?eZV^kK@Uv!gk#2|HzQUu{l|se4 z@#INjPV;|3`g=wJ1j~{FBW`*0%|CrA+gfjU`qPJK%hSJ#t`7ndnlgq+!QUu&7{BAR zJ74C=U=SVgoc{0if6xyzms(v@rd)jnJ!kqyu7HxM$GVsZ>bk96qtf{hPP)5B7hnHe zw7$+df0WNZ`y9yfSeW>UeER7KSr?mfG?78SfB*g=rSs^IuB2ChxCDHU966d<0fs`P zeL;qrpJ(gL$9QL`PVmygK>jq2#pAPX-MZ$@AciI7Pr*ZI`u$T-Z2;py9KU}S*8dR( zXxvU3-u{gA=?N2Fuo;p57F}2}_xc!Dq|ak(aTSp~*hg^OjNlGKjX)i1jmD}moq&Y5{!JpTbHSzu9vHqk$LtEqeb>zUE*N}sDS`%|wwUxdGG92k*ucU!+Z&HhLYk>7US<{EmzK7+QOBkxJ)3b(Swiey-_u4X!Hs=);e& zIT7QkRiy8-mQ4RD%CDoktT<>q5zUo7GsU=WgUHG_LLES@409cOSo8_PCT_xp_{H?I zL@(k6HHoS`r5|xvf@qAWjD((A6L+A01&*zHW#9naJ;2>E4`B7nk27bYV!R9=?{!q5 znQ(Jg{0SIDoIQJvicJz{oONcnn~PMF?7%?vB0YrAS%$uav3Ar7QX#X)6<1s-0|w&Q zV5T-qp9dbmRlzfKKziLjfGoXY^(>sO#TDSaaF;HRp@X*)5SflE0iJ#KSsc;$fJ~Y=0Uf)s8Ww_;b&Wt9>62L^bpU-A z(zh@I1|YR2ZiX<3`p1hQKm9b@IuRN`ADT3d(P)f;Ey-O8KwSb+5gVEfByAM)*>C)1QDQMIpS#g{~N;`gde}tG!VF)tUZeH`0e3wjvxp>gj=n^qr>& zq^j2k(Wxnx-~IjV=;@C1VKe!wKp(pvbRr+eAY+vb&%RR6?oSqSQicBs6 z{m0y0Q2v7yHB8^@4HFyBMUj8oPT0DHiF_UrhjNk&m~v(qw?D~f()9`i5uu?n{~MZ9~1a(FoDl>3Px3!+I12gzM7VJJAl5k{NLw+2Q5}; zJA8pV^n}~7^A7U-3(v~oM;s~N zU{N1cOfv&*l3tiRzwNeL5(k)VR0E zZRngHgQH@vz4kggjxdBfU%`nN_*X_@W#tigIOf=6a7Q}t{=~o@b|-b(ZhJkBl-JL& z0e;(Uw@L5by`!9M+O*MYY0B0=hS0u$d}fKEAw}iCc3m{182~UYz9Y24h5Tnn;|EU! z22gk;1~~DraSOk{91yaF0)$q)+*R&)=s}`1`pVLOQS*Nx{eade1p0x73JRiud?9`J zL_|%j;MO?$2pAC^LEJS-2!`0#LoNEC|Ix=BgGHcMs(bX9QCI=pNv`}q_PztauA=(; zy!3=5Bq2aZ0x6UbdPfuq#lIj$K&1;v5f$-MX-Y3r5)f%N1S|-IDov#ql~AOElu$wl zp+iDRC**y<-#K&U&V6O~?e5!>u(SK_opR>%ncJq|!~K{w$I9DSO-gP1r#}ZAMHw{` zw#DAB&QoL(1Ts2Cqc!b6g&$i&`*Xsj^i9ah_UGUf!h6toiT_0a6#Dg7;X7#mm%niY z&q2WUzW%o#eL5vx^ub>Ae{X-?@Bx2^&4%Cuewaz@hNk@i*ir6(4?pq<8HB-1Ct|7V zH!xGkdIxXIDv-G^?Ht-)5A3bU{Y+0j`Q(ma|Db0Xfe(bT3wtwU(Zwb9J`ph64xDHXv z$c{lg&EUS-DgsIGD`#=>~SE}8;^IIX;^PP40uE+l|S(! zRhz*OFti`?n=+^IRDNWHJVsLcbHJ+y>i^|eUQ$q^r2Q|x@rtn2`Kt}eeKsDXY)*e`P4q zw?=tju(PYWRDOdAGKWMp`L#IEikeL1k60po1-5D-s`MXv2p{jSut`cE%s{zWA&@uy zBd>Vw`BM1N|BGl_OuvHt^E55h68X8M09D=-+ZcQUTZQhk?MPX=FTR6>&>{5i%p~W` zQdkTYt3%#<-`z59%dO;PyifgL#tiwv4-b`_Zp6~C3Ru$qLtZpgWA;zzdNif4@e1wF zdJe6j6x6j1*xWqhBSjb3VphT!qtS_p(^?uM5wH1n9v4BKnsI?7K)6>RUv60Cd>u{k z5a=r@PvWeh3G}(o$L_dDUv(w+o&Zvkz44|SV4xeNfB$~ilWSRdb?U3S-z-Ub7(H^d z9C^eMnza1P9rXo0N}NsdS}s37d6}wQRLd~YxSZBrd+Z?-u`E3iQKTE;H2e*~$0NSz zdv)p*odufmH44kv9(9!Q#fmIbK8QnO+}i~&igwsh_eLXL(kCFH|8Lu`#yS=$$o4yI zFPCE=K}v^-U3T7CS+;z~PJ&pL6tnYwgcmlyI`=#i2-)1X_sWS^%St0w#AM3Vg)^|; z#7)o(oV|qEtj_1$`E#WYXI&97dt3X@=r<=nGq)aa2&84W(fGdN{UgffB`j!M)V_V0#3u6zDtSq+$6Q35OuRmRMZ( z>SU=hWqbx;lag76Ea5A!|GL`WiR0`SU*k;~46Nkor}8Vi)S+>oU}H4Hp_1ZYIbNi> zv5vzui9T-li)gonY12fo7rPvP9KN7^W;ihmSw8wm+p*%tiv-%|L*(mkrD7OgWDQITY zsDIXZ03Y%ho_qdzATpT@BabF-YwEwQ*I)3fma0*Hg0V)hE~3qp|CCcOnB_((qMa1j zPL4fRFkQo#S!*p3EMKaqNp8qck)Hzwp>Fhv3_>{fwc5W{`gVl_0qx8&Xk^Vna==a_ zbqRgLQ3HJ{nLlOHI8EtuKeel_x)J~x*zLF9F8o=@=o5*e3i#;%_*5)cOuRyVjt~E( z@_+4XU&D3pCAkRuxn76?d9P!&7-!`(T)!{E-j>`Ge$xH_k#1b}3#G;(mb2RaYx3et zFYDf?zy8gISl)Fy%0V9hpMfg(#bn+8Ou8Qf)WYln6^#7iA92M+f=*PO!3GMPsSmma z^Z4VB$+p{W8^KUR1M72d_rGGabFW@}(a)e*?dWd_lrQ)*ixdmb7DzfG2E6m-{hvYw zzI=-fMUH|mZ?T1ND<-RCLa5F$Dw+@ddRv2RlI`1jDf#Zf2M1z`)e1sXf&4{*=GCG7 z0V%uXny>#sPE`|>ViINQ(X=VC>h(vrsR|Xdxb5$5Ctz`e+ zsQbNx4+)AOToyECRihl_-TaR+geYHL{=`EJPLC>irgPx7929D#Hu{t+LUz)p5%lM} ztakrJ|6df^e^1OL-hMNDh}<3@@NwUy`d1rehmLgrOI{AZz^*;_+zZ!(3uPjnq4|+= z_N-aahbTYyiyXet|zK3#x)YXN~>s!2Q4#59-HfbrvI#+ zv^9N~)+A9e+Eo6wiN$tR&XGYKgsl!#t3EG5g}qpcQ& zewI+|{~0pMOy%XLu5`~~rDsHkguQiRJ!HqYMhq{pF)*e2XBp%8F>01z4nj%)&;8#S z3?1)s(!do4Ue%0RjG>^~SQU66OyCK?ST{-@|ERtDjaIA=QK+(d#{ju%C& zhMGr3$a7XVhrOBfl~PmbKZDf^_<{Mp`=s#UhsvXzgfs`kgVEwGH%s9fEc?FWj#9w; zhm3>Kc5=KSi-$d);+$)oWtgXP>0s8u!f; zbfSWS$zy!Kft94Kj>~Ul1v~>^e)(nb{f6VpEZxhc zh;LwyTY2SSO|1B_&psl1qd(ajXS`!Ba3zL%(FZFP+duQkW}W2k*`urMyybA&|7#;; zSx)H71=DuaIS9vio#?B&a47X4)#J zIqZr5yDXXi<%CC~cKWQTc2R+tz;?yVG!)21B<-)%xXiGk%0?x!0Khpl_6pdMY9Rqa zn+hFcpqM`yksN)z6sF*{(AQB+NZ(F>3v4>^B($h>yiGPTjh9leu%}VzrfdG6N%m zW@GeihPf#B-#3EqNJR@3P~9msgXXm$|zu<1(r<^dGs+( zgSD!pJCp4=c+AQx3-zP+!Ft*`}Mb^|8GqGl=s+9T_ct3Z-y_o|G)tvH{r#@L1-uVV8@U6p}GjRXNe)S z6T#Lv1gIPR$a6qyrfJmvO`~6w&BiVx2kbmbRv&?Wfiz2GsOtajG_0c)jlq=uy?5U$ zZ@l?7Ayjt;NG*CmT9CiR8(pIr}vxz zKd~^)q>|Ez#&nzA-wI-^?{eJ~LtfyXNL z&vkS}8<1F22}ty$g^DbbpLr>ar;L%*hg|>VJZ5-VmeN;ZJT!$qo?!}4J;f0&dC^>c zW6elC@~8;6ygBVOkuhVGbc^Y$1nnrls;i6iz{@+`dT84b_8~I##})Mt{Zhqs(SM!f z?~OrG?6bAs26OLV#TE|u;_O>~Fj{e?VLCX=JudHt?X7?G0K6ZiPD5xj{|CLa|EZZi zpZ(cxFgS?1Oxv#MHj}3C2=1fKI#UX#osM?1no{gY`frZ>?*H6+Vuvqs%PzK?^m!wZ zpTOt@m5vIIwC4G#CH_YooP}@L)qaY*A^P|*quc)A`U6VnCcp zV;v{z10Q=36bgqe@DX~gwecP*~+JK_hzSSj7zzB~q-_BM9<|)F6)wxE`Ths>S(TxZ{*C7huMtHWl}dG&X4FhSMP0%E73tq` zCr3zwUs}l(gmZZQQ(0cafa%LGmjVZJf9soE0xN=AjdB?l2l)K{hA^_pXD~)bW?fw;fJ-}#w~|Y0*=M4;v<% z<0HKXr@so{S!e)N{O_>i_PTwUn%x-jDkw)rCI8PLNGf16$wD#+6;l=DH~QZH`}0@o z!+iep_D>=zgYuZ+&HSJA`5YXC&<-PmTV>QL#%MU|GcdMoWBpJgi3Fe$`87A8&!~rn z-aiw@^mVU@|5wI$(6|u8c8;VYU=gJpSPWU+Wux^rRDEoFhgKV#m>iFvLfDZ ztc3R`zrxDvUv~9WkY5=D&yY}~em@Yas6E{NmHK~$RGDFSyd=rlpUfo+X$||Q^r@JH zT=1pLAm#Q)g&>r&8XPgjk?P-IKv5$Z-iGX-GE`0ax}9(*C`G$S*> z8f1-G<~UiBG3Qx7;ydqt4xoG)svi?IKNOmG8OjFe=ctw7b^3#r87Q-`#5}`*0eG=r z0#^(DUKr(a?z!j6ju^Fg-0{cj^6IlNt3YuDMfC66Pu5;%T{-BW1F=Nx0ZPy1k1dnA zpwAc98Uio<`*FkuL&$*qSst7W`sNGr7=r#eYm9vDw9}5*=j(WR=6}y(`S(wN#V?!~ z4c@P>th3JAn7w_l&W?yULZq7f+im+b8OpuO-h5Lb$r<-zw?y9l0|yS2uYY|fxdF3^ zDJ$u(gMCQXS$AE7PrW6G$bz0&M)9;WPLprp29nD#ayHe+7#zVsvs*Cv*TBf(ukW(6 zd=LA-a8y}PyMpvzx<4oe%7`qD5%3Jv^sj~CP)&Z?f;bO4@BlgYyz^vH_-K^Y4Mf#! zbXiKyBlo7KPEz{)byQBUFu?jdkBC?D|E2j6N<$Ifquf6%=cJQQmOJja!;pcEFF9i)zr_(S4sQJ^s7KZ zSyZ-^B1I2j;DCW11@E*)0-eCs%l=9%!Rygf|E4TJdST$OG5e=C&7levl}cpUC3uHSC|9yn}B#7MqA8eZSvJ}|%Fn|clSBPK*4%v5KZg!>g5{=e}zVaW0D6v1~&4nh) z-?eK|b{RJeOXw#b@B_kX`j0o@4fOvxu5t-O-X#5N{eYxI{&m+|H#YVACry$CpMHv^ z(Xh08s+3$Jqd$NCJdBXVM}`nM$~q0zJ6 zc2dCXY-~z|^%L1nmKRx>?PPtlll{<6x;3r+DgQx;68q=qQ?w@ff3^OPs8aiHy5=&n zD@LZSF_L{|;H3`sPw8LIy|q*`M&iH&50vGXMIQrlq)3~k@ZSIZ9y4}kh8Z3Id-hp* z;>jn&j2*2I*$pY3{k%OsPVRu;gbCxZIKnYnL>l+pb9elZpUUq@x?riT?YG}Se-kj< z`sc?S6M4HEH^@h0pg`>ZLWCx>>#uWQQcQZ_ez6&Dycg<+>*1~PH*AB%J!hw3o1Q@! z7&sPN5pk)x199KhORtNDR4q_R|4u>yLL!ZaEPL4XM_%GYNZ?lH_B(D1#HPMF6-R^? z`k&wbDMb^4NENl8ECK4su5|xFc&iF5@b^$V;AsB!&n_jIE_8OkGC|bNBg>t>Jylnt zKM4awroQ^BR^$EmKX1EDZiH>0#eMK=(_WK7*o%_e3ay9!`vCZP@9;sugs`K_A2^Cd z4PJj7DrG8-hw_H(&ky-3F8vGPRmxFOew3Ti6iyG#>3_%{NtN*F@>_l=Y@z=-qe@Ou zCWMV5Rbp966HNk^c18UoYdivHRb=h>gpyOeocGC~5z^Pcpu(d1vDCx^3$b7FuY?=L6 z9z9C=B2LogGsVx6`+w%;=_$A#va&2z3S3qH093p|ey6X9Yi$2Y)sSRdmM~i@{Y)yN zDz|^BT)K+mzTZaQW)q}vBaCO{AtL{}4+zMah2kE&N#Xzf2>X}5hrwS@+k2)K`hUcY zi1}s9_3t?KT#Nhwtgin=e%Pa4{r}#W70VC#-u^jZihG1V@ZhA#{@$29YdWzkxBuw) zA>Z;}o$_kV{~`iLYLwsVgx=o$&d&j%(>G#SA>Ces&iW5=Ke+$|cet#oLs=1gDR7|q zUe$p{<64&neGY1&|GEB+AQ|K@Fj&H8Mfn?}A22F@Ka}8Qwo=vnQu<6Iu#Ac(6B@#~ zjD7$mIN=iePs001t_aPVoS=Cd-f!G^!;SbB^o%|O55~&OtFJx=?>E-R_a8h1bH;WO zD$0-YYW5GD8wO{asWfh-f!DnKOW|6>{(*c0`gaIZedYEaM4}=}oi@>b#8gFot4gf} zfD*Wh`gi;5HbaTVaqW}yFTi_F+#hBbjRnPo)^iBICqU@WFE4hVd{Y)gbo2{eQLlviGncQFH?fDs?Ik5t60 zk*3h!X6tQan+La1-M;qPRQVs4E#mU;YvDur${3NC?O$Syx>z(~!n$xNl$n@8$iV>o zNE$!VC-QH-^;QNm8!+gg^}}z1L~$cc^<;b%<+azQ;U)FcnAE#a)>&&ESs9I`9`%+8wm4O@x5kg9zS$sME!3Xc-Bk_F9uIMVu4_V$_7o~am z0}4G65BHG17cc0^tKs8nfD%2Ic4Ssxz?v69;Vc?>_)PjN(O5 z9z!fqo#yzVn*3gVHum)8PeUn^(zI8~S(}*!*$O<)oH?V+=GEwP&)x|WCIC}0%;E(a z(3`hlp8V!FzcETkZ#iz9!dZqNv7|<~ZrF=^GyTo_WR{%st8)|q{o=`|p45Cob2-cC zi|{l1u%Bs;zC(YVG4n&2^$GNg5A&(Nrp?o-|LMmP2;8uQ0b*VRA$BYm4C(Mmy*LVIy&Pg6HfEIAnKn3b&z041!Pu$)l(bEzb-x=+egA8`o!0C zIWM-)#`MpN-hTV-C-djw1A~U5tcg*SX{ao}D+G{rkiR{zzb0#){O@VB6YPU6SRp~% z$Tz;3Q%}_EefQ!e^&g~wb|P#iaGJz*A{ZlWq5oRxAGiOS+W!QujL#<7pHgvgZdEDkOqm9iH=%jo|M+aO$W z=^y1~EDuJ`xM!e;#`OO)Y>#m1AO9#XVPF%Pek0V?dO(KBsv3yb<@Pg+9(?dYuuS{VA?E#uFgApQGtAV&zPf}K8!6dTfMv3t4podDoWC<|T^lF3OS91Qr%>>mhN|A&eJ zR!=4LGqlF2qh%$_394fOGrYb7B3(v57+C4EP6({>;3V4CQ1$P>w0r@!{gT?2fR5BE zBvmaV92E+O2+g>@SP3;!1v^?thTDu-FNf6t8?yiK;jlfnmvGx(jXgCv7_%?#T^sd3 z&{Db}ld_kwVDvNIpnu4er87v)^8!>W(|=09&}wC^(iHm6b4EJDQ;ojAM8H5=l^C;0 zTnMc)dyW)-fYmlQIElP5=pa&K)X}n1!n(nQqrx~KfI5==!!Wyh?8MSjaXfNR%;%8 zRx7hV`yZlXFti$NGy184Yo(869obF_=i#LvBNzmeR29KWCKwTqB|HD>&55%dRK7C# zn?pZPQym3u<>+gk9Tdg^0aOqALznA?*_c?BBZellf2;(iI`g{Cl^-(Mb)Q$R{KNxC z&2uIBS6O-03|)M8gGa`S0p42UPtYIsz&*NnfA{ZVzhvFSX{k^6b&!+(GY!w1BAk$7V+h< zdI2*Tl61@cFX#?9>vR7F{=jFv{|UV6iRuVx6NN|8Pi%{v2XLSCY8WKi7lT=EyZtt~ z;csCuE8g)fgRKWvUmb&4aX0Euy3XFOw@OD=MMy1=J@)lCI`| z-v2p0t%CXwkaDCfb&f2a9u>+#fuu3}JAK6g_)7i1Q**Sb z{ZrbE*WdocN^Ov;hZ6mJwnF&XURQ^bxh^w^!Hj$t7Qw6uR{+mJn%s%21UB;NuE*m~6Gp*0MMD9O9n48*j9+emGXp2I!N&;lo$NM||kB2K`r~UyJ>#>0fd3 zP=&tQ6EAS%i~0)kJAL{p_uLq|!ca^u?H=v_%FD0FC%S5ZDPuRxSaMxgqu&Sn5Gi!f zUtca)#t`Taz>NQ46o|8|id(m6|{zer!^cn-KmR8uZ* zz4c#0KcTEDTXmIHWtH$kfs9=Cm%r#XB^(e_F~I+x!qTSPV&&RvvHu+Y$T|ll=}b%~ zpl2TP@_TW_jSS6eru@WS&+Bh<`@d8kjjK>R)F?kc%I}JH!o8u;PNcM*I03x4(sqKu zWl91^S6xO|$%aFX^c_hX`yYi+IU1!OT5py9?}nRh)PZC~MpWO#K!~n-Pt>^ldvPBw zhER7R`XPS6VR-A!x8&tl_@O<73iP}4g;L;u00!G>0Is**dJ}c+13&XB^*=VyKs6u( zbh`wL6Q9O-2)Il8e?ACMIu>&P0k4$+)x-Y{ z6B)7lZ}8~+l(5^-^-r-7hu2YAs{5aPR3~s!Dag;gU>)D|H>c~0QP-irefp_qu-7ka z`_|j?F$Nzl!j?~wUO=Vfy)v+9lu(H9VLlg_#|$Dt#BkwgksPQdYOp`=?{?f_N2?1w z985WW`~;v3GNo8k`pmCt|4^wy`&0jOepMLaQDhqNtk$CMq?JP_))@WGza%nrxK%k1 zGx87rsm4`)ZdbF}7xZJK%FzS&ncVt}c@^Yu7JZ|pdQw&$Ni+2CRhuM#Gw7H0f2Qhs zw0|(;LL~CP@WKoFd^mWy!FZ9lv~ZiZ=IH;GDN|(B=+UzBsFh{-O2g%X3-}Q_1PIYy z`q%!a9qFqEno6Hbr{X!9p~yinKmAEGoxJ#>$R?rP5CAI|$pz<0aWlUBrHNao|8dt* zB5F*;^vr|zza4-+`VMDLdF25an&(0!{vTn|2>Tze!|J$C@~R5v1oX^9UjCsg4#NOH zJ~u-jz;!hS`0+Xxk<>z8YX`o8UzaX*CP0$c8xtx|fAlEK_DZgQS6+T4-Zy+=ps~I( zp79fC`H4Qn3IPh|M&yt5P_u&m`JSwM&bLAo}#({nGrW8SSs-_ShCw z3d+kbYxe!%1GSy=5=BCW=9$Tn2oRo*fFV-if2^GYbpm?kp%(oE%#pN|zPJB6(;tF& zp4)7_l_>$Zr(p2R5!h0Q(vkJmu`0s)m>G>^gs}w5^;vKyO|t*B=YjDNPOCIUepJZF zRDS+6078}MKlaB^Ic@2<0{!^?MQh^V{$$(&-WfF>T>i{{0T6h*;Q4&Vo$w5-p&%d5 z_)yN6aArt{QwjZs{Lgz_=h;}RqyK>ylMcsB%o2XH0)Qiw(0BjyS~XmMOp}TVQNybA z`F)2;oTg8oE_>|08{ThRCr|VJ#w%0gt+(ICwen+*+;eIXmDbZk6YQVLAK1wdhgdt> zUpXkDPk}RwBuuLsc}V0(U3>o=WZ?-Q3hGEw{U=~C+7kNURA~_zzR6%_D<Do z!N$SX*kV$Cdbr3+D+*?a3ik2F4c!NUJehSy1K+@y=k>XgG7_#jmnS<@v(@ykP!U%|(ggk6dq=e%kbw(l zM6>8n8XN$LBpQK{O~|sF6R)mkhX}wtIW9i~#H%pNKThX}IxNZ~aEy9w79x03&pha@ zXUuPmzBVXDln_m#5q~7*1hrD6V{o1HJM~|?|JjEL`VkxsuJh1C51CU{@xKq=e;*T; zF1O|h7%ndyX9cTUE?NGk*D#gK)MC*jgV~9@E1rS$cl6B4^mx3m0Rx^|JPCQj7Zv$P=Dy5^dzbtRFF zH{MwG+GlU<|9Uj`G&>5by6-C+Y`6iIb6yW7yjBfP+p*5?fA@PEOyR_TNzd zcLlBkt^nfMW3N4Nq%s=!zj4@am$Qf&Soh60-;lfSxl7YOJND9ht_3VKKQC5>(cHUWT zx#?y+D+4&-Bm&~_VA(i2qbzV(#v`z0M*1vc@N%3v3%h%_9+nRzXP$9}d>czN&X_SH z!wEQ6;mT5%TzaW&fmL`$<;y?XYN)mB?gcBXA_hHbS{sHc@lpSeL_4&Z0t&m;&u5Wa=>pMX8xdvKs* zFxnYso-W_sXCFM1!}bgb{r6z|OJVyjqaAV?PW8_SkifED!mLF@|DWI^n86XEy8a!` zQ7VcC`X7havtnkY0Zs^jE;gNNkP4rScUNA0>-F#m9WEDzfoLm_deFWDI!1_1N=iN0F{meH2~pnd%a%j+{E2pPMzX851tb!A1ANk&2^Bielb zQ+hl&uPJ`SjdJbu30Xh+jj7xukCZ=UODSfmKzUaT``d9xeAGd%e$@GJrf`cPhUWVJ z@|ehy>8PLs88f1p{<#$NhU;&rv`GJ>Pd<_}$De7S73lMtGYyZ)BypU+=AVL5#dGce$BJa|e3%0*LhM#>3h^xa*w1oWL1LX^^Hp<`%F z|Mk27B1N;m^27r|Rr=vOO{M;ygLSnY4*b}&JMNJ6!z%3_jlkiok`Ax>UT&p0$-e-75vV5N>mf8IQ0s?dka{04!Y$5Q#Hy2h!VnuTbXfl+d4mRZQ)9z%ZO~#+kCuxA829m0Z1w@mEW0`We^)9CBloCk@E-D(~<)2rKlN+@r#CkXBXD zlnqj~dS2VmMP!=ILnc2}D{Y+s06+jqL_t&`8uEXZlg4bUl-$_!FQ13GB6P+*Ya@W9 z^QkPt2AU)P2k*Tv7hZTF z2BqV7;RX7;5Ch9Oxcq#cS*GXnbv5jt9&xyQ0~5?vT4^P@^KN@5YUGQu(T2R2cJ1Yu z3fU}c!2YZP5XaC|{j*oiYHNZAc8rdaX~c<{S!p(lG~Wt&v=>jYfRR|$QN%pQXAr{bkc`nsNzf`A?|wjvnr5>=GkAXM1H{d^i>zGcRwfkUjs{^apwHT z9F+{<%dtG-!iAsd9%!3=;R|}fUWkt;Z(=0ZZMXeH#^Xc5t5c^&4E$iyB`lZXdiOJd z8l&Ig+njvdSf-rXhiB&P zzu*4yP2DnqDHF_NekN9pyWD*C@_%T1WrNK_n6*k#3gLGWsg9_J1x^hWTxb%)aB!JA+n=y4uT^zx?GqYhKO$ z&dHn8reo1b^$KK61-&}6L z{dVc2FT%8{*0lN|O}a`4X%?gb`FIT096eeTX+{i~v6b<``G<%7P|gc`fD`4F6R*UA z5))-bJQ%EwkC=8n$Cg7c;$aPy9MwE*W2F1O6CHkv)eDhw`Y27v5S-27`>(zB+A-gIKEg!upwR^nfiL4l@u*RwB7Ng2gJP^9`u5$9Z8M$&g)Ac{zmarF0|PCzzuEpUtbC2x z<{?}G+h2)6V|*xF4Yub%m^a>dLtes6S}s}Z?9IcA+TOi;XUa479Zks3G^_o;zJN&b z0PXm^x|Vvv&k-x>r zkxT0j9vs`DQGhkm_Sanci)s7AnCh(GcRfTMG(rEkHn2L?pf05PAfgzFu`DN)q9V7| z;|urC@F9rh9N@=g#V@?@*D`33_jLe7;k(%1`j%U6)|J9upZ>be#8c{e@iZZZg=a|P zsVKiV?KJtq7r&@0LcWA;-6mt8<&OARPFfs1xdZoOMT#iS#LMFun5}#K3CHWIhrA#8 zFWUXdC!Z{L;hvIO`C!HefKNkq{UuICtgJ9)Jb5g=AFHPSv!D;k$ZZ~3?+n$)RJrhi z3uO@denUx0`jP9FoBuACUG^t=efk@^I*;jtr71OIE6q`!9G9P2O+yX#_xo=!G>0F1 ze9%lljQM`5FQRk#d*M31O4uViN1u?$a5%=ytBe~J+6%!e*^T5i-wLHVh;q={r>B(jv{1U6k3 zKZ99D$!C&fxn4P65DWHA~eM$7pKG_=&|`nm4(4WV=5XXdt){;pV| z_Xj`tfy|pXA39C^AGr6%0PR45q{`&y;NwGobeNp?D}Np+%ET-1EO_Na8KKVutKoUz zGpsE41{T(NQQzrMMYPJi`Qdp$F*DP83d+Nu9Xw)+b-De_Qf+^XXaHudoTSqhfHGpz zS-YjSQlwTnhazJ{W30xNgAY6GNBSJZM=BOvd8IuEjZB_{xT4-mScyxw?g%W+pF2-_ z^}%ycr0>-1`V+uReuZJV0ej5Pgw?p?@^gZOS%lGNU~FA%>GbHpfj(}gIG}yh$Wei8 z+S?hfG^ZMUqfok(eE}Z|Hdzt30%`{NT7P^nc?`H)2kJO$wk1TtDeq z)q#VRXGhOx$q|SDL|3KOJg*$FmE$Fxf)O2qpDl%B)8+5RO1iINKxTa8F+0)7qml~3 z0W?*5HY(Y1WK7x1J2FTI#+ZIb7TwM&&bv)9x`frQ=NBuDnD((Mnz%~Q? zD9FHmWVhYE6_(X!S}hsM^n^qC{U2XM{1PAXO@jrw9{X%wa_J>9c8#%^?PvbL+Arh9 zFn{Lj#GQrT{r)1^fXfTgKKNT>jWvKLtNhAKFAIMjqP1|N@q5(Or3pi!VpbCy4IZ7E z%l|1}{&8@D2bQn7#+uT#TNjy&mjL`AIs?m+%?^VR9KrqgczD182g=n~U!_$IZ(dWB zfd4Fue1+y@o#l2q=v!$j)RRf?yaEQRBbgB~(TtI40K!gZ3>%3IYn zM*9PA05`l|*I=5W|F)xVx+t}OYtX*}FI!264W$SA2YwuEiT?R~@^ySnya_9!FmMZ$ zUt{J2jnHoTr+DF)c^SF0{94;zYv{jG{~Nrl2xsze+pgGdaKSv~RFUJ3fm-eU$hAwE zG7n!|{oiD0w*QBFd0Mw+?Vr84HV&#aQLJmY;r@rS=??qhVRAmV6hYwi?@GM1<(5YL zs555F>at+rr@AuI2zc;bH5w|az zhihgxz-*5}Q(ySPrdaLoVMi$!U3{@Fzdmr_K$(jbdFIbwpzXp$@f`neKehAR8;@h#Z?rPMo^j|q)R|bVib{5LFtf|lI~^@BqZfW z=TL%lm&DL5LpKZ{-8sP2edhn-c{^*?ntRu}ch2{mv-fA?O&C!N!OB6K8@ZB<%?_ye z(A6YK(&ySm90f1e#%i9K`%NkP6-WWl29Z(J|b8pEjgEsE?U)?7uK$bUHA1xW>VT54x2GdMw# z_#A<0$VL%g`7oKsuk%EOQpf|Z+ERz@7qJ}^kt#G+p_h^iD#UVrQD!Zs%=d>LvHdWS z%B*t{v);zhmDRXd+f<7ox z91uz_O9}7PW3y`MO)!R}4~NWI#vMR?@nX>wJWp0tPLb*yRRzVy!bDL&aQqYTnk9ld zA)@G3m|@^llPQF`-M)6=l3DsvYab$w@`=#xt#UycXto_V@tS?Cbm9NhcEDrC7`D+q z*m}((t7yz0a(AxFwll|d|HcjP#heo40loPO8L_ju&E`Jcc?rK!WH~pM-Ie2r^q zrVt`S;U0A)Onhy|LddSismE|u;YE)wD>ONfL_g0bBQec#p$thVc1ae_wd!Ooh^D7R5dBi<<#?2BDkMz;kGrQnq*%?b`mfxr|rsImMUnLoNiN&5Y z^)kvki(3;!l*e|=?4{vAPZ@a*9AkDa=f5q2UcRWKxmmIY@yY{jgB|D$($Btk{pX&X zeiVeu+27+iVAcXEb-=|8@Bytz#UP%LKWh;YUvbVpizY$m?)ZP`!DzZJ`SruI+Evtn z8caTLsG_!)!+d^KDV8DZNdoyaRWH8r68`0E+Wo6*Uthfi(`#UQtL0aiQM42i#OF`4 zmw45FozzVNU&lCpRjXF~(U2(kd<4_7^Zw6iu#VcoWONON?^O$+mH!osXg%qmof&Vp z@m_!cC%rEM*^uBotj;;?YR**Srl#*07UiQVp?qC0bM~EDWc}o`f#u=+T3z_duF`jX z@+f>LclL>t@f7f=-2d2j9!%+}75CFDBfzz2z5uXV!BRg=gzR!}jEfhTK*C*w?gh+Gn6Kj0%g->1%2F>Xknhr2Khzg9r%eMzih9 z$WCA40ixxUJTMPl3)rW^1n0D&l7S;xN?}h~f%eNg@!@qd7WvBYCT@#+#RmdS6^+Yk zDDyt%N$&6FR`<`@Ln-#aU?iRI;Sc4%vyTFWI5g-8kx*<+tojw@ohPg{G^Z`k!bdcT zEI+={6L1|_L+<=l-or*raW8A8@rt-~zEZMW6NRA8K+;S}^DvO7b~bL3AcUL^nCxl{ zFX!Qg2IMdM`dcp*m@n<*1U_vHR5RVy$V6rY+^j2~vpV=ss&W^Y9+&E<0;mIk%YEmT zz^Kc^DY8U%<()(_tlxwVv281@)-O@br27*{9??4>dwAh3>;4sV2f z7Ml(klzhif*e3^nup!;IZ6{n+3EAm9JY23e@dXksJms?#Zl%NS1da)j`M_NhR$fOjhbafla8G7)>ee*s&!MrGG5G3GYfX_H}kbiG&N_Hzi)Gq5N%l6`U!AHsH@4 zg1f7es4#q{#9t-aU4+lS*z+ehYOk30-0d!WNw$~7nbcKbpZ$+KJ8_N0=`*g}CXM#VPgE#s3bcdvB_9}oODIc1VrbnSD!>P}IKPR!#Q_}Hr;SQMYs zJmNeWVp8~sNBe3KFJJXX?y{)1JWEL!GExE|KJDBF54b;3b;Yg~EMOqDKEN51-aCpT%toV&4Xc*Nq7O zivcAVdw040of_TIkkk2cC+E}tTc`&kX-W8F4LbtVEY2_<#}>hqA6oR8uW{=Q1=jqB z1mD!jk1pDPrvdl*IUQnmz6GtPPoW{~6Wbv>aPe6t{sG(}o$p2s_wZu7Jc*wMAqso8 zoJ^6tw{$^=q3a9C@j}_5P5st_=K*AIqf4OQ6`NkvY7MFd0)ZSiAbV%N2uj`8dSI-I z^pGb2V?Fp+I_{zd0mF^XD>~2Dsqg zi+79QJLKKywn}4>U^<%A3asT)Bu?9FQM-D;-y3l#89vI$V` z(Ijh1+zCxN78=m4x(Gb*aJ!sP9;)*p3}S$H3U3%nIW8Xf5aRGsvw=ss@*9O+cK92IR#BCy2PhiVZd%; zlI>&XEcqpk2ng<*>ihr9idoJ1jM+UNe_&hQ<{`{@O~*P-bW-_{N{|HwVs`;-{%I@$ zuj;3c8ke5(ndGN~S^{_Hc*&xCD3wZ~75wV7I1>`>9c zhQZ9~=V?Fr;Jgpm6nh60z7#g_O~gwveT4rl2Az3Mo=ihzP#~f4KE&sngN?D)W5eF^ z!40<&LA}GJCI}V9(c73#;bVLnU>_c?*dXP(ukF47{{~cb9PQy50~XYCVR7_%xdaWs z2Z?p7e0>bo`LB9V+fk#1YwtsTOKBlSA19%wO-o~JDTBz=1B95Wc#8g%8vZI9lmTE? z=7i=1>H(n=rjDv=JO6q%r9sQp#`);cVQ}{yBrj)$+3D|?VXwYJrUEe?j8~YOD(N&!AROtilndWO~$*Dx4 zl+%c7wOl}H%;yx>8x3=#&OhMyd^Ja-6gD))L@UYFREdt}kYT?$WA7bDpyt+y>x0=N zdOH*G+bf3ZBRgGH{-h?DY!Q!}(0`MYAq;ym4qifeV289Ijc=|BK=#!?yPQ=(kuJ2) zp+(<&0_+N>2|3GfIs7D-Iu571*nfh*;HT^yfAP~dCX{X=LiZY`zooE5Wuk1G`Dd4c zHa+r(sDNP#6^*U|02<4pUo38z!UuoJmS!Z}&)mY_X;~BQU%S>C`>$UH74N}6V~a&m zZz7(!3)p znAn_(Rz?Ookp0q|b+-bl6wzk`AYqH2dQ_on3QN^!`aU>bZ}ZMIR(y`f;wNp+Rqqo{FG5Na0?n_vu}aA62!(Zu*3yIF#e+y;G5kqO;6m*6+_kF2;n% z(j|op9?^Is0yVyO-2`tGswS2%PaH@mLJ)!1;y$~?p|3eA)u?nm0}JOMZJuN=&g_Wf z<0xkOpdfb#*Two0`ixhA(`O?rOK^1p?BxwWrOc$$JCJi9cJX8^cc&g3bCel1k0K8E z6LS9Yz3(V%i2k72Y6toObGMRZC^?x|0xE231542up=HP`6mTdke=|<-meg8s!;-@1 zg*-h}%toD6w|5iPcFfWY(d-uYxjQ*q4_L+<808-E;)}2-xW$F&7IT&&d=>t;NJA`q?Bz*MQA5Clb!(bINO-O48o%hSp<6Y zq9d3z1vXlQgjLnVqa}ZI=)U{vqYeKs1#@5Hk^Qh$?Ax#l5jYMVaJr8EEZ3825k=#c zAR5VcxnOMqm1l;#R?&iQHxp(3-fFgD0$7^SPVyus1{h_Zr{!C>?u5s`@PdsZc*t1-7Ir}TP)3(fxkk;9TiCy!<0tCKH5XMt&QV$@oO@ZC-KL+QbsZ_e$>@J{@a zRV-i4F#0l<1uA?N#4hOhN={N8;WG?kemnYBDML>cN2B}i_jC8{n2q{co4+(#<-g|4 zJKTN$#Y#qB+_JFUmnRTjp}?|M@iyaP870Z9P4)vgrKsdE4Y8^$NTU0|fXJKmU_YmIt;NK_@dq!ct!~JSZVSQqU-LEj@$u#ViMtO%h z3hQ}$@o~{a))WpU0eXtt#^q@cc7K|S zdW4zZfltL>QI!-z#saPvPB!NRMYobuw+g-}o*(-kT5ss&Y23Dm$3EETDnIYYX!J@YRB1TiTc5s9+7b18)>lMPiQcC0&S=0Ij2s@HJi)?4_RJ{~=n9#96+NOz?F$OEcUD(AhA^7kAZY;$u zZR4&5S5igN?i@KMC+mrj++Rqujl-q(mxsR&CERz*?d`IHO^u&jos3P(-nIkR_#G0Y zJ~Iob`}FZ_ebe#T{?EfON-WgQ5pwx~OO&4B*NKjPY+e}BSr0YOnr!?^ZW9_3=!)>Y zHqP#9I;@@F2=4Nn>P5njzDxZ}yDQw}2}U>77(UQzU7x(U$&I5=m6%FEp6BMLDTtR5 z=~}!bsAJLS^m>NJK7L-{9AW_G(x{X6Mw3i)M67-O)76$W!aR7=Pc{CqHTxP6IDul;< z{N%=x-s(i?KfX;|fIid{abwm!0C4Y_;BRVNTdlycj>N?G={}x zSE=MCAHlj%2Ud=E8=<62FlGmMWA$R%jJ9b4^u^=Dlc2QWs$tts+zYZ<0VSPYyj$FB znYgl^$kXb+OGc|%wIbYuZjuM(2E-q^H;?@VZGi6 z()NfbT#$YF*H|sq&0-4{f7ZK10Pqe5p@nVX%(RoTJg437-v68X{ zsAGkN3j@y!pr6lot?P86O5Y4;wWZS9np_VaZoiik_FoQh_1y?|cA;Ww(O{W^omU3w zI=^G(?R65t7L~6M(r&k6^Q^zmBW*sIR#Zk|)S+|hH>6pE0P(Siyn^YG`Sr>fX-qN$ zLq0-JkMfFv9YuPm6M|!`gOmgnF#l+r#-)j{`0Bl}d0u1kD9}4{dlANV+faTN;NVv> z<*2ZRoKK`-`c=iy2sT?uaP7?B_m7!YB*+Gx-?zLT1uO&Keyi=n;n^x^gO5Gpt!3&#T#H}(7wyR$~9ay`bm=OJE!X?r+pGE zQT}qa1mJ3bsRIRsFKy9Fc0^squtCCul5{sLLR0@gY}>_YXG5{s%P^WE^(PA+iSP)Z ztWTjhviT}G#py3H{ztvOXh68ffas6C=%}K}0W~_Gnh>mI&B-%o(#0uY(BFSj?_AIk zB6G+Smgm4d4=Sw(c8K)r6YrV6H+!Vy>&YisG_t}dfIOcH34-hJ!3-o>s3vmd4}PTV zgM1kBHAI>Ba3j7}@vr4%FHd{ib}52X*ak}p?WdkXfY9r$s%3kD??L(*K}X_q$S;{! z;1(QIf~3pG5sPzn&4m;3HP}nZ;mjMKXqH*i|XfW>bFvjg!I4yu0ioU#{$4RCgutG4Ze^0arBu-3>6RXOT}}n` z#1IOIwX;q?y?wZp<5OK4UU7&51y3S}%ZnzPUUoj_a|1gn34&^0xL`Tox9=6-@C^gg ziJG6QpSzVTuCqO7?q#*Nj(jWS-d3Fw(b!*GxFSG~o6|))YXh&7x0BPcEy{wXiaVF^ zlw5jp`9}Kbnvkp}uE zGNe8pC?>K#jY7Mrevr)1K8)El-d&pcKENcdWXxtx&5+MP>T`Fbxv}iF8BuhQy3h~BJX|v=CW@ESAsQD z&UoJF{sH}gX+%_uKP}Y0d-(8M0{!9+B{Xkzj!aLlBJm9#PUG^SpeiXD`e(d&^lV=% z8k2pwvX+ga=oo(toLs^P7eOq|H?@4<%Na$&yj1|1vu6&20ii)Y&z7E`VIN?u55|T& z-z!SrZW3-1g{&D-(Q>fWy&0~;B)qk3%g6AjJ1!VvPe>9?J9EDESe@2}dO|lZgr&gy zmAse5o$3WjvHvA2$EPIgepVIwC`)xrDPO9qYe zva{UO#%yxey(VFr+j(TWHXU#M(jnz9rfmH;%(MK$FP@{LwQ}e0Ng(ZqRkd^S z{v$A)C)6z6-~sBeF6>YIGDzr;io~jqWjbn6yDa7({wR}&ZkFlRPSwH2HxL9WO}#WM zq)(5nb(^h=%19M9n_;Si%LEF5__5XT`FnqOQZ1`?1Ce7(DRo0811!0aW_+09Sy8nh z(KYMGzj$>ukx5W-yj|X%V$M$hMye9v`%H8R9yROE3m z;i7LJw(TV|pXF>^x$wTEb(vtf)C$8lon`p(-2zv&b4f#X;os`mwLZaFnxvl)@AmL1hwjUjc zhfThW|40tXsPO<?4A-=?I-Rp11*+WPq$SmS{`qfp55-TQis0V$WS>OjLh%t0!N|K zRVYiR2BZj%Jih*do@s(mo%uX6dsF?>_d@p?|HH~$ohaYr~JNu z1j|DbL8kBSq{GAdOQmG4G{j$#R1^Vt3kOQk{Rkn7LZ4hS`yNy@{GCv{>UZ!6d21cVrPHbpIUciCa zpaT;~S6fsi5j6ou^BLhVwAk52K#cCebEfhmH*|XN!>dH#;x$YK^8H-q${9Ccpr*{? zS(w#(mzC6|6|X2yDm(jEAe@?xAa1~yz1GR|7>81o;io2O@*$P zX-8_{s4wJnyu(TOMbOe_-lyXL%?J-H6m8I$IZmrdJjIs%5Iot*r#xBFnsSquhim!+%oqTI$);fw{rMta zHLc?L$j&)Z=V-D$%bx!%_;vvQj|b95?mb+^p#S&~hT69jDJrfCxIE+DM`Bc94?*7K zcv3-446bOh&W6GuGU%I8+Aq~C4q%0ZO!F?gRY)Nm^|^lgd(S5m>3 zM@N8<@aMwO{pQy}Y84$%`%Epi^p>W?Uv5cqHm_irKIayoVe$kZ*Q6|;$sf--vHD-w zb0FLs(rNL8%Umw2ZN9Y+v8@>y63T0Kp=6T(k&NwlRg~{MSTFzh&I#>24)~rumNjMf z@IdPY;Rq0vT~+t^A`KA^jPWtQwX~bSDT!hBl+W))NKKX-o!!<2T`u7*P}^e(+$@lL z-85Em0R;MAc@ee3y9DR(Y_M{W*QJQt--fxt)_JAqQo}&JLSSmlG=zoAnO^YZxjs`%-tgYSVf}+X5*k|>1NM^U69|qwxbPn zzsaB1x6i0RzbC1+?^36;Prq3U$ud%LkNv_OIHOvO(;^zWr6d`ic~c>hy5e@RgxDDY z7G~euq^rw(MB|Yr2_$1r(#0rjMv*MviHhwjC(@DDkcRrj&qVxZRt@r56F5RP`g{B4xNna>P-*s7G=1qWWF7QJ;gFVW}-xp>=)E>{v?#9IIqbeWLlwm)PDy*6F>r$~WV>)_&;|s8M3mT=7-lnT(9Iebg2xJo{;O*hf;T1Rzj5fU`w#;zc zA+2t|bY@wWKsC)Zd0}LSO^1LbO(6Sn&-2O=@*p5r4l`Kb@SGp|V;nbiH~!sKO}re_ z>F(>iJFRSurp;?}5@}Z#<2XhnSXOVNC*0)lH!Ds-qvVsGiE8C~z09MEOr4Q+Gq0XY z#%3HAv%~J(_L}BlFv*-i(Io+NTl7~-EVoAO_q^t@KkQ46w1t}QNBgQAQ3&f1jqOA; zWVIzH-XR)vX`=9&G;U@oDk$IV<5A&B(ZMv#u3N6#kL>F&KeWtxFZGx8ZjCe2fVj~< zvFB(z`>O||l-7$4!w#**xLKCgp_Uztn%~v2)%~6cHav$$hj}%@U@FRS>fHst-J~yS#UnW+J<#v6Y9B_c^-o!XoSjsz&e%>56_iZ2i<$nq^Ie>*TAa zW)@w|u13QJK;VnH{v7-Y-S_x*=gTC7Ye-NMA@I)9{~7xyXO;c3CR^)mxvf3{`*gr_ zYX1?N7pLoH96JS<p0n~xe0V}8A`&oZ+4d986UmUlHUjDbI-}H_er{;b| zW|rQwiBZqVIxxw$`rE6_%}h7&KKz4ZS405%Rm-zQ=-l1F>>_NKCxc)Mt~rk30N^(9 z4KocY^p8LO-x~h)!rx+8SNOrmocE%t)1N&q@uh}2V?37Rb1A~*zp4~tAedW9?UMG_ z=E5w{A2It%BADl+(YQB@@nObp&(pAX!rw|eFV$t<;rs6U&IR?^y(rc19Ebm2(QcN> zkE;_qa93vasRr@9TMD4Y2*0cvq3DwN45ZG-c-fHv=sw=!;RIzSwp7dQVPo$VPT@8w z1M4>6E=$19aIh8O;4W3cm)^mI+WpcpyhP)On?u)7RotdLW2*iwFm;JGJ$)LmWV}3M zzsHHd6`O5d;IkKmAnhB(EkvLKlTOb|g2M-cuNgxL$$lnqC=U%%RUdsV8^-63@pCA} zxI8I2UhD>zl|JkP856jvEYklf3msj#bR;R``j8q-+fUgpuBa9#8e?F8?F}ESLqn_5tTdp(G19bK* z*9^}X5kI?Oz!Q%>hlGV^tV7QJevrtnBEEvzwF5*E0@CcI(ltG)elD3B)bNSf$3SX=+A-`CdE{k%x^1)$Ft?0E z`~+xkltQvOpKM`y9?wqP{s?1OtOt5VmwbRX>X%R!rVkk|Px57{;@A@mzIST|3WS8< ze}&h^g#K$UE2LS8HMbY(g5GjZ_O#K$zN9Sc9|b@3lD zt`*%yzvhq99&3_*C!zeNDD@VTQmILoYp4j4(rkRWr#xd{s0C=hDF!&lawjgw{f^>S zSlWH(;FCwP+IuC}?Ha&4Z6jE|x0sLDy5#&P@eh}Rrp4XZhI zC=U&tZ%^6?I8K9;e6APWDJBBVR0TN|&p+fs<|!a_;7!=Yn9TWLLHvHSlZBle;7zWT zonRqfPmkquc%cGmu}U^DHi<0L?e$Xl^88^Hz_g~o!!#LxlhJq2kz04HZE~D7@G8V5 ze&QaQ|ASR+k{|X+Lqd*j=v8y_A(PK| z&pG4`dPsP?h^1@$Ga@aM{11Vy2ODP$?JX6#mYNF_Z=Yqn5NJUit{isOn%>Y;e z^p5!h8>xAQv0C>eTT}f<*Dsf1`FFJEdja40Jknn(0;^n?U7s(HI=#(m{m^e(c`-Yt zgmMp$w(rz49=pq#>?aR&FY z8hA1;sY&+KumrHH#AwLzhCQIWOuN7Qc!A9DKRpJWIzKLIEIMNPzZIY^TJLJ@m8$q} zS8;+NP9vPzrVlW~3w}|ZwgkDeD0I?tTPA*k^)ALzqkAYWvF^(4IQOzyJSM+!Q7sCK zI-AgYBV}-J;GT7uC5);3s^iaKwYdm5_JtE!b|w`Vphdh1?a zr&x^3BS{NEZ3vSw7XbsMkKQ^EShFO$>C3G)V-HRsypmRzBWcdYsA#hf1>>&~ko(6k zmq2YrqmrYEFW;j>#g>#?rh6FUnwkLbN$;l8aa>7O+#;^yTCNLM#SXy~&~#idSO4h$ zGz17Wf84x=(Jl}!S*;`org>GB_u&=~MPF4hz~6mE|BHG`ngqd@^UsUG=X)z~)Qx3W zo}c>5^=Hz1Nr_JmBRS%dJZBAztp8?k*4e;Q(A;vt#y?y9+~mJ}tyqH)f6X?*6E=(t zp%2MS`yVLV+Om-2km(b55=7E;COrRp)Y(nh#%J$mOz?`>rN(lUxPohs;FWMs-2La| zx(OZB6|{kzO8MXT3ce|EW8c$OB7KghA=+WB)WK35vrCi!u#8wt-xp{BRKZlT(19dC5~l zeh$OV8qG$A#Fw`X^C?OEjFf$mfD69+ZiID|D@(*ZeDO*!?!WtFHra||kDtY9e+A0N z^=;V$D`<2dXkQbD)bnCr)u~>dQFR0_1h#b2PGO{~UJmj1YrPI_`6|-YP5H77L-{)- z?Ud}{`U4@wU@Ij1Qc`!{C^XgOmiKUH0bi57@e2-MZpQo#Xw8^NhWbdozW8B-Yt%K! zAWz}NG%&pa1Rz$|(>|?%&KP%reRb%=B`uuw#LnZrU+<}@6sqM4u4;kW5g~p(i`L(c zjSEUoyG#N^ZEhZ-SK>in6G+Clo;52%%NDq=D2cFC+pnSvresHLP#pa;Nb`+FnJ%AN zF}BPED0%)BwssUQ=87*>6=o3e#}1p;ny3te_3??liwb&~4j0Yz&(CR#rvQd)@SM(G zOTiMtP@@+A4a#HBpF4RR zm%v!Gq=i&9+GW$fqJeZD0+*^^o%87~lb8nZh=s!^G9mbMpD=PwF@~Gya~as4!Gl`T zDnnHqg-y`!SD1d;fWCEJ!mHS5z$F1iM54$jf6NqV&? zaXfF&uN|`YpMR%1-Se$2wJzH`ovKeVy0u8>M1^xA6IH`CtM!A=nZjr&pRUP2wZmt? z>{?F#em3xOLU>&$u`n^n3f=nX<&`Pz{|3?Fi(r|wfRnZ3;vO`9+32wkW5e34m`}GV ze{}-(LUkX>x8IwC;uJbC(`CiRSx<#Vt%G)iBt8_v9{R_RnFdGL-Pg^&T_5#H=fBW| zJ-;W!ZB9I|h*HvzfAy7?hdVJ}=b`X#N$jhR#Gr$H#Bq&zn`n>MqN(`!9&o6?*a7T0 z07c)tkT8YR*(1pONoV{191z23VBA{FK+UvE%qK{?E-gc%{`fGi{ z)i-?V{<}~DHo1BtY|${R?`i)LziO+wI{yM#;%8qwc5f1O`WCB^l%tq=h2uxX@eVrI zY1I>NJjlMhz)PTh_vK-KVg4NX9y>r58<+o%Om9A59tVcX&ReB!L8)f_`|Qpq3EK+(`aFu>bKQX_kcIB^26$Q9|fcII(&Xd;~cC1ZEQlWk`dLO z!=MHRntKB_`Wsk8?_oV`kB!9_YYv2%9zPP((KbdwQ+7c;L!B1Gn(x`_I-k4)0l;A( zQSO(w$;iEQ^Fex5g6`m`;PU%Zk2wn-@~AAo{b)e&o0ac1t*19xF7A)@Bd%AR&BJ_r z(khRKe&^m(BM;%19(xZUyw$Ty|6tGkv45Ps_1fo}+_D2ool5=N2UWT0e=gK<_*!9K z6aKQR9DY;HTMpLEt}AWiFa-tOtPh%qp6}QI3YgfyZ3?-==lFFuSK=ah&&Y@gQuOb+ z3tke|=OK9$FL~aRXjqss@5?l9@rgJ^OUCSt4pRGo7c_;=sPQ4MXdc%wyDv!`AVG4J zcJ48-rW{EuyGOH+CHRq&1>>~_S{Dhu6IDTx9Y!`sr79$ zju+G{)A1q{<@rDl^|!zDE(NpdH#|N#-7CKXHOr3m3mCs-SE$Q%btb{9W{8tu79GZRJy9;KEu52>8b?Z3y}ml?)HY#$YwibSI3 ztZuwldH+t0-HH9^cMj75Ubne&Kjg25D~55*fk;k`PPUp=5)ONP_uCFkaYW(uilJ{{ z@0y2$6<`Lj?!}NR1h2q3T)H~gp7j2X`q5T==AUkjsn5|B_Yr@J2j9|03B+que0gEZ z^-+5aHb^!KG1dp1ny_AO*jP#4f)MrpX8{bB;O$KV0hoVoCo9+Y5>sv$xNM&oTp5HG zmE(_K=X)9uHT+4;`8^`t+7A@}VeM0dpgn_}@D*9 z%4b_It!-G%Hn>tUR4D9;^(I>J2D~x{%OyYVk;w z#r=FUi%W<)A73-{kuNvfS95A;iG6vksQkakKM^>t*lM}zjtU^oI}&qBWNV)^6AOHz zxF@c)0iL7%*x-(ogNNl(Jlm~L4yMq-zm-GZ}Xq!>^S#*ohW|mnb>S{fb zt>@V#WxwJ*G6CAYS|%9bY)fxRS(+7W+t70C5lj|qJ)b^pzO(f49i6MG#i43qr#3s> z1%P7aY^V4hTlsLS;`AVL#Os@b({)KX3fti`#7h(8qQ7wZH?+vI&UFt6_jCQ8s&L~j zdo6%N6SVMQC>vdxB<}n~Ze{v=-baT9Kg6((Iuyt7`WT&e-MKV1lw$^RGe+dc|SQnHT}6A$uQ*r(fvfcGM?a zp((aqEX4Ic4mW|$s=!O(hRey?&HvsFQfr*XipaK&70=OYQ^s%;0gLuk6XuxClcA!u zP=3xLC^6TRjU`*Q{Wexnl484?+XglK#hsA%=+~Y z(z(hdh&ESoUs&dP`Jjdgj_ih@0AcPU#5h$Z8o&-OdI)XviX`lJe}H4%HBEp_K$>oN zmjE%1=r$mb-Pu2L@UP^$KV&KDu+IC`q-PY|_APMy^C9OO%K}-WoFqh+$uyo4$&#`Gms0k62 zDcckMk=4UIfW=18L3J?mTDXflMxu8XKZj|8j+)rA{ez5^y*vf)ey`&A?>IL+&Gl+R+6*z$3;A^(X@j6&AjYx_GD)@4zk%|e zK}}@l4c8$b+wvjPxXJ@?mGU@$S_q#)N+muFrnNHs1a<^h)UMWv8%PBD07bG&n~`E1|BIp7`kF-Qoi8%jJjYVq z?9RJ#E56|##yc<-OUmD4tJ1F>exA2rBp2>esQ*;gHpSaZYx}m1)>1~wt{gE-&M>){ zUcY{^U8_^bMsp9be64u-+Kua&95~pDXQL2aU2VHw3 z2}eH1)fgIaZQFrA?zS(q{3tfLnU+i6;<124{c|uk12%(3 zh(CIp4V`M73-9&6km-9#6@P|HMF}La2F^F7oAoUfJFGHK9E6Oo z5iE5Y#oLJW)7@tQp?#L}@$o9)Bc)qsxlQXP#)B~}k{t_KV2z+i=2&X%K-hda?hJa$ zWEo#ko_zpW?RDPwOjPnc+n*6MR=D+k-0HQI{MFqZ2+uatrXTW;4&-$(F+6T_Ned@x4g=lXUAgYIEw^1uK{u|A1S>z-TrN;frE!1R>+eHd?5Hc0UbHJ zFE=AxBbc|^F5nNaWG&~z{9D6il`n8aCPM9(nr_pdlq0U(EZ_(n^0~ZN#D%aLA;9R| z_5-PcdE-h{1%~G@8*0>kNNI7~*?kZoE^6)%k$D}WnadyB>$H(LYxZI_+0$SVZU5>1 zh$3-KAfLjpT05-g%4h$SdOB&Cisy7Fwi{y1qTDWwniE26=%#8~LnN-Ir*Wr6nwC4+ z>)DYYOd1N7fHm;&wSM1Kk>|1gHb$SxXs;TS^`oE%PP;LKXqgojjU7}d&N&i86d6Pvrdq*>5k4LI)(vsXQ%5xg zk44z5Je=Bt(7bBh1p8CPC_=(>KdJcBM08Nr)FhQ)7*`P?4c0HY_HMsumO;_p z+?iGi1;5!NU?xz5B-!LaSg&A%;G1e)?11zoX?x8K2kxY0$FpT_?Jvurgh~tf0c5Ug zXXi$j&E(}zB9Myv8-8rUVcJKfw}Fow>IT*PCjezXVr(DK#gH$r*bx6CA6R>}sR5k_ z&q4k?t4zdNHiKjb7WI|CxklOz7!o8uWSS;`LcIuoav0g8YY#+ z>1qO@gNVJ`bG;4$dwke12???Z)m@)$|Nr%9Ug=fJMX?&=F4My zC)8f((70gjx|+=c1md(=6OKgk`r@jrH@egoAWuimmD&iuD17)O@r7=_CMNV3PK+Ud z9Hf@;C;tg5>>l%7qR<4w!OWdgM*-LuIf;OXgPPZAvH=6*th6_Ld!IH>02+wDLL_Jt|gjiz#^x%MrALy{+5AM27x+jG*d*dXquk5nxi>@MWW{2ebr&s3x3fH<( z#}hACr2YYFp_kc}x%}EJ;0-xQGvsBD)HG1w#EYmXTf`4DY_&_WPvb4V#+Qy8WVZ{m8*98tga10Yu+M zjZWzP$K1<*YkElv*F4~DqL85dDC&4A4O-L?#Z1y5+elcg8;%BNDVN9jBN-9PV<5bfY`AbMO z!=A?~@&BuF-qU;R5649TM{GdiLOhx|UI)AT&VCngC!d?dA+DSfK1dFnKzkf$o(mns z|JEon9fGv3`pU9kjqsa{Q*l`4=`8k>+e9Dj2=lxq@OGXHlBm~i=)Dm5WLxiFzy=t+ z7tsBKdsnvmc1!N(j0MW#+v}`{0Z7{j++J(|0lWar7@SA2nk?FQCxv|8Q!R6&^N}Xn z^*3GH^UF80=m9Q~PhnN$%&*GZnUzDqn9f6+-ra{%^D0!fDZjg$!VTz^8NzVuL_((| zKs0V*f6)%YWPi8nh`kx$McQC~vm6{i!pCUStHcvO?PYBLM%MQM=N}pqHKYw-3&W0P z^tclMry=iksl3%}8Q=34b3LqU)`Cnh{V1A<~8Uf(ETP`YFH>)7G(E&R$ zh!j)oKGRW|81XjmWF6YbD>W2%giw$!7(0XTfMrNv^QK(LGzS=_K--+-^NXhdk-Q>= ziXQa%Xa}tP#nu>jik0lpjt>K}O@K)n&Hp3nEaRGdCNAt}-=As`{$$fQwH>23ij z0qK~CbV;|925BTEMoCH|E!`nw)Hb$V_x{iG+@I(D_ToBsZadH8cO2j2;4XGH_Anw@ z0l>R;o-{SJ`Ypap`W+}X7(=yo!S^Gwclgy<)PU+Tt9`(Pq2FW9ps0)Yt{`ERIB`0T z%ryD4=871TmH}eD4=-oU|By%+i6x#kEt_JSlqi1gD1W#7$Pq~U18D7JIH(=>%;-j4 zegXPPh{kbtlfgu(7+eD-kpc~`W?3|IMrJhYKug5!3P#+c61uHJQ(3MB8G*=Fo2xlp zu&Eqh*c7ekhpItSRo{_d(tnP24MPt=r}uD-Ek*YM3W-L*zf(+_Z)?D2s1Zx#N1D`h zj0U&wW~Nc;NQZNdUIdJVU7sxd6Kxi#eUQ_SPuN`^IP-vKH1r1an7^(wwbNHOBy+(+ zz^sX2)E$NK97{x>nDX{a_Nbp|6=X+#3a^L(t!(S`)$gn8DY?>_n=3f>Bi6{5UiMH# zkWwMr9(*f%)6!#1E)Dte6ip$UC;nC7ReK?-PJsD(BSf3n_>!Vi>x9U{M}eWZM%F7o zQGS1&Emk=|mA;Y@+1B8DgPwHEiVxWno9W!8b+O^l6VhCbWen?N7>l3HT?-3#L8q?U zzGP2XB1Dy&_wp?B{^ezV`0h|e#Jp^E?!ekMl5Qkf2UFMQc0A#Q?B3nnQLcQB=F9)@ zncjHt0{xHsRZb&ynSTTANd^z|{161+f-iBucP}=<$H?fzU+;&mmm!K3Pl0HJxShEV zYTq{g=Zc4M3m@ue}l*QISuIfpg7l(c!=|NX#4Vi zf3*Y%f<8%`??{HB%KHH?Oi93NR(lv&c7y8R^TuQzk9MbGk!)|o3c-2Z{UnkVTfC6# zmv1NL%JgM`@Aw$O1qdIPvFL>H{n@nv4N%37eE#!5 zP1)mRn^)nOF7^8My=<)DcTf;*< z#(Wf>3-&x&Qa5A+FSaG+z-e7}#?u*pbg%>4Qjni8r&3?`Ml#_rXh!!kO}w0+(&B*7 z_>o*3Is*^ap)17;{DI2Vo4%p-hvXtNS58t@}t$wN(s1}bMhF3 zi`(x$im~yb=ayK4BG7%inMNoBZJO1H7D2Fe1j?}lDWP%lRzVpTZz(?K@>3fE4vlO< z?Hp!`(_=FY9fw};`Do7LfaQ6{L~>}PA;p_MzwFAYx8<(*>-+ zs9a>%vUa-!XoW@=xZ%stPr7HMAnK|tSymB3$#&*@uY_6mRRB z2(|MdpC{!`gj22-(g?aqRsk$%io%E?THg8+_x2mp=>Q{648@zK-N=!d%Vm-4wzXr5 zclMj01?w`~^juAWjJ9zd2v&lrGwN+26D%X_>kzB2_(5xP^^t1NNTA3GvaUrjG50Qc zU8uGn=X&J>Jod(hriMK}R5^w4kL1=_hlA~GNBeg(gM8Oc`4UM;k!?d>#tguD`#F4C z;JE@;IZZ}|`^8vPVGTEXue{PnHGIvxcS7+8;L_ygkE!6D5A1sFt6=%9@hrlbe}2(& z_ET`WM%)8=7sH?jkQGDdBHQ1UhX9oXYj;u7H87%kq;1m$YgsDei0-y_xK`?6Oz-ri zNZ|x=H1>iRXmF;bCMu+#Kie&+KDp2>jx(=j z6(p@(YnpePX2ZCknA1Ov(}PT|KRkJo&q#&WvNw}bz+^Nip6ptss9HOricbV^EQd+&4WD?KF zlwS^uIt{}tFWP zb*xk1-jY+el`^eWKudM=wOOO2#~O6!iBL+SReRydtdfZrt4@&W|C zWzlIXBxaMh*on8UM{e(}_v$>OMbe+#U%$VTho896ty4ex=(U_F^!rN>(ubNk08cCS zC>L$`_8+m@HQ%H5QmVDRI)Wv~#``gy*egNvZL+V}n^jcSquMIW?519O&TzDZIs#Z4*iP2l-PY;Ot5K%jN#dBcuWom8(O29|591iK$AcXr}PBt zcd7AknMieOEAeCTS1_`qRuSpS-G_krWgU#|$QJcxno;OCLjO|tlqW}Di{>`|RY`0d zcOZd8|1E^Uc1YKgUfych?>BH7A-L=xmS)G0VU6MRXCmK+nsg4bG{>-|C~Lsg*=x0p zdCy6g$#kL0BxL>B3K~NP{vSQg8Duk05-PQ|IWQ2`OX$d|VE&f*9#2e6 z_s3nA9m*!@k}%A%renNd%Tzx6*;MZ*uXAwR~qn$ocKew>94t`j-LuEhOHQ|GYp9ZiuK6D+vA!QmL%~d2$Xa4!g%zz|n z6tLleKU*AG0!hg|@Y|%%b3;u6-ygxBB|UnD;mrHO@DNpup+UJ&s$x}vG|Vi`0sb8> z1F4`MKoI&#oIFIX{*S&D$5+desp{dpa}4yfC2P-j;pKL=Zp=xWas|BwrhJ+1)-xG> z{!7c*8pZX?VcuF?YGOoIP<2VG7}Mp9pMpn0{}(vBe(Q2l-!p6%g`xUMRJ1eqGg2Q<#uY})U=H<#FBY1G+#l2#JDXe-5 zDDhMIo90U({uyyyv-8+8QJ5CyUjRnvDQWFWD#jBC*U8I*7aLLox&>_Ckl?Aybpr(i zx(H}OEms51{)>j`K03abzeOc`JEs-cuLiPHW{madSRm?@Ss2O9^}r} zfz6h|hTP)4b(zu}g^uQIHXJzi84bgiZvTyc7)4{K?6X^mIT}tJr5NB${OZrH_ASPuJ?7Z@~ z33LJ6jR@w%1cmotp5k;qc2~w zuJZwNTEB=%HJ6Pf+NQjn!Y1iTU7bqbO%b6}85E7JAvGt_OVqGDDI+H08E!931JfrU zcd%;>Y8;@!%>zKW*nM9uI*cy!VKlC62HXmccWuXZ017GQUNsj+t8_idURuC8t&1p_ zB&?Ga#e-%I5~e@mssnAHdDg%hN41qJKngJ_B{tP*Bk{iS*XK;m3r-{bttZk*!SVXdUsr+Nbb+(94VO{3Yi{#yK2Q zIu)eWnD4_@!7n37%Jp>{emZ8UMm|~6@QuR9Nw*)+cf@-NbNxsaVs~`i`vppiVTp71 zpfJ#~=`=_z&I85fF-+%aaFqv`gF{njV6k;^zCDqTD`ojc!#c5!ZD0$CT%vrTJrN@?py->okv2wMP0irY$&EoxKw z%`V>Y6XT-aaS3OWfAmDU@EGJ=_r|znnFrkQ!kER=3~4Qf`a$Deh9AL++YD9z^vUAT zhwj7p-FX^A_RL=M27zTVtW7D+J1H#oV>&g^uR%&XZIdrAoIq6V7dkHTm?pc_*AHD> z;Aqs0rYaT*QD#bhM5dFM@y&`Vh3|C3{hcB}w7-r{NWKj!FD1XwEn}Yuaq>>r*F9V( zLZ2NU3#BtBzMvUviXW<71aCs&p?2tK7~eC&k*UzU3>#-{14eUWS{KWs6-gxYoNQ)* zN6puX9>0i-vGa>cdf#52=7nfJ72b~+mno7=89`1Y=qn%`AV>aaHaDjDo!t2OH{bcx zp^-j+XUSA@u~V4m0ErDx!KHgY$n6#S7A`drW(pAIsg6fu1A~(mZ>}YFJUiExti@>J==Tkg<-I zEP$_&njJQh-0Oc)13On^&5^P~s7OTl^-$d7hZR>Sao9I;dpV~RJdkM4OGlFc<53A+ z2K{~DR@VV7M*Scapq|AIPYS-9xBf72{Z<+;=A<=QygdUe81UU=yt$;e^$OAx%kZ*a zYEh7KU?O4GQ=&^1l>Z_d`|-KMAd*t7ICVh&Q9A|X0G)Q@zB~IE0cn|p{)I#3{>7AQ zo%f6_*z!d8=)f|=eAYut&(DCQay?H<(_r@$a zJhe+*6GZ)oVv;$kDBgkfQ)CtRtv;1%Wj~(VoVNzuu0uw`JwY#Uf`U5kN*l&52R%>B z5Rq-i>QDAu@h&ka@oXOdq`x3+K&({w{)lD<61|6Z;#QZ~zug;VzRc=kn`)J78u%`FF{@I%0d#Z>4afpTi@(lv=L2f}$`X0<_ z*uQnK5}?Vbav(4V@@byig9*3Gc3P8$v`0bl7GdqfM&@zVGRBOAYy$ABx_J3(=Qp{k zFzxptv!XwEt9**|#}2?h4!OSx>1y_lGuxKIPu?Cq%{M?q@Xw0z({S;-Jp?uxDX zSdIHXUnv7G{?q8b670+XoH`Z#+Qo5(Wd|HOcrLmigv<|qV1I{Mz@AcGB=EP0BldN> z!&;H5j&CzqmQR_bZQqH~pyoj>s949@n)MCuWkkX3378mgwE>Hlk&?B3G&=^ADPnOm zjo-y>teax@kUI?awdpux9$>=lFsg(Cr>ZjboKl8!gyZNNJZNasj5l3KR%wRYN zxXnY1qnT&-)q9cG>R-+xuXv`*cl!4e@mmk&Su z@+v5N88x|F#?4;p-pM|=-Gc1>T%$^ElL~W5HC-Sk$5&WP;arJF7;`*Z0Uo?R#`if zzfQ+&d7#98@VG0LBxxnadMc6#Q9%C;;~o)0Bf)z~=v8&bd)Z;m@BuJBh{kNky`CX# zrSc}|!5vT?M8fYqizG)CtjuXjc%G=TXYTdSqSLr`{M$@kcC zkXUYK6s?j%??NxOWyOe+$q+D()&=3*IMO5#gTQ@7j;dNlqY@p1IA3Nb>I0@NAOO2v zNPbE9LHwRVdjRuzWi6ByLugI(qyEo)ovze*@E5FBn&Lxzt)CWU-QUhTr}L9Se)HbH z-ro8q`i;DdX0wNu_Jm8pL0+kJKHApTm5d{;CqOgaeB+0IX(irlc64lSau&rFjne^5t?SuLkgT{6%q}fI}RbQ~h@x zMGH`Vb%4w|j@R{LywIDyfGm-yS|-uU{!=Kv&hdAJ5|nK5foIU-E!rB`axu<6}C|abqx={KQ>5g|^-6E%teksll*;9t;ra9 zPcR-$A}9|rGy9&1D#xS6Nkj(s7g`&!5^_FaCeK*Pe_?Gtlh@H9!26b?$$Oc&{|m_> zJ?jvW?UeP3Z%lG1-V~WCjf>4_oNAKUhag!K+;)N>q4k}Xa_(VTlsnok1kVael60My&eQGPh-3WLElUY=d+?n;KWIu7E9w&Tv0_P8rX{qh z6QOX!LGUgDvicq(v_nZF?udOeZKMgqFC^8uCo6(D6!}8xoGAbsaUkJ-Cl_S8qe~sM z33kBG7SF`eVSGYI6P|HhF;7&DXd3>k{unspM!J4hDwNKi9oaj&v{YSCr|(b`3e6GsS?52@K51yQ##J5|2sc#47D-XwO`aN zmL6E8>_g~~{4?q-?iBth^}kmIb7*$1e9J*;S^D2pi>skW8-V|kFWr)_vD+?_5zBvI zf2j-bmefT7{eU%UD+^|^)&lRr+_(nU+)KeVk3!JbN3M=89wZ4$XQH$iYEL)jv^8^A z{Vv)I;YQ4{GQRT`=Lg1aJNeg$RW)R7hm8<8(NB3L1-dna=)V2!;jFB_?XD#+Uww(s z(b0`H=8o`A=6MDFhR1oo-4VZ1*r4&*h&67eZBLCNG82!(cp(<0xw~5D%Yp>=*n6Gg z75DyA>~Z7Fnme|9SoPL>gJcQ#o!O7jlcQA!_1-J)i$P0gZ;L&eC8~cVN;zX5!`9K# zBZAm0c|EcR??j2M6n>0rha!v6(K}z%^ zao9wSd-E!PsEwbd<5#%!Jj#UihY}vC{iDn1TE$)}aop?ua{by^kM9_nZ|rL+F6U&? zsjCH*ib)y?Q+B+qAqIN00t$*U$~SaQuXHg9$Ew^**$J^hqd<4Xgjg-QVmsET2{ z`m`Y=nI#`_r32COZ#hQ(nmcki=1-HZDobys{FYa)ChVCMgXfk2%D=o$Pq<*85JL`H z14Rapa%KLq{(weJEP|m+);5-=K&8SH_lxwQowPE`GkGHdGxvg1p(2+Gr8Nde5e6!j zX3!PMv@WAP3so}b@`Qgfr!7khU>fi>AAWJJ_}y2va4Y=+odR!yC~DdGJ~R?^(5El(DR^k#Kt||?SPB&-*A*! zd{(nHnGl)fn?0XlBOF!3Vg2c8vG@8uw6XfglWp&SR`{~-^WW2q%^o!~&FQ~d{CND$ z5QrvNYj1WBr;w5Lv0!y@;KMzA!+hYO2){TrlHxn|H4S3T&hW_>pWRBM&)#)&6TNjM zqPl@vG_y^~ySNu=y#V%V-7R66U7fH`!vZHBO*5k$@q%#|Z{@}Pw_;~PZaPZ6`pITG zGd;IB1zIg`H~88A&swJVR+c1Q9)DXJ78&`BF9gpn&GqXJW?uf`r+2k3V{|Z?anHs0 zYo|0{s_{>|U&M~(3qj1Tp#sUx&3|aI|9^iyA;vAV-D|$y3vGHHavG{h0FpExJZ}zotWp+y2B50rU4Ft3uxy8w|(iMX8%bKh*@VJm>kjiKi;|@|A5X z%H2&amrL!_nIO+S(5J-Mk$xUOQhWsmQriS|EY3;w8gX_l@){#`%8!!j#4s3}V{{EV zvMX}1n4NMC2#$0s92iisNJBcV@buI2c>UGbZWhHyKm<_25y@vx9*)HU3|WBtK3a24 zjMNjHfAcyz;#SI?S#>}52G@a9TIr~^FTruK7t$Byg(18Eo{X~@+H-f zAOPok~g z)0ULc-yw^M`WQ1^+D*1+S5KAC_i~GWjp5 zhCNslYB52`Ctj57UD^*uGghpAB9izXr$vLWn&HQ7|6NbhS#abchyPbV?fncdF+}yy z4>1D!RhaoyrH!-*Lh_C9=X>CMxDdL|+XjxZzD#DOU&FrrSc6RL;+wp%NJM}q+GkdM zChN(}vPr|gcI!RVOOiUz@$^1}Y##c5mnZ)l9Sf(GxB&O~5#$3J*!7r0K@`rCXw|-x z8t>rWnb?BZijVlLa{-E0C*MQnj<8jDxr?Y^_p!uu9M|HQ1^Iq!2_!GY5M-SfQ*;3YXu+YY7e)OlmzoW@g4h=$iRIL^7Wpu_y6i}NeRA8-liMJ9Z z`l~@5-@wm!@lKUE%bpj3N9Q)lF^Av+zU1iQ)loQYtxpAm&8>dYTm}~Zg;wI!6EmON z?b<}3kFMwrT3OLMKUA5 z^iG5(1bmx8s={^;M*gf~i#!t3c~M!?`aHLI&-ik4YXJo~WeN!M+O*uckLwbRqqAxJ z(8H~H`0?+d5RQiSR7P{)(qd-FaCTOiz7%|V7P#8EA5WpqdQOSFbW$k3vD_H!H(KbH$jzI5)oGeJx%4X|Jh?!w`+ zUc=cf64q?XS@8CUCUK0>lYQ_1`bdfw$-~-rjlUGE!_>Y=gdzBVr_+;^FD|(bA+qSo zb5I}X+9x3DiF{A%)jUTNRme?U>}i0Zmx+} zLpyM)^5YT6Z?-KBwZM%`Rs>bDhG9%M(xgI&kc8S zoC^EoU8@PcYN!L?HT;=HR-P52y?-a!G@s+JmtbEDfYoo&wvod_fH|Ni9NrhQYI6OH zMMcbYUJH3y4iZ#n_EPy`VsTzMY^M{sR&WL3m~n*x*ckC-Y=wCJMYuf1ZQ}`t74rl1 zq<}IbPTgr_5qdM734iF_r3TW-uE*GG>l_hghIzYv*|k0pg5z$;hmz+(im<0zJVLf% zgYafq05N|)6Tq}K0Qn(HDH+}N?j{6H!4ob;KXgsoG-+7yEwGdww)Az3JHQ& zE^=RAJc|Uvb0!{cLoLlzm&g_v>dy5(hnxkcl&2({rJ{_+ULQ7AreUa@)Y}E&`-a3M zYT&`h@Nerm3Xphsb4%7_aC3LCVX>M^FF+ z5TyzT5818ev|;|dz$!MXDRS?|qO5lZrj9*#(PH~9@hmdQ#i6_I5gwl<8wL&EPMy{b zUh1oaU~0)1phNE9fF8@1rKD{0_kS(?SyDVh#E?J**tzW0A%2=ExTL#m{q>lLUj;v^ zE+Hx3EAx*D5J7O2iRGV38B+z~FgX!Szkn%fcT9Qg%Y1OV2%&Lqr(5vlSs#h&*DZ;i z&t=c8K8Dyw4ZqPOwz_M)BQ`E&$Ndz1e4_lZ8|^}Pr?SV3-q~Z(&d1nla_tbtMp3UE*xww+|17{VGl#V}7KQ_lZyNiN>it zEb3-Ba#7Y@**P_Rs7K+(9_f}K{rQz7^>L;0L3Xl}`>8|KNKF!4L`l|jvPf8~+8viP zqDzG{Mad-G1C6xcOiR`tOh*Gvj}JAT$m!gk@!uA+xlBl@F0JYO9E5{IRY5Vavx-${ z(}gl*w0vGQRgY=?a`%vNbgtHvt^|Nz#+>_xC0J_{zZafRS9sRYK8tw3UGRMSi zT2&C|penz#-lPgA?#XoH2U6D~S5E*jmts1gkR8QQQL;X;ck2(KL~y zSM;UB)p|$XQ35IC)H~GH>Q=b$ZfLL}@DW_d2EUR3n*!WcFQitF-qirDC zVbGL9&b&acOW*k>Mv35}t+Qm;t$yik-bmp7w*H%nMaL}5T?_fgfosTpM-zA>^`H_) ztFs2YSQtGz6XGhIFzo_};c2815DXSXL2lE$`lJ7D^e07Kc#pl@W;uKKz^~YPVfcPg zXPNa!3yartjnwqS^o4nSwk7-trG7Z_Wd5 zS;r(7iiz1Oyw_5J0G@88lo<(7`Le4ToZo+O^`IO?#{TR=HwQe5CW-&}eCJ~1@i<7q zYd#4}WAw1s(*a-#F^wq`2kc3|0s-{YNT~C#j7us{04LXf2jtafHYiO$HK`ri1UebT z7@PY3slK0>7~4>tx@$sonh$>K2%jx9*Iw3j#xqfU%t|dc#^52((2~dgla@mw)A??M zw^f4e!i)4oJG{Z76OJv!h7UDUu^?>ZNcxt4g-jd7$UCl@k4tXi1>K1{C{3vxhT?YF zddxLpYo5E1W$)W%s_P}yNy)j;qNtv74!8re9(fG=v^veOdP%JPZ8~r>4NOpYKyEPW{|gZ`a4pZ{Ru2 z9}Dc@JJG++(*_u1ouA~_W3X>4v=68VXdkcGE&f(}4a&rp<}pe%0h6ItJN{Dfoi0u( zUDuQrAwBysI=$TY6d8~r*8e3Fo;^P#(+qk*d{l5Ym#4xf_Pk=dr*}|8Rib45wWE9# zW_t1%9bFd7eXSgL4jN43vrpHrtZ3a`ufmqY*xpsx%3cW}1_s<-oKONo7mf) zpsOmCHCWVWAaU1Ul?mW#l?Rp=N~Kwr2K==KU!bth1uH1F0^QNjiA!~@{7)VHBRn3`1C8dqXZn?Idrn4ULNn(+&tXj%;ShX zN4AbsiJF9vS%MqzX`adX834rSgbVJ-K5$;q7>e+_Z3}~a^W60WI0%a`Oz@Oz&TC`p zE`nj_{20_Szd$nXW7I#c;YP@`@S(@rG7d@eq+iW&f~*C|!(Iusl9aGAtP7UXNq0VM z&{5?hWBv45sY|s-e{!?N$}#FKYMyf?x@8h{%gp$hH18c_AbI7e$oK`w>xes~dHlf) z2>jJfz-Q0EV@vKPJ=#kHp387QSx{>9DvKNr7QMuKJ{$KU>uG$-fx6M_=m4S*lP}y* z<%tAIVfzuUDJBvEFf}0Dsn0+7DN2{T=?MDO`l;)?GGJ1HA3j~|HOnd|XXRfYezT^| zm}SX_OD1w_fbO{$h!zm|d^FOWhQc#A(aR$)UMobj&d(V=MGX43Trjt4t zye9a`QWIlAZ2mPCO-6tE^nY0Z2(ZfA3%_TFqe;J$Hi6F_YdF+H%HFRm6D3;nkn2_QpZTA$!j1Fx zA2@BWrZvVUm@d@Bg@E_t4>7=LZsU(f%3nV`-}aI;f}PUmir^Q|^_)J0zrDy6@I@&r zKdMmKOOKMRBkpt|#E=MzYhw3#oD{_Y?seop|2u_Gg*NOC*Q9L~wwuIu1UsOrxSj^D zz2q370py z`#H5P=VC&2N3FR3ilJ>2epu2s@(RLY?Wgt1R_`pARRDTerQdg+xG3A{mxzf^9@Ewv z{>J~EqQ}6$669#6*|WoqLZpv4Gc$YK5lEg$8JvVvzK+Q=`8oR%Xw~@ z>)YpDJBlWJ<3VVfIwqY)m24wloRn+LM?7(-e!f>a?+Aw{Qw-f5b*KYBH2w8-2$KC) z{QvF%+1{%&qs7hmi1l;2#c_8Ip->m6O}wDT&!*;VQ9KVIXVoZGi!##VmoP{{;`Vb@ z@XBS~l=aTKKPvg+Rrg#b0wCC1nDNz=l9jBc zVc`t@lk?bcHx>7Tlj}nHx)a^BGQ|q%&+TX0X z?W!l66k!9Y>0mY~cCSI0kv3i1I|sSAxxCmUy1}jSoZHBw#3Wa}DQ_7aeK%r5Jq;}H za{4G_$cqncj68GlD02Sd5T;XJ)9_{FyrTNc)%>l1qAvoo5k_ga}J>PV`F0J85u%bgr(onPKT7rxJ3nW&- zWa6IR;7}$pf@!u=GN0TQ`oN{HnG$bMn)jF;N4+QoK+jQ9+?ce;GZgU&jn8hDNKN?; z$P^VZH@vO{G9p*SPL&h^HupY+8*}xLI1+X-;feO1`~CqofB0zEroeiNI|*F5KXbP% zddk0DNBA-~eg>8sNgVa#YN%%t{Q5P@5?i+W9hZ!H zPxzKX71$){M@FN395n~?WUGNI!H|Qq6$LU3}+N&CEbjL!#!h z%t4Bf<3!41kNjp85vcb4|CAU-Dmb<=(w7lEWdm+Z*RgbDZ;FcAerF zYU(jJ`tbRVnayFRblnq9!F99ITKB)JFEo^;@%Y!S{ExjQ~F&nCkyr zO5TWQ1+H;?5(kB~-@pzJsF1Ff^F5beuMv73Si5mta?Bc!=RZtB0>pFahxBNp&HL2c zq}HdOBNXBJ0P-WrP#-%qwobKLPp%;-vpMQPe=;8L-10c6iI5_ykR?JSz%b!95vxJ} z2p7~S9=7)7s8^g_Dm(y5a0#my5YFVD_MvXm^;Q3MQNYIIe_cQmrCQEH)??0~9vwBq z3J5H}@+BZiH{df6K|Yg*Q8WA|s4_3Ya%+s8?Z6gDxU!oD9Xh&UHBa*2$E79}i8QoQ z-xr*UYH&QspxbX&8GsiyHK&q29DD+<6`^hod8p}sUjdfzv&Q7&OE1v-l;h^_{#z=u zJNw4lo3B@gv+_JdJC^ADiiw64-(>QnoQZcF=3?ZMGn$?u*!rwL#%B(?)%RwXsz0Zn zObliKl)bFRBs19xX9yS|Sg-1PY+A!IZio4Tf$p2&Y0zx+H=3GJkq5Wi;~qzO_)PVZ zsGDN8SKIkMs<<&X)JvDyS97N{lN>?L_7rRNOH1G&6)0UWnAZ^sLFs>DSU@NH{Q^=S zlwD|u6ssF%>E$n!(Q+ERJNxfeH&^nF(}257LaytYZO${`mXI|F)1Gt;4MbQRlORHf z`@j2;_U}WekdiHb_S0r@eXo55)|8BFa%T6X;E@#m__VI7e)Q%*4@puypx6R{(Eo0{ z(dX-_pe@xdT(@zZ_t|s*F@cj$7dew5M&y{D+l=Cgt=sp{uAQ2;ThMjA%DzXEU8391 z)9f$6NBhtp7O2h6GJO28@9>BEqm0u;h&WgwIiE7Efs{A|gvpQ~Do(r)sUIQ(u}PMc z6GQr`5gS_pJJtx$XFOG4)mm!;tKI^o?9x*uXx;)mYf}lM{<(mJ3Cc>ECjqRbLzKFGitrddqcm{tUJtTsXz{f)p!*-m-f@dl)142U08%hap0J^ApPn z_n{b**#2E)o~;<#(Qtl^_;Oi$>?Y-}9N8SYYT#{%viG+G{~3ug4{B6Ay7K;aVrBAX z7??#CD2M*;;>&r>gi1zLh@W+9GH$wu(9AhzZ4XY&a{!mGQjTl?6rpXfkSCZH>_{cE zOrp?B!e&-26rm+9G39@(3Pk4vmBn?NC?{HvvD)t3;lKXz`M#OrYgf;Duj@H(K;L}y zw?oe~$vv+*GQ&_g2AQRU#lUFzoba*x$ntM+kJsGoq_S{p`zn`wBX%8!TJYXUQO8Ah zZ&L(%G=l>JHY4eF($Bz?9y?Aj3YncyI1~uZl*FJGpw9@|Lf)?lmQk6jM@(dp$n%uZ%z_f%P^@DvYKkQD<+2;Ql%WI0^l0RT3P$5oWGj-Qm%9JHn%aNbTjM zAXJrvmXSSPFPWEsLvc6lV{09sujE#q%bOIk00}0-9qgM0Zi&4GBtfw&y&b-j$AkEQzq{i0xRG!V{S>cwoGLc6p$kC zY8YK%a1^k_<@(F}Gz!n2FQA!sZ@CP7d48XV<-*wB-e^Dc7!*#v*|HqHY3MLg#V6;)o^@RD1P5WW9A*lmGkv zuOgs?s2~W4lLjg2W+17QNQX?Mr5mY93rLp|js}sIu7Skplm-Vf7%3ea+wSjP@8kFR z>vtUc>s}o9?rqOq&+9tR$ElcmtAL$?Ip|OrAz43BHjFzM!2P(vMG|%UO$nU$WMeHF z5Yz>qh!sV8`R8f)h`NODnax5(nM7^RW{j(cyjo8a9X{6+hG6kqWH<|$`P#Ir<*J~n z5->Q?F28-OhzVD#J~t`uB7f46$9%5)Nc}H-y-%X@^ykPub=k_XC8uDi&&9y98H`~6 zYWqL}pbVNJ8a-^3?Rrf;&Gw;mcysP~mih)8@nEY03>#{qN24c9a_GPBC zgn!YUx@EXZCex(QL>*rX z6S&K7AMi0MbipVyL80WL@?qSa)Q|p48Y<-!pU?E_3vX;c3{o7<53K6q=9YN^{8u`> zKMK(xsOR>4F82xGyWq8yC3c)2;M+E6rY;WvdW+Gt{>*zq@UQ{K(sgqQXyj3I>U!k+ zocCKX3?GmFra;bYQqN}7iGxknHP%Ta=$st;K`)-Qws)n1zKu#;v=4(mfdM%=USrY{ z26FdrgtFTVrfpPSouat(xpVx(BwH!pncHTapOV9LS^fju}oj)uxHhAcW{ z?(iuI1>R14EZ1xayk~y2_Yp`myN`xoQwZVkkK8=B8q$CQ|I`q@TUWE-*c{(1F$PNm zCiNf_8?HuLj$c*eFO>?E1UpS`(oA`jW^7KX7xFUcTwv_`m6Z`vX3-~LTZ8ynI6Bww zThIm{q+2yP8x=|pO2eES)sT^FNWXuU4`8Pt!}R8V1=5mH)z(^~KnvYI7jE(tvQO6N zwOctAu_8uqYEr|QwxCT3&AqunzgOZRINLst*L*A4ujZ!Yc{L75YnKR|vc^~zg1Ak) z)dQC1iy9mwo2DzEg@eeubJi@l@w_YIu869b290!f{9>9T8!EU2(3{8NR>`N@SA!g# zoAU({!a%8k|L(2n_}<$2^|Ab&rm!^jF(l;cfL*pzRW%-TI{o;ji_Z=(a_*G;15GYK zfp>kNN(gTzgcGKPyYy#`2u@X@x+5$6@zqdeRASS+AHi~4pLN?Qa0czqqnqfcw@f^`P!WzyB)h+#6yF{S!a_J7_n1%}hZfq>rpRNeH?< zTsPXd#6v=yh*uL?jyC<_D!>q5B@eSpU9tsq|Eyfq7!usjz9HatyCUZy;PadJ803bE zUX5u^SsS3LP+Tl3ePIr7D35L`Won2rcz4NGecQO@TGWcjrJWuBBcY%k@n$YxDl5;! z%AfJl3RudZq4GoUw)qsRrnU;8k1PNx&(&F&eVY#(Fv z7{LY&H@JCoCT;ay1&3_sx%e8HEo=yO`UCE#_Dq7Z_I6nNtr22LZtbFe3?6^S%))YS z)SIPLy331T|H9}V?PG!2&ny$)96YHiPY9-|=x@}z2q#8nled9=x!U+Q8u@PI2nBLwurs3j5k$(kH>iG~C`JLDw8k68J z6knA{t!{1K!9Xk%S1Ah257o~A2(Dn@;3#Zh)Ia?0-rA$`_o;=q?!S|Vhm-Dq zBa1fMHO&3Ni5;MI->xQ)TPHt95}uw!xm2~l~dbynn>DnRQ=yr8NDwQ;u&scYJ3I8YdQY; zP>LxThbq}fGzP?XQ;%31bxoZwFuQRFX`^EKROZPgkI5GZBY+GlT5u9i{$QVx|6RQ_xgg84G5j|UDu$^Wg zz{Kc%+ZZ3aH2@tK{pZ;bV$7%U%XLavCYdWr|4z6ZaKaU$Kc{-Pq|Q}}U>g>Nk}ZU; z%mO#yZNxbjdta5#Z&Sq+U(oQcB<>=aUE6tI2Mn+t^k@g|1>7Y` zXWyfj3nNt5iTFOxyRil&s}mjvaJ6lt&N59mhkq9{vQ3uuMPzWF-sm*_{>I>%d_dUE zVnt)77y-74yJmT6>u*dmi#y=-sx?|gcb7`piYk6z5k=Fbcfd;8-yenEfGzXjB(FS( z@W3rdP1cJgXkBZX$kos9rTym@|A?|h+xcT=$s^lTb2)7P8hfI$kEa}Uo68^Eekk; z83OhK-tP;+Bg!LMTuxH)p`VQ^X-0M+f_s zINhY)1ftzJDun;e0ag6Qpy%Y}AG_dL5W8#uaCn8}W9{`LasJ`P_~?Q3)HuwRz0>R+ z`H~NVxQUQl7p*5HF!?(t?$Y)Ngo47K9?)L5ar#?N?pmMQtPW-*@cE5#Q~rI=-60ajbJlu|kIJs) zl>`NzI4#c2IQr!WKd3!ZKbYK#&K0|i=WguQwK}@0Cc>c~S(~j{<9v^h+volkr@CYc zS2h}fQ^RuKBrXlFB||OuJVJ^8^7(+n+VS-4fL11;FZ;`?6_-d{GpF>o4k~aTURnkc zkRff`xXtW!W~3;h8pw9Fd)oJSc}Q@1!0*bVRlc!5cLQwYdqcr$;u~YDDj+^*NwVe5 z6V+Sg7Lc0b{x{~*h8<~ec=u`Y3|%&f6=K2sgR{?fnqdKkAvIG#3Mt4Vopap`KcHd*)ZT5)NPzN&V_zE)hR% z*ycNDjvEI@W6-S$<8xg^$#HB>4TE4Tp5aO|eS&?aR~fPmzc1c(4yM3&XFJ824Z)Dw zOCH?G5XcRNj1>p8z20N!vMcIDyAB3uF+s1G4^%YQgXdi+tp>2rS9rLmwGAHPx?tNt zN|TscdaDJ0LJ$QQ%9^N{Qb&O%+>z1w1Ry?Z!G7GqjJxO{NxSvV=4Cwl(QHyWtDAXB zWY&Zl`;>KVrO0Db2kHi2HBE0`VR^{f>!LTtrD?b~<0x;V%5nTR>s+v;~Hy?XtunN>hPs^rWCybDfLv^vA{O1Pv z-ksQEaNoMCG}l>6c0pr5qVj=#DaianqPck07+xZ-N=Q}?h!4}!Gm?gTGo) z&b$tNQUrW*=sEQ1j9@jix@EKVNk>#g!={<%B4cUOa<9D)^;va+7jMrv+r6bBEIW)= zZ`$w*y2&^Va5_7UdW?hP1l}7YOU=hmmZ=IvXHC;M6RkoP=S&&5!=FRr=B*niAWm0q z*#z~_CXQHKpbp{E;_V4hcmDyQBv5zC&s%4hIu#W@xpnqo+5XtL?n9il;J%VJs=T=lQEJXZ844sSCmJYZ1`>@L=mF@$z z15gu8xGu9x*zOpsMF?g?!*FB#YJ53|8(P~x4ko4)nqu~6f|7FcS$C^`R&ho@lI;OZ z4VxK<47^p7KNTf;KDp1}Q%SwE@HivC?SAnl>&E^&r`^mtf&E|pOq`$o10$~e47q%@ z%KU@2M?AeQ)Mbp@hF3TKK4O7qg90!=gW(tzq#(3Clt8{_49oxSMofSKg2p2@-pyxh zhaT3u+ZW)F^!TJz25L_-V65$sW2J}951G5RyWd{|n_W1;MpI7C^!@%g3H!~u)%APp zU=Wk(`+SL0=rE4o;|flBesst_sgv^cB9x&1F8s3llL_k7UR2OLIfPiT8DjddYvGB^ z>XK5t%&i|}ZPY{E;>k0C|KYyi87BvKvr8(LtMrRYUT4JLZO`Qj7b<9a!MaiM1Rx^d zHV>fO@lMscBu?+CE+zkzb?RnUb+#&82ffhv)bM$*B9Ynrq8<3rgT0tk2k!Z=q*XTl zc}f4K*YzRI10u8pwp&`p>XQS^n(H#buW|>@>a%apBef;DgP})phH@|MIc7PHys)8m zA}6;ez`cZ?#&Qx(sEH&!)+_S$ek}j@LVS}p(OF_RrZ+D@NYp8*ItG@}DpKS`5tZX! zN21t48-%I!+l>e%s%^&SsSkRsNhG0nWHhpdg}}wTZZ{2viKcy#{-DK4bVKhW?P6z# z{r!5)x3*ax^E?{?ks&=YL*Q4hwxapta9)u{`w9Hte*lFtPG1V}p%aW8jrWP9JUA{A z1Cwl5LY0F1?0suBiOLGLc`>86dk;l|=8d!VqqhzlQXel;LxYH*%`qYpd(I#IY5Umbj9B<@{{2m0 z_GhYOl#us$T-r>AExkc=+#l;)z`USk909+XFPpuA@Ur`$(jyYnu zbDqpu1Pb{2K~(+#go{BsG0Cw72WC3d=OD?zWihnC?le&Ba9nF5pO$3ob6gSVK>J* z9G`ygLfX(e=7h*}dL!7Sb557j9t@!wVp>(sB5Cj?q^wfI<9q6b^80d?= zm62a=xjZ8m^IaC+N;-R;O3_i-yJ8)`1*-ulPt{xHkXO6#cL3M0CqZ4Io~((e2dj)F zWW&U++;+sp)|0*X(Iv%gtAKy#!Dja85*`iJNQHrlp-uKFVXw3^sPREUQ;9hF+kpjW z$QqW9yo?mj7@3xtYugg>>ieJ~lyrTQA`#ye18!g{Lb zKsWjD^Pl#J#{~bAJmi)W8FOP13 zKUYvw;0EIVC&4wwzk_-oG&DjxG9KqzdfPuB{D41t3m7Y7;h1>0!L~nmSLvf7+M}89 z>Y*89Kw?CWeH?5|%yP}~9OCutXS5$6Tpb8yzKL=PXcY!d_i(!zIXLfkZ%t0qFyK-2 ztG>l6xa$@al_bKmX^?qC;Q{bNpn}cJuXl--)&x-dL>gj@gRDVU6-Uq9viA>VZn|51 z%!UW<^L}}j>gV?}j^gH3HW-3}D{vkcf)?tZ*&8Ms8Y{&)G~{JJo>U&)HWp*?>sj=Q zxm9ZC59pHv@H2^}sBvboqp1O1R6bmu?&d@|?xLaWr#!_we?sa+!Y(l(;_noh;KI!| zN6%cwo@dx<-xCNMIASH~+l*1_=2NfdG;_^^+aU67Dg1RW$*aL%x=z;$!_?7gbb(sG zH3%*R$f*Ax-;rgIdp+%huhzAWfLO))E3cvb4OiDU!}vR>$|vnH8zX>V|9FLg@GQa1 zX2U36?4JzJSln@{TI$DYlF}-cQs}S47vOD`y$YTS&W8$Byj(VN_ZTRYv#n4-(c&e_ z#cy(k>`R@Ws4#4i_s4wP4u=o0QLeY*2Svi?*zOW_Zo-&VaxjmV`?u35$$uQW?iVBx zZU1mT_8R8?)NPk!|1Iv!9BN7cxaOU3YoZL?-~7W8)*fX`0fjF)>k6+S)WKsEUZFR3 zle9IYhhFM$tkyL>Z)d;z4LjYb!2i^6E!n<>Pm2l39F;;5WYvfnNoufYWK+VFcJii>`|+yPbme@>{}JN{)V4DCF`NFzwcJ|BRG! zbNxD|@UPrZe<}>9xbQ!##-AMYt{QV}fAyHwHM!L@2pO)sUTHgQAGGrx$W4WNmAx}s zH7q@??vm`|$LacNCD7r+ZC7Cd!uVa9s?6(y@wEv!qz$_x;R+GYnyA24w~@hS4!1=8 z)>jSjN*@TmctH$OJ&IRscR5cN()7yByv=3zDonAp^N3orJm2v(UyhByIn{ftA<)`E z2<{8m9VTOT&TEGr50=`1EcX0&zLy$|J5@r? zM0$Tq&PtP*y_%c@GK6+@{aRdKj+KU-!Bq)@R{g3#=^Ma))f4}~-+z|@KCz!+2>dza zN=7_J90)tEawp0^)sFwVd2sZ_&2!8j2QqPU`G%>dnC$~deK}wd9Q3cM>%pt^m#Vcu zEawL@e!dgXmr!W=@F|R#=!CxwDaY=HQ4Bq*d(qJL77(?6NsBfbs)ylpH-A*#Qm1|o z4QkFE3C6T~48CEdY*?QHbN{DU8hcq#@X6SsQKl+lVRwqNTy5-sSBZ!^&5pf`qm-?m?)7=rz(o^iwDE zqngIBZ=qge?;nUrd{nO`f3XTreCrW{y8M$X*&g)vv|IUlEB>Rn6`%I@WOhT>#lJgl z<%iVuiAJlbph#fsX0IynnX#K7`yRU5A#Xt$zeg7IN}&t^XG>b;xd9D3BYtRBX{3j!@ z&GV23_|_x^eqVSEnse?q!mZ*v&r2OZM7>@CKNhI}a3r^Q6(0pyn8=*IN``+~*}@=L zqcyCJ7f0#VXl?CN@2D~IM3sj$YcJGF<@m0LliAacWqZ`ZOtV03z2lChU%gaxXd?3%C+|@`@ z)V?iWv{*69iILjhf8JXB^8+TGb$?9ytAITm*ZhD^f@-~)6}Eu$xL^@ zzgCHuD7aOW)P4m)ng#B6j0azAboe1V(2YJRe!lX7`}j(?$`0KQr@NK4WQLbCtd+Gg z=*q{_DR8X8Wa||uRLOS?+Ld%E#QVnLrNWfN_OOAYereJt+7$`l#32VLFjzR_2I70w zkayOBeqoeCLHzo?$#}3)v=%63ZBEtmD4&`RB!y({2aI-eqDBRK=-YMqIf#>IDI47 zbC})Af4b&JO*p%G&^X}(;$&_6VUe8b1G*5p8@h}GwD;U$NBSH+38t>x7 zNYZP<4<*TuHhlt$Jug$81sL(^V#)-#)_k-6?Jp6f5 zSkFJ>w^33#00WQn2O2}%+Luzo@qd>97TdVVhl>tM#TkTlQe~)-7@a%uTp0ZlI@*KNfF< z?jLlgZ^k?-5FINT%8!2etwQ)}0XUjp>L9%7Jbj?hS2ghhB>(k4jPcjRDm3$O;K5Hx zyga=42?LLfYgmDQLH`Yg=>QY>r>YN?bj0kXVRnsXuU%I?{11Jy(jX?4#Oabg1qNlvq=T!fWkW7r$czNGT_l-IM zuowdCdVev3dnrsdE)B%KFxx?$K|teR_9LM=`M=V9G6~QSLMC028u}dZc|Qms8J;5b zQm+DZ@R9|?g6q0yu-b!mVM0tIt$=w|v+D^{`ElmnQ>pG76FAv}(A~!_K@=`5I{7^~iR&Kz6wM8beRN&lvSHRNNE@mjGWS*x>T|JP z|4&9VW#8+=lVI~bh%2g)=H>gGzY&CIO7AS#-etX}m3?8wZEHq*b%W*aZR_{ObM$?U z6W?A;XGjeimIt-xSZLZEJ^b$>0d(h5&t@N02RX+33Bs)6%Wj~o&JHeR0XBUzM<`d3 z{yq{*ZjD)y-+3-o{*B_R#dRNgxl!m^ld80fm*`MApSFSQt|34I3%ji(q003)oNX*~ zDurmHO~s`Z4pdB?%O0blh$qlrT{jnf+xE9I;g^y1NAnJyr@FxNqel%Nqn{n3k4njkMC z^Tti#@`)j6mAvV3fQGmEM(XB}b7yFxZp6X38|!-I&zhLeGV7q!ASbO~sGdrf(Y5C> z8m@A`cIsz{Kpa#Ny8x+G>3f-gs3j9#TL!9*LdFBh;$VhaD-n z#3rp7NB~2Rpqu&Aha^kExKKu|8#=;xE3&gWPTSN-T`8-=Xc4$F4aJE-1BTuQ^ z`z{qei9L-hyIaM(;1V^l@;vW3#W6W%`@UGd>aAb?oKb0MrjfY$#X)sxc{PlualCB{ zZY?*f?D=(1TzX2K8yS!bbq?&A2dz_nn&I_+0!V!6gLN%ux=pQr7yMU}qqJq~>$NCy zt&{w_HH+a!=rC))P3Q-7dzn|;^TCGS z0{frBE+4Ku#p(OtuDr{I2bto7ZjB+H8DX?x(BVTc^H|Z2!qqEM$sH$*+cBfkhQ{N2 zCO%EO7T=Ss=ROe-aZtx&7p1UC%D(7D9opSwNK*+;P@ZDI19rjS0?aM-DLD=X-}i$dqZSc2Kw)?)md>zG=V4uFy7kw(v#J z_hUo_^wg{Pg^HWZM|K>yiCr$*Tj5J`KQbxk6Kz*o9^>*9Z0sDqeDN`vOc_!0{HkZZ z)@T3Hr+o{a2zu|0pTj0~qQKc=d)Lj4KSQ5FpM#dBYz;pA9aqVAEpQZmzv(?sOg}tv z`Y0DLV+T%v|3o9DjxKRM{AZZbJxnJ4M)gH)%#XcZ3^dy%CL~aF5+Dc}Lh(-JhvENj zg5ZMi+hdX$w;`YV#wRQ5Q>fm_?Yfx`h}`|a{6-iUq?1gOEjQq$W0Cv|lxkU~H?9dS zKU9BLbnU*UIXjg9ayUMS(bad-LWUn7ml^qv9&6DS!#ZG6C%Tpu;0e+MMX&cL=NJ6= z{$N08;D;lFEf-Sj_0HUH=0x0c{m6*(k&SGUyBu_BOvLU+f0+`2A&;K}n++vXRbNCt z#l^T2Mtzow62E@1$zuKkcoefn&*xQI{b0wK?q>z$zK33Z&h^c``lg%o5LEi}-u5S_ z{z$Fw(-B8*9iy{T4D}b;*&nWplVQTchqupVU8fa}L%f10q?U%dAdd?~gZQa}d6Yv! zD+|-Xj<)4E96HE>1@DE7-U9dhF8Hn!%lo~8tf>d@0Z`uk_~O+V&m1JbG0arMAtO%^ zlK}l?R9o9|GQM9;B=X3Pj7(rkZ=Dw4zw)_e0=czgGWA?=={ond=d3JPtL&*~C=E8D z1~Zv`_69=0uIe14CZdKgRb2QaKbl!8zt-|Me2t?p;|P1n9%BKh(cI1jIte_s_49EY%9{p_++t$XNrNanm4zvWexGV7NJ2S6lzzaYV5_7{Y zBUc~cu!;TwN`-&U5IRS|rL;8Wg>?3py2J0KWmqJ#c!*3Cv+En}6}m0)qi1pw1OF>* z-9lIqLPLZ)PT7>@6n|7UzQ6f{b_`zU_JHh0USI<{JjY12uQU3SQ9ZkdCRb;giN4{E ziK+q>Y?P+Sz1~hRPYLtH!`C67ToAVAj!P##^>Gb&zE$D1eZB7$Q3@HIINxo3GJBl> zU2s4WFZoy#ey@64R#1usEX!LMRxoV!O$bM_d(6?mnjR;o zs%oe3n#k5`@q`yFKL*mvahx?gH^sfKo9YdtuFiyJV|aUy{0O!d4u$wZ4A^xiqH=v7 zx@b@ygt}Rv8yvgUNnX~va&CZ`i2hWfrFTIeb$>Dv7I*=u&umxk1Yh;!og6XW%0Z)v zMHKoZk@4oz3s4o5tQ<>fk?>mWxKWU4-Q4eR44aZ*BIftN_ zjm+&a*#p$oYiEzx^r?cM+)Io@?P`wp$B6lO?T{eNi4AhnnVaHRKU^-< zb^+8J)zspmukVK&{snOdmEvjMiHz%H3A3UIaA%e%aM3LAb|0V z$Xt)T>Db-x*7utdh)Q5BbCIF zs4U`5^)}>G@TEOlGa=s=4XuG%ZmnUxtehkPTwGjC9AFOS;GLm@S4XyD`-V;lBtTKM+Bo=Bt&=xL zHYVgpNA^>liQA+R~+aO_3>OL|K z$XYQUXuj#NOz2Ku+e0eWP5Ae1U4OaKH*#YemE@kY1nnv#(qwm$pOL~uZ#r7lJ|$Tp zj(!Y$6r~dK=~cn?t5h{-`<1~6yUDPiZ9axYRzbKhU_YVh@YXKRoG$+g{ILuAW(%Oe zQuk=gvKR9@zHACe@x{vmB&@Ap0lpJmsk-arFZqsyA1{nFE2l6}*FDcEfRX+qdZy`I z0j+Dc8f1~M&zvbfOiaffqJ;|A zC=D(8?%c!3j{!?=9onvct$))RWhq|%ve>`WwI6Y75pRp3dF!8g;+C+E1-{~@CG26r z66nRr*PNW3=EHRk`j~3TV<*(^uP5WEW(Qvfh&Y6@7NNi2x^R7aB8~P~DySAU|T&>pg1<+c0^) z^p5_oRLoMdfJ&%yJO)y~eIgY8#Gk8Pz>f7vM)R60D}{{{72xhLyiwSuH!hOoY$MFX zo*ctl+J>l>p$KF&kf?hYe-8v!=FGXxnndgU`GUScv2kn@ekS6bPM+z_lAg;ZoH4Vr zK!h?R(H@GK#hKn(=5P&YdUj(M>o$i~Hhe~b3&uOW5=RiM?s_nj2B9~e{2vQ|U}Q|i z`?KOm;U=6n6WachyEL%AD0We)3_ADX+ah5Wabo7ow;2;aJ%0^zCl~I#2l%Ar^K@DW9Rl zCg}DeuQq=jVog2zQ^|~{;p~&%FvADNI^z3P-Y|!0RYhuNR#Thl^8dak@{VinSIO{P zly+65Ak_a#H2#bXumF;GA8m6Gtnmz^3TkP=*2i#>vj1H!k7eG{JHR+la+@2wHE6Kl zBM7yGf!9NZcvZ%U9g5XZi^nn)h(55R+QAyb*g4vr?O55A?mj#FQ091hVZd_#GvLz~ z4n7La#wGu-I-iZ%L8ox99)eX7F0)&vv_3lIZY%4^X+;2MZ(z7D=MoO4v*~} zO*5jguf=C{-I7f#&FCU8RNr$P*6}(?WhT;JD?@p({h(iOIBp4;=bvg0avI!-V`JOw z>hoT^B%xk6d)y$0hOr78U6I$#?<~O@cW@-nqul?$qh8G-=z+Vs4ZVc2 zRL3u#xoMnc;J3?`0qe+jOk;%^k}`zSMe%))sX1`0lWryOSG9Dl8XS6i-U6Vy!K|UX z4H5gX7;o_G4Vut<9E1zb2b5acoPLI;6cz4+;KlW1P<*1>*f7%IPG-k1$qRt#*r@a+ z&yR}Q*3BbU$B2$yz?)O96$24>Q4q`z$0f|dy~TIkKAZI-h-Wg3g)kDcVOQV&?@XwG z^&;eT!a=pr-4)+#p86r7lzWyp15dJJz|@+jxquDOW4#76Pia*LvOwqXmgSb`v<2aJ z3U*Yqu1U;7jN6O!bsD&NhNK2zrf(FPma(T_g5FE^p>qNr@1WW2cz1YONby zY}491BJ1Q5vhbpm{;jF&zwUDIJbu@=7*tleKl4eYWR{{&e0wC_Q*?I9{djWlz3nS= z)KFn6jKhilV0<36d!%%aSTQg>f9Molr|I?nVhkRkak7^Pjk0Kbw8FuixFMd!jWla` zmTB@CEpE|iM#J8FuXm$AT#;_?<&Aj0gpDhtp^@oVg`*hV2|SI0;=xrHnAp{GQT|a8 zp*#OwuojUc{_L5esQp+`(_<`+_pU(Gly1l|ZC0P|?gL4AwUeO_U3ZLOqkIX4y(omvp6% zNA2DyR5VcGMj0+~lFMh34mVnp7$Xb|L}PaP?H*7zxON0v_B_k93(sJIB0%l8+fx+R zhe}c~lD8II(vVx12LzT$^FPn)H-CJZHH&lTiT^~{hkW4gCb+MoS=9B<3gW1Yl-mc1 zr;e$2gW-3_H|lje->n{d391bL{nWn#v;pMGH<&plnvY8i3CLMCmM(ZCu0Oar+Z?kB zR_C|Dkv~>U|Bkyxul#!9ohDnmYG)t)=8Scj$;?vX9@JfMArTFx1dbWLUVWS(6P3Js zczjX~1vW830ArW2<9UDfRH*`nx2kY?**jHxi#SC3Z7p@M2A1PL%5O_Pa*M+6mYym5 zz0Tv96D&%2drK?uJJ}yca&HGa`jn~I{A+Zzh3;|^@=^_-+mlkB)JOU9!>kvb9h9;a zBa^C&P5YZSeiThq^B^`THxr;N1oEnf4>+B3fHCm1%1%7>ag)c>9iJ5fDJMG%H~%6j zPE)s#R98I z?gxKgRKu~t$dN@yd|B7^UA|W8O`oBG&A$i7`Q!PQ${mIn3H~ANm6Fph_b7r0j^6lf z97$W$F-?mr@LB7lnn1PwUgA}LqGKCA?q$F<;PVT6QAg|{GpJuqceZ!aPnlI&Mj1@A z%=JyN?C1fni<#ZplQ+jU#$Uv0`}AXVN6jy`zE2F!fj<_s;lZgiyYoe+pLvQn+zGsM zq852xN-sQ)F)&72p2B91UL20^ORD13o0ICDG^nT)6)QGUXA>}l=M7!II(EBOFlW`H zNI13ExrqY-?2~?Q&2*xzYzsuQ!QIWBh4@J#w#Nx@!v#McaSM{mF zrx-~OiCTk`8{yJxCy7~Ae^c(pkqMcep+P-@5243*Z@3NO%p3WAtYyFveQHT+JuJVJ zP5qGCGwgRthn}VuyNb?1k{-z0$o3<-Fw7Nj1sLLOmtnx>SNmdyO{m~{=JCI_3x&~j zJkiXdPRph9$#E4<3VszzeThFhQolDt&A7-PeTE@le^9pnP*eO9xO4bPwvJy$SdgLH zh}Ct{ZYf%t>`nV(BQC31E7@K_vD-HnHRLf<1>(c`svEabyKj$<6Tzc7$32H)?!;b* z&aL$?C)L{L2*}PYu7h)$cdbC&reouR3IOg(z1+-pQM=mfpy@b@cPkp*bbcy%M0=b7 zp^8u}{Ja_pB8n75yZ>qj zp|~zMeDc^QX?-m+V#UwZmmLNgb(#rDKI$?;_$DoLYj?AgpRf6Lx6jn4-p<9tKw-cC zlnVcDBiNK@B$0o0{6f1uU&RdYRKqcsqBkvo;BKf%Ll9{rNfp$7k2!>}rn_bQ=jhy< z7MQ8#T3kPq>E1jUw~F>)4r&2R32%_H|0uM%L7N02tLyGTc)ebd{(}>UU9tk8{;bsl zU%039$PErqy6a42_YnoqddQ}#!#T+MCQkE}2LFjpl(qtppYUC_0@^hH?&QmdKX90O zczA&J876Xk@&$KJ9w8w+SMGyI2Y$k>p73vnz?zZSfl$9$-j#(@@CO|5<*oU;icF!caT24}Se$y-y{}aWLLG z!=_m7I21fS}Hz)n(1MCE%`*~>(c$+=N`)@-bP~WHc#iDAX{xVRp+O&KR zj2AVKK9JKZnHSl6NTPg*ozvf-giJ&jY&wYfSJJ08oR1rN5UjJ`6>nR91N@d9?U_aJ zJFSq(_75jp4b^yA?BIs+1c*3!c}Cs|9K1OwaGWk-G(Ydd}t`7{R1 zlO*vj!-A^jw*%^AEkC|w0NvOx5t4au$rqgq-J$Nc1^DX03Ts0OLN1oSFlZd@Cb7RV zwI{U+3p~Kf2VRK$5r!p{aomlJ#zgiJf`c~r1P0;=Z zHFkh{l=l^7jvnkk0tZY99e>D?F`!yJVxX(pWxylyhX#TPv=7;M@U4VjBL-{T7rU2N z2*MJau;K^9qf*10=fkP(PXh=-d_pfmKl$H#_1+zaHYR-jN6Ao<^kLT2ep0Y)?|F6h zMXN(wixA%~j`{qZF@A{^bjj%VrnBI>oLN4!MQRXC^;C+xh6P$X^)aq2Py8|wi-p9U z9vm+mC2V4yV1Hq*@4c@B2hIBY4Md4$04+#@$l9s&Oz4@6$u_rtqjMOKvp*C4)$tFo zdBa8<_z?c2uWj-VbW5L5%3Di$36jaaU|ft^w0%&1Zv(yyx#v!ZQ?7~r@|oVn9J(GQ zBtrrTo?96)tYN&pUGh)L^uwNB0pjtxg@AI4%L77-@yaU+2?9C8AY!{Vsmk1K322|| zNH|~dgJ7kBWv%oh#CY49P}N7qgzx&LQ|h{yhdtdUpDgLVmclF*!5x3?uXD>WG+BR2 zWC7*}qz*F=t{0UG-VaTa&&ddh93zb>+1lb?zlQOGvR3){C*Hb)82k3B=2<;a>tgwO zP&I6FXfD8^QTn*$f#cK$iC*4E?OIz0se%?^?u_>d1`neN7=Hvz$(O)5eINMpAt?0O z-JAFR&~f+XK(CWtV~rY?J!zvapm6KCLK-cBu)lRMX9$1_8uBkqUqOe#@{oneqEicJ zF)y69Xn#|v{qn|2$db$HpNr1S&dWZbDf@Am)H@2O(J6<(%eg&F(+svc5x#jtDO0G? zd>yFWx(Ny@%Er*R>!pujmZimjF^rf;Sa8;>r_mBE z0vt}8;M9rVn8c0M*YOl6#lM_GnbLzBnW4Zn>q>b1ZTX)no&7~LR7CUKDrnA%0=|BL z+vT}RJ>grkW4Hu3<9^2)y?OBMGqBoI7wqk55e2JO3OJi7PS_8a0jvZp`r0W>YJQ$5 zP?B8hkO|TAt%5Q)MJj|Sb)6xjcR@LF^O^{v2}W}M<5OH}W&Nt-K5Z+Rdc4A40{R+o zG)r#-sKRXoHwOe=MBS!Unkcy0P}Y?J);Z5c_36QFlEEsay?j_G!G&5%s z?u(R{|K#dZ#PyWYy)Xe z;j!6g5!8(eErrtFKNKkwd9Dq|9_=XV2VLZ1Pe9QFRsqiMyEjNeA3ca%K4Q=!QiDV3 zoNtjTuU&sXP+1GLq5OYLop(HyfBg4L$)?B(9kQ}hLe>$=&ZrP_NU~RmgsW__WfZbU zWbct<#K~S|?{(}w&f%Qvy07o=ci)f4{ipw~hwHk|_5QqH&)4gfQ9Df_UY{Y{N6iH!Src+=7 zcXEDN5*lNDswkghwM8i9m2H}~}=KO2R?IY8V zkDRwxaPYkJEJnk4WP@y+BS!7xye0md#Bp@Xg_TGh^smEAC4Zp;b{~K&HCfqj~zg7&SB_A zQ;4^=#LiLWAbv%hkSZj2k;!$pV&wKmDwFR;jx`p<%I%YBwv#FJqRu+Fj z3om%D^0Gd$i7<(uzP93~oE_uraqFXY`~ZhLUP)`j^h)uN*sEn6n+(_M!A=Z-ozp$P zRPeZd-XtPb`xF@k=@JsADBoM+EK6d0Y_r6kPkOB)HkZ5z1BT59xiV7wLK2=kruYc( z@BJ~Dm3`=K*Gf|V9tnUx*X)^~@25>6fHk&S!hTe^NB?`2J;=rZEvgqOy=Wl3ng358 z2=IBy)pjrqMPi@}zlMsDs@6b1z{legeU3Z#JLRa_1FxtVk-z-Gv$i=KLG;96GoCs9l;j4rosXse zQib@BYuXS0)Nbvxp*llb_cA6kp_-m(sucwebZkB9cz4OrdiBiuGOkd<6h2DT`I*Mf zQ+I8WKM%F|*o^NxT!#%CIUb{5ABJ$8j8>W2 z#6xK|{LhD9POO+IF1>$e7CvOxASpF}l|+uyMrxmwnVmO;)9}*`1BH0IzXu`fopsBH zET(=H+s2{yIo{e+rDOt_IXs5FtUgywJT=(D4P3dSCO73#2(?=e9E~WpVpgLh*>u0Z z{EN12esBhW6WFyc(ckG~M(Wpn5x5Hw?bg18b>#$WmVoz&w$#v%{O{d#&&&kgHNe8H z%xW5D@ZXpzhfqQ=@Wm+N3m@9xK_;6hLXKwsV{fSD+v}T%l9xSz?Je`7;@}$PtKqNa zaZ=21rI0E+kg-5ucv45CpstoZMHU|*XBxqoH=B+&(eiVOW2SC2BBwn4wL|j}j{}1azkL(68Fs2s9Af!fmZI=i?>5s_9@TWeYpy{TON|s>M;rS+ ztl;YuLKN2#GCXLNe$oyfSF!_@Wu3TP0$V>gFGU%ZBYH6n2-YV%&`IqICjwsy=uJ=0 zp<4>;|B5x>K2v(%?IKj}gG#CDpIjiF^9BmGq*a|NoChz3mJt7W91UxVsW&_WtIfd< zP9KmVt9zOGmEIWlBjbE1))#R$Dud^aH0^omM|O_V)5GHC7V_Lqk*%99S6>N9D5OeH zuAYn)osjWbhf4;H1*ZLky5jjYoyEi1z}>NS+I)%g(n>!iGR=5ogv?97J#Y5Fr;Pd%V0kQS^qf$u*9g*t`GYI(OP(A9)~OGV zN<8(BVNTPyMyo!2lOaD%!yUTB{{6|(Np{%k&f}^c(?a(5Jafws&(MtL)0HNGf6<0jK!pnOk&a#rAq)TatmEqg8cl%!ElklJsK0C{6RO%=i1tV_94vz z0$D7ks+@hrrtW>L^*YH%A(y=C%v6<@)h*#75%T|>Y1Dkrt6s?CaU4`2#V-dc5m#Nq z=6}@DNy(;E#nL|Z)++b{l$>OavTs3_vb@ZML}u z5v9Qg<3cQM=hnC5tVJfTGJ8Z2_q&2#J_|TFhzkG2T=}Z-(FrgxbDF_C99iGE-CrJ< zdEJO?^G~{rhv0VE60Q4Eq*S=`H(<j{KiAw(>f|!! z{RUr`R}-9XA5tAVBwk}5%hVHMnhvmTafQ=Rx)4}Hmt{j*{t7}^0`p}-FF9E^*!$dK zv1Q(9<#NYRBPKKc(ZAMz-@FQ19!?R{Z?h18RC?4wFZTZ4^C?}}KH72{_;J%qX$sz9 zeKxPPATk5SEHO38ep35%C-J9J`4%CE(R;SO)5V)_&UfDsKD0tI;EX*aG>=Nvll=!R z_1uRKiGs{#YE`4|S?zS(ZGC4mc<+v%UwuYEh9U8k4>flh))*sDqmK&4e=H?==&a4g zdA9UhexjYmpLs}1Z8Y>B=g9K@UtT$0P4wodEld^HNnd(Q*56Xf2fuW!6 zhHvj)aNdEGnZiBnMZf{M(+G;J2Puwx`{irEVV_hI8@_JHd)N2$@s2cPm+)=)O_AR2 z51U?G%E3?Vk(axR&fKEG#Cv`tWzk60mg7>aL}%1&HNa+3;=35H(^os}#|9wxP)r(* zW{|=;!R;tum-rFS)J%J!ZJ!{5>v_K=Y}I0}6dvYB?N)g|q`@9ws84;2tCfzm4>NT?;vwT>aa zy8r3|u<5onqhN*R$l_a>ClH$>%S%RiNO))r-3p-iCvCX1U_cbyH!Fk#TK7iJW@|T* z&B#yo;=Zz=bO_%EL;ZYr5s0Dw$b3on`vyFhVvwoOCL6I`e9t@$YHjP|5q=i1n{qtx z9ElF^CvMbBKvDRkzyA3+OR1jMTiwxHJv0}YDOegPWtG2xaBxo6`@>cB8rN!C;;)l{>iag^u<@7x7WTw0!+XlBHY@h1_8%8% zsZA^oj?Eq2+?PD@g9%INxJ2ok^71_-$0nh)8tL+LXG|s)n6U)3_Hd7zziO>QJ*!Zn ziGzR}v%?WcwzKrhc+&=(PIhx%XDY~Xd)?mTi!xhk^&eYXnZpDFE7PTrKdXIcnG5c{ zL|A?M^dyoaJU;hsn%%`+?fSkEv?ymt&CfHPgDNV4;$O>EDizJf39kEek*^(VTnO8p zb1St;Fe<#{eA93yBKNEva))>BAxOeC;$euw@4P?bmq0zI4lGo3bPb%DJ(e;;MLS^$ z+}AbaC^;4{EG^Y*=hyCm{!{5(xzaBRskZ2(yp|{8f0j0bwrFA|*>)9>Gd)>bo7HjO=Y95TnZ9t|?QCt)GWwctB0w1f37R3y20 z_>r_)rQ$seSwvJEuj3qX#6oyktk{Bges=o|-Fxqb3b76>E5DSZzy=8s6!9S07K}yt zM(SSH9k0r?NK3()Qrua1arH<0I)6H$l07>0-88hkds(r0OG#C$i2N!0Cd-8I7kyb; zF6KRoIG=;(_TBvC65m>4+Upx?Faox1lMfvkKHlk27VGp_gba zxLE4`jhZANAsitE|LJB8P{Vo(?BRBxuX{~+0`?GH^1%hP63U25hlWOIzd`>JspSI& z6ov`uWB=Y6%R=t{5UOd->R5g?_|oa+`vLvB5w<6j`sw%Tz>DVyMQm!H6zAd_#%p0sZ+ zJYRDZa*~%bRN*DPE!labRb0k%!`@!Hqjgk$$soYmN(C)7>^ z7ZMB~97`0=`qDiY{S{isj1rW>lUAlU-D+5udiDV1R{8KNc5Per+OO^HXJ79;jCBX= ziFiU1K~v(K#T7EigCff?YMOhPGdirTt`B+ujT2D#^>x5YW!y4{7l*+&MdIe(W7Cd` zAzWg_KtIF>Z@kX8Dy8}+U6_g+cf2KRhk5pC6uBSONm|b*Z3oIS2p++qzs<#C2!3Oz z&1U(A?mh+0h@% zbd;;_^mr5fX|f6wo4Yh5`^@Yp%M|L;OieiEqdBZl@3wNv0rF~z2{^7%Y1?S%@b zGEQUe{4yz+Am#$8ai@p4%tJpGWGDHD&QVba7+(k(l576BPdFUV}&k~K(Ktk)F>2-g>JX!i3Gkgt_LsV5ync6^CSMP)ZTIFfCY>p}k@+Vab$0bmJCA06kZV;yFX;jC;=?=~+RBpbE^JM?t_5bGc zbJFslc@iIDl8*&{FWR+)W=8)dY2X6}%*Ei(xKQzZD;} zpDoUmb?kw@g+FHNbRAX}Swct+FcbLpuIHZAaY{y(<0 zjrPqS@)I&znuC#%@f^FKrSEN{zZu_dT1pH2p~U2N9^EXu_|<}hCuI!^?q~k`lYZGc zJn8oLvh&tc^Z+V*DiT?Qes&;QzxJvj8G1KsYyGXVW3>vd&%Hzb+_-FBpdlswBfJZe zt^0nryYn#eus{3Lr%!`&-Q5f6U3C7MNvOH9ZZ-h@C*&3B^E}4U(6$mAu}9k>-`-x1 zYW+k%Y7p=KVVjOyU<9HLYFUv5%V@#RCtqCsHNQQ8s>6Xb@5wtiIU>g8cX_lJD~u#Z zMQ;LvWHtr7l*bZBrTL{{fx2JGFq97M>8cfmcH4Op?}H}JXc3Y!0f@S<1Bp{8ZbK!<7xxfQeUXH% z7v!Vw@<8c%6%qrPQtCx7z4Tq4zw*de8#cd;OhyD)Cu_Y^9TfexmSaMzQ!Z-=^S*56 zG?#X5+eh4kLIy?Ou#|+k^ds|({JA-;q$7Kg1W>IQ3Ui?gGip@NSVAfg#APGc0Q_~I z65)}NKWEP21eK7%Fv{(^ktgn*U#$n>tdQI93rMWJHXUn+i)-fu<8Q4h_9ZE42laa1 zBtDTj8dNUk43sS4(S0TG;%Rg|4GxtK*qcXpz1@pm3|9Fjgeu?zb>4?c;O-JeP4R;H z6en}9!E;=QGCbTr(+rF^(zr5D&HqGl9tcrJL?-Y7f074_=vT%_TUVq^U{iu4NC-H7 zDv|C~4b-A#$dk>q8y6}I6C+fO&d3)#E{HP_ z`YX>9mJ2^;HxdCdw5iL`qqEbsY#VKi|CMrWlu#~J`+t$|wZ0CfT??ARyrhZGbZFPj zh#^BcEtt~6CzPJVds7VTL=15A{6Z&cy!VN8WeO3n&{7+(i)-V3quzVbKj>e)viB@R zT&WMbpz8h$bMwO=ZJ&UgsUzo7n!=kNT6b&iD9|2_oID&cX8ID<o%#q6YX6=Qn=%*rJy%$%37s1SFG_Fr>J-VR zQKRixNA3Ep64)8Y>4t3@v zuBPRG3Q1lALf=itu5As_%bz6|29oJdnDgzEQr`08`eb%Ksjv@(rQK*Z+a4z(mty9_ zgjR)nh-f37q;cSFbL`Ul#6yJ5^=AlRe zn~+dXqI#1SzV8T4c&xx#HAR0>k2P{NzrrDut58bfI#t!v6-M2lht~qi+jEl{{$Qbl z(F}I6#@mDXnX+66GLyMiG`3{C%F9@flv{|qFPm@e;sV?%NN>Y6er$@Q3{knP8B9}% zoQGzU-3g`KHT!mjz5AzuKB?kMB|Of0@v%}&;t`gSMh_N>ZE}vh@P?V5zcyK9U}yfi z5DHJ}3s<^lhuriIHS^hjv{LO7ZPWHUtZL!si>fiR4?}Q^gP4TdXs}0TrmwvN*q)5J z1Lh5&Ex)9n{`(uU(9S9$0W}g9j+{X{^7U1)^!*8)N@R;=84?TZ0lgXlI)BzrP>Ik} z)I9D~2I_^1n#*_Dp=jP2*ha7YnMsWPK6Qq3GK8=FTuAUw?R~-M`;?Y=lON-+l_tf#jx3T*N2+qasH$Ozf((_pPWiQ(d3=Av*(Zp|Iqw1T)Y&0WhOdcA#f`F zb~D?S^Irl&9eVKNB~${`k*+@U0jTZ4tg))sy}NdhlpR=!y)Il~p$eCl5lj+^yT_eU zdmqc6ud5;nT=K*t5@YLa)>r&t24)7U@w73H=XOvy` zf}r}AUwqW=+zq^FhXQ1v;Yq$52-S@#l=P*0l}uNv-w>apn$MvM0>EDkls0TI!v~^) zTv>RAgQG&j6GNof-IYF(l9>e#9y}13D)sY_?@Kv7JLyB}-tcDK>ck=gvdFNAcc%Xe zeIth1tp@8hQCma!xpsmzf6~z=fw_(CR+)>51@{g{!$G^EVkWHC zUKU#kvTGuu`T7s3h36?QajO)<&$N4*Gu)rpTw-Ejt`qq3$SKgralkmJX<>Qu?TB}d zl2z%@;2nN{pFeGhlaF6$VB?;0$odfA#Rz-@#U@f%P(@qOG)6D4!)Rhz5a)zsw>~fp zUeu2w8NQN!j?|i{`IiE{=}mqJ`*~!4Sqw-G@})S?J1quSF4SGE`V5Rl!WuHpw7Hyo z`p)OdGS(A}V+N}&*1K!4m`Jl(NRBqsF`bHhJt#e)*Pd04E3bRLhT8x9vxw>{*1Rv8D1^;i^z{;+fHGh=q!3=u~rXd{Pb4R z7Oh&>-DqXCzsy)2$Ijj85+nIZrP#+e3VBoDI5&au59am~!}X;v&K%V8p7<>8t>Cf$ zUc8L&as7hLgqzAWhW zMFv{;g)Cd1OMWk_!FjI#7|}}z0_RJ&ndhjz%LzrosYr$fU^*tFlL>n zY;R1ywJU1=zD_$o15#&C;!A%VmM(1J?k&hf0Bf|MijWTEtHsjWrFzCtI zs1!Q)*^<1scr0?IabWfX9J%JCwplD2Fd}rQD!lI!HW_qsJ!J2KIid+3T35Ad(m?cDPpT({Y)P#DqX{E-^`7&p)RTgW^= zP}@AQ6>aJ%ifC{pbi`Z?N^Y70iiqiE{kp(BVbkK=@`Jy6^#}x=B%1U#)r&F8;4%W% z2+p8vUvl}*poe}LuwBOUDlI0?ef92ywF|}Fi2fM9gm|-*`+oacM$w(uT$@zz4dCAe zV`fu7p<`#$yEg>8COeSuuOHpqD&gMl7$3woP2xQi3!d(A*I80zzv%;htpR(o;~O)X zntHd}N%wg480sKbs9mnhp@@4MRgmII-)w%{(|)Y0A2L3-<~%e>@VK+ZPaIp)EschP z&`m8T{f1j#DW#!!wPcRpemt}RR4IQe1D_xgjhnoF^a>}6PJu@wXE9Fv#pV3$7Nx-! zubr~_G81y6aB*8&buL$>25j*&TwQcQ8!xMfjDH3ipKWRBH7nUWGDqD=A`E6p8mZO! zaGJwx@ICt<3te6~lucpGnls3Hy&viN?@gEnQjPC#TSMD~@U;a9BT%4@f2r}MO0G-UB7LtcoN3%p< zX%)xyvlQdy`ydJTuwxvdwm+iUcCzZe?Lu3SoT-g*eTKOD2CWR83d_zJbjJ2GNpPXy z#`mg8C`V@#%hNwz#DnN|J^J_9+PCvdU@hCGqdrFPEr*XJQp>{l7sbIRv(aNEDN}{^ z1)LCnDj;tTlzI5(^xJHo%sGN|h;k~1#}pv2AJ<{~#C-U#pCsAK@|2@RA?2|cx+tz6 zFUCVsa4%$&2i)bJ8#ldHkK?U&#Bd>I!|?;yIj1Gyz%(eHJU3}i)y_pmiq0zYmt#z6 z7A$j;MBbpEGF2MFr0Zc6HRSGQ7%u~V`3p;h$+3Z<@9>nGLi^Zzj{_czeOhwnSuRV! zS-&#=jP!Z=VGKPW=JSM_ZL>iBWL(*4(X$v@7&s4LuCNNY+PL8E3f}Lt>v-*~0KFb$ zRil5G3boRiu71C3=wh&ZW}VCu&Hg>dh_oB;99l=T&a=!=8+ScH_8!BGoSgggcKX9L zgY%FBMYY+6AYgc+H#8N9P^&%A`SL>+i&*01EKo3UB{GoQ9qAe@b%rkCKDkXDkUkkz zn;S>;0}h+<`v|}BYe#uf9BN`{LIwThr7P;Q?{J1@{c52IxMYut)|Bk2Cm#2%l|QwH-v%? z44DG!NlBi2HT*!_mA=r}a(j!HSyuaP%t70=IPQ)n6KCRa^|mg`Y=v%{%Uhv2iz<%2d+L841Y``sQ2EHj3HsK2#nRq!q_M8xZj2I1^_Gkp>HxZ95 zlMls)T#`zVgPt#Ui$27Zn`7D?SMVX>d&ka2{eX-GxNQLOOU@H8KQ};R38HZtTDsKA zLPeK7HVW94%GG5EvEHh)XD!J1s^B+AFnIX7H4SwR|545rN`EWsp}cblwvqP(Lrj~ZmdZ?# z{S5^#Z<|QT%02aFax{ge8*oVb^pP8VcLy(NY9#+7$NTOAT1iZIMK9j?S9cDSg-Ts2 zV2Lz$eb*_1#E2lVe2O9js9x-?$UYFrTl`HhrhS^{13!}~V zo&i@IQ4ki7W)5jS1Yb1x-}Cw55!PngW0c}OI|~Po!w&9IXf@)I6>EXHI>r6iM^uEQ z5SdEl-#jzWrAM>94;izsq5As=J1#cZzPa6N7f5(_2DoRY9bAm*AtNVPA1!)iJZ?Wk zxJ0Jh=gZ`)C|Gi?Gx;-rTjL?`4n&eb&H3cb5IK{z^;Y=N=M~oVih@Jfsvpy;$DV>g zN)$)h^+cf;TFY2=m})Kn;=$BM|M%uVc~g)Jf#K!`Sznyat_cCZ{nixPnc62yAMaQi zW5c)EW%`W&lY4Q22F966f8dQneV^|{Wdw7?yb|DFh7W(1?U7wc?b$IbVe;w^L{VjT zrbXNdmq_~2ZSiT@RgnJnz)l-w?7g0-Ar{wj>yDP+47bu2pq!_i+UlTn+iwa9kBw() z!scA8K`*18-7bjd<&8^c?6iqcc_EO0AD8V5<1v>Pp8jVCS8}G8ntc&UG9fbs{ePbC z+g`2^Oh<7l%UbxM zTxwwlP|#}5d|fPm&nzr1uX{8OWV1QpklyRGGn$l{g0HlII6Gn<*pM-_VLkc$NN-Fl1@~99XWSVUjL%+)CvfS0Tt`hhDJYnUG?1{D7Q2_D28%SDN2Fv1ClT{?K zkF-8U#`Lh{9qWE~sKB3u_!$*#$3zdYGKnH=z7+k3rAnm|TOlE~$oN%(7JXSk>#aR3 zYMxc{L3oO$YJS}m^6K56hXpIulz<~T4Hsa~{POL!J|j>0kBa|$8rQ}3wT7w}b&B0e zk%aghGhb?wvU2s{OTEmUjjG~-VLH9%d8I7Xf9?=IGp9{Mk8Wp~(a3Ucfpb_S-^HhG z4hnBcrmq&h_6gJbOl=9+sMPdVpkY_Fm2d6Mw|`y`@Az?8I!B-k+h#h@W^J9{RyO`$ z7Qoz3%=j`E(G|eLwM^zF3a{gDiqQa(#<9DpL3h85U#SR2f@>PhM$m$4$YJ1-x)^BV_JJ`nxX$!M%21Uu)PbVWfSYtbg{@QyO8Nb^tUo)vPe3j6Btm7ie#S zk36>Nz9)IVPQ0`A2d)V7ANmXh+wV_ZdMjK}2Bx$iA01`V0ik)?CY2Oph#a;FCwIcg z_x(RP=uumVC9t%SfXed@qy2||eGmVvM^5Gky?zN$V}b>~NzH&C`oO;F%@Tm@>vqkhV2C9NJl=(8&+0{)x0eL>4hDNvM(C=ou~Syitv<|6wvcbvlu~c?=qqZp56R_y;U`^{&he); zs%H$e=_$`WJD}pYQS0N_w^HP%RU96&BzB$2h8{`-SUE+i(GNk%C%maH55ORSHtOnJ zoKz1u6y?4X zEU$a^$eRAMCm%U(NDf*)?pgk<#biy#Q1CjbRKt@n1KlArwt<9dkTuGUUIKg*IDC@# z;rs}9@%@>fc!cGU$e-AZX*`;3C8->g$8Nc;bqkeV_sU@U6-(#GrTI2TOF#8*wP^yf z8sMIBS<+6tw)vYJwlTl_MzQ^0zk&N+D;XK{#u=HU;`m9Z?f2CNqWE3PM=iWuOm|7l zJqU<+I>Yp~g^!7~F>x*Uq(QCQ51ZqHUrg#mp zQ>1n)VxY33RL8N^uiQfvPSzNVK18~X66shzX^T@Z{p3qs;s2f0s%emRj4&*#&X(Y8 z{8)LJ*nw<${k?pSKsV~Ddeh@N@gND^tNRhGSNs%9_^I3Pw)^YU&-*ZCPAkdGn}XP$ z=`7+U{D4-jZT~JbUO1$?>d#jSEJhM;&2XWDI`mvHB>(PHSub*_{%72!vJfs;KGKs` zn7w;$2nWBAhK!D*A25vge4%&UbtH_K)XE&d(Bpp& z9aNPM7Tu?{aXRmvz615BkC)Ba6JkLQ^seXYJtw$U*1opFU#BE})QP+6I8*Qu|f% z#ocK4EPfN&&#rVV+_4^*c8D-Y&$JwO7ldD2x(y9>$#YfOeXO+)Xu)&P_)=>(Vj)#+ zyq?%dWFEJec7ga0m8#h8v7VwoiC!Up!W=vc+2_Lj3k05A2mFTCloaTFKpI!JnS|e+ zb{mM4+K;@(2)>?nYV6JJlbTD-E2-y0OWaqZViL}PyEdX%2htH&hY#mYo3KXS+xDKB zFDl>g;wI;zqao!TCGR(!gR)wS_&zFL^odkybgGJa&bWvA%351^R}9U_KmW6Xh228j zU%hetpGJs|_b)1!H5(@w!%ZLm+7EGhYraHzsTzE7H;_AWZl%(U15y*hq5`w#OMgGn zzlNCx!)c{(s;9_q`p$(>6S#P+c1mIPw;0Y%vMR;zkZp3?qj3bjW-Y|oJQ?c~qjVCR zJ50BkHIU$b^hkR8@<4BRdEU36*MFR9Ae~`pgR4gI4zYy5kRE4VP#5*7G?mFil~fum z{*AaJ3$}(`?tcJf?Ni83b|dTC!$BSn#=L--nJqj&*F2bCd=@fGMOmsIr4MZmOn=NP z5r6)I2ebBRVJ~Or>SsK?#XBq3wJ3M>cgnQ6UpJkv-mBj9c6osXE-5?Yn&R<3!%_Q< z1hr9fnG%5I^@|fM!3~_aJZJ`Ctk2$N_idx6w&%=_VqMLx`IwYY(A}#R@~hz39}9ac z?Q;S%dE09ID?cK4ZPFU*YHZ(DVdB#tEaguQkVG%=5ay;wFbZeo8RGx`ozFoE4G>6% z`}|gFm=tEAQG_<;wH_3Bmk1~RXUOg}#XBt%h^JFxx8rQ!4TgKKIOe*8GT)7tzkhoN zz0P=a?-(im#OW6O;@j}W*VBK<%bDniBO7sp^geNlbJs^gqUw~q&I(+3ociB3#u*CV z?#6ms#z&XfS-&z%f}`*ul) zLHrg|N2h-!g1%Bn?d+#W!@t}`w6yJuO_5TQ$^0i`L(?;>!`<-)nJK`Bd+1NlfE<2q zV1@C2M%!tzbWQ8;9*I?VOYqLUhdS}tIq@FDh*V$&hcwWUkUItDZ`SPWA+wF41|t6K zLj)F)Tr1dwvm>6tC6h*Ja2_|8|FHLe@9M8Tf2uY3ke0`*nN()-9BpUMyNfmX0MU$KtuWZ5 zJzn-$py9o+!8CMEz?hY)QFp1maDrwWptnpU=VOpehefo9!hw5z(!seqk=c~z;{tk~ znKWT`6`CQqHJtEDn?lI--wf*x;Dh=q8V+c>-fjJ$M+^?h8&5BRg+_fw)R8Zov0_vG z&tR$QRYBDfu8d&6L*PyyO=4iHH^0mr9A&PY3gw_O-Qr4lLp%F5UM{bx%e|5|J}Fms zY8TabaoeLV&Fd9HXtUf%zpLvQW|)6*)>9l{;3+NLc6L96QAIby~%x$U`+(pgLNcGu(MQc$~@bhObEViGjIF#W4oAWaCRHfgJjo5o`_*AKbb; zRE=iZG5;Yg9B*Yq(C<>+ifku~6 zHMZ$Kt(_}Lnb5^f}eS6(yHW3g{=0_qja*+w*RjKo&M6__v(mAl2A~|y$Hf)xvYMPm;1Q6DVjkwH@^=|lFRq%mSkb%5P7<3`ptas5=^dTliL!YbtLX^~hpm z+08_BPEq$h$z0h3@Pd+_Zny0;9#eLmrU{m9yckAE+Vo@I_APS&S2|l_3brn2Duc|J z%%DJbMV7BGO2Vjfvro{b>FJ#|qlA0L5Y%H|vM?JmDc9nFGVOf;ryo0ozL7*9p+0I4mij;ghQe!jEG!%KQTZ3&D*SM~esL+2 zw)rE(9j%~2hddxBuIm7l+w*mN2f$^#@@M?pFQz}0#OlOo%ph_h4UTYH)z19*cP)Go zRYud`ek~hxcV+3I*!KZFNh}Pz$?rO~(O5pl(iGfrU)AG= zMu=eD9f(Gx(>3j`-;QQOEGrbkNIsYBkd_+>9xB5pX{U8LV7P`2|YJr1)+Tmg6fc} zi~sHH$w{raPMnX=yTj$^?^@^Ewa%ZA&VD^6{=aM`Pr3^Qw~vFRaNM~FxcL|z(ho33d$Yf{Me4o0iT?QcW9%IgwNJKj z>x9!|w0DXD9>JGNsY@>dc@KgrKXUPN=R3cmC6gb-=av4FT)1_`PwFb$snSXIF+_(X z+;MDJ=acwwlJ_4jbhP3&Kx6v6EwLOd+67k9u1Kbs%MNoyJW(}b_*3Aq+<@4|=`}VJ z-bJIMmk*U5%_-U?K#aV~{~E-Dd_cN)zeA-VUt;!b-lgqm;Kt+8C&LdDkvas-~j<+(_>7}7ftPrwvfF9xc?8bcG0yTx!$|LT<$maVEPC0MT()i$$w z+Y78tSX(wpoJZ4tjvZe1WQG0;Q5e66Efh353GlvcZ%R3boQ@%{GIlczvGl*2b4Cz~ z|6Yugb{IRxMwvoK3-|#hnCLMgy9OgOzFOodhazl}_Xq^2<Z zma`d?5LF6sAmahNKJ)~uJHf%i8dFYF?KZsi?fa|1Me80ydL83Gi>IsUH)gnB=)jav z9Nc2>m`x)UXC#j|IZ$!7L^%^_9;ZSEUz@^fx|NNaed>R2c8Lwwge^I@5o$N&H+9A!H^(j!Z%s zLN{#)HsEV(=WjMUz~7-p+u5e~-&ib=Dhg>r&r|IrHa`a5 z_zlElb!P$5M-(bj(6<6u=<`{rT`_6{$ubfPi~eTpKGvujrU%{=3z&S>8>?&f0i6$) zIX!T@o?TbA#EcwG-hC_>s0IkNvWk;hY;KcG3@^R-sNR$r^6t=QkgvsThxA*Ytdy81A*}lCT>a8($DWLR ze4gmd`}HPU%8VeGrdF%owzPD}P8u!5q8&k+2X`<(e8vu$+eSkQ(pcw~yF2$i@J0~` zml&VO9BUZ%JfL7u6( zC31~gYa!kngU+zLLqXsTDW13R{*d4rW@O93e(7Au$N4s0Ay=LqH_W`F94RpXF64w| zz&9S{;L$_w*ihF-LPp3@aFfr44S-@FpCyQo^VaX-H+y!6;%4ot{I_p$tkM|B`t`64N05^{%5s%^ z-gijAr!Tl3K;>lTn`nUEl$oc`}!7j@_(j{ zHS7jtpHpxb^QAD;Df!IEY#`Hbm7KiMBTmuR&FH>cjHed%YM)8squ1>eJWv!GNKL?e?u-w9TZA2`=RQwIv|osz$~ zaIh2CGA$+VTs&;#@!)lt^xq>ih3|ggEDME2`-kF9xO9##p3~=MXz=9`In+v?;~=bzpMTQ_t#hoO8Fg(K#OGwr+yvJ;jG=Nkhq}`M zM&4e>Qv>Y64TKYa!gUd}{RSmd${bp&_%9T(;{p}8d~2H`pvoY-mcVT^a6HVORfA;! z=b|h=pY0i+F-q<1cW%Wg#Ns$3r9H`vF1C5LW@QjZ_Bg_Q{*40g-t}e=|1}=AuZ@pCW>|K5|`SI z91v22VgRQIs_PrfJMoMph!8#J2PpHx>o{hoU$ajiIziuZz40sDY_88V@2)Za9wGp} z@>Nbl9jMCbkjUQ?f3B|P3;p8VSK*A+@w*c^d819FmSyG6E}naAE$j<@q3@Raa>Wrk zMUU69jqOm7rsT8RS>TO~9$@6X%yy;xhIz-V2gu6-^EPyfYh4TBx9?i)&(yH!z9%?M zDA46;%Ly!XB?nbMy$90?H`kP42b(c%s@<s55TbwW!k~|DWT_a1(dG^WDS=vfZcb&NrjFdRcf<9C%IhpzYa(J1j*@Y&X`;z>Lm9IlIDrD|qXQy7JOA0$g_e}-XX{HWKUgCjChc8_40Saw}$1l$+ z1ul(bK_p;+Mc{=8P7(aR!7$xQb4hJ0W|`Sju#L*uWDpPhkv?ro!w(Dk@7C|hiV6o* z{=O_|_F|ka3Ss&KH6lk8@fVg=OqcPlo?Q8}$@L`IUUk2N5}iR^KaySK=;G7B5a=c= z=r-{=${rtA)T-b3wf?)20JB^5{WLG8LPrsY0wWQ4BR-a-z}!G_5?-E_Q06~faFMDF z&=vO-+~Vy)^x6>O%Yn!wT3Z%ton}4gFP`y;+ba=vV;V2g%GYnT)ARwc!7Lm|9&a6` zD0{*KEqtJ+YUuK(x}l;9#Qj(GXv_?=)BdZ2A5X&EIe$i#XW`xd5oalsdjXfY>HZUR z15>W`$Zq@|sKTW@`Hp(Oo-0T}p3C;RZ(Eovw0DI3c=jIDC*CT>s#w#~c_FqJzu#av ze*!(9_1<9Iih3}dL#-|Otcc-pKnW2tEU~hSC~-ELkBCc#-0fI;QO)_$@zV-c>WHAx z;_oXrzZpi3-?Ux8KVZ53M=(q*7jC9KeP&tn^vNi_gBhjohoIr5xfjXBrnSH9p73(l zKGISPNY1NZjQ!V4%N*Kc(O{b@6n4Q<;U(o8xKxn8wD6{cP(m*Hf?cDHEpA~KJx*X= zj}YJ27Fw=yNQvb>NZEIu|IH=icl3JnX3%-P$CgNbg4fd|A=3^ZvVm!8bmg|1A*>}1 zKHxI9v09izHpcIIv9b6FxC()HRpqrY5-Osq5*z{A1hM@47+xpBkVc$8P?<4U{GGZR(??pmXYfSMB1% z2f{(T3-7?to)EEDi!q4J^q$HMSG6ggXY<7^3**2Y_E;z=mO*A#);c45e^QEE5{ct6vq~N~Mi9qUlBcpTy-yHO=CGQ-7R)+gEX~BN8rb4h zeWKe^@{ON|VL1HbdKe?})|@}yC2Tg3Vi+Mu0q+M(&p<&EFK69ABIyX<<*)-4 z9vWL(cDnJat1F7%<~syC7JHaIw$AtX;asKm{Bk||*PPGzk%J>=PBrdf zD1P`aAUyyn*VD7h7k#Sa!ZIHmP%fQsY?C~m%B&~)Wr)w0mA(iO`Ajv(!0n#CaX)0Y z9cImNyEa!B*@ZRADyj9;RWlr_ue9Rk_liiFy!>TP8;9Lnc}M*Q_l#Z^K^bIq1}apc zT9Mi|n=a-|KDH7(E*G5N;vZW*z5JYC>qPE4mLs4yPQl8$hdq9v=Sqy?>j!zaiLvYUcQ)@_lql>tV!##}HXi@uovX-=&rhb+QD->616ke`1!GmL|^aM&6S&pXMD=toN}sBR3tPm zYUails3l$`lYLN@lfziGL7DHiUrgYdtUvzU+S>a2(76b?`=(SY>s@J^zBBJ(epVr| z1J9p3tkG6X=OdxBZKrLXwjln%xng5+EI>rrJrwA!AX7b~%Zxoh9v=qiIzq3mZ8gScNLd+;(9 zKj*Vu?ieZU6s~!S#5tc|$3bxq%?=3vAf!9liYq@zvP_c5PukU-UWg)&S>f{PDtS92oIA^}a>btTAI1u}nOm(PaNX1sn2|Dn$% zK!4X7K9KwfH_i_GZ?+5Th#R@XQ+1pVW&za~%EK!ypU(FfA2(0Yf(aCtf*|=- zowLc0{`}~T*CXpQw*_?x^9>c~Jv)M4vHwt>Qab(!!*YYy_vGnK1TZeO>$-etcg-;P znSROOz*5p0>5PnU(q1GfSvcO+Gz-7Q_qy?^MkP`4C)>{+>qZ%%HnM7!95{$Ilt?}A zX@ZqSOXi6VM4t!{j6R)9ylMN$jKfVU+_(>FK9aim-1)={BP;q4DtE>qXtc!*@W@Yi zR19EW4SveMD&jV}nYk1G@YBl`^v~}o_I``p8Ptg*#DU&t%XCWtg_H;zpMFf8pWY{B zdWFKTa=*{U{-SMr^-q`opGkbpSHwRl2>2#fR!)(_<&_iMCQjiair7ZiqKL)h{3Xnl z@BVlryF1%;yJG!3bZ3|&?;7>!h4HHHzWJ@{A9H6yE3dT2KRA_%fmS(E8j=aiz*^-b zGKVZ_=6T_tfda#4bL^uhOx&K_HQo&?hUJx!7Mk0cfCHIf!@?>xV;_nd|1iW0&qg}# z9h%szk?!VUJbp$C&*VSVG=+${0S2M>J(WyU@O-!E)BUwl&gxgsB);>l0NxWqj@a)R z(=Uu1A2Rq*T<-yJ&GQpq(Q|LjVxKsTEdxT0mp1N;pGbVo^8CJiMs&(x~QsVJWwdNB}wsfEd#R%cZ&q-?g` z<~MkN42!m`Y<)2yue&y-p(|tHOwEJ6P}2U4QazKhOksx7y6E;zFcB}rqP4g#ao5A~f0z^!T3RZHCbD#S|$|8YO~8=~c#)pGA!w)3~&QLFJ_@KsJo4AL4Y zxtyp^9tqXd0SwMFh%NNa;t2Ts4QkoI@SJ(xhy=Px)gF|wh- zS3Z2FJvo)hdrOu zX5zL8GxM3|aQiwzcUkCXH2&6TJvaV9Y*McB(F5J8kK41`ex@e;Tsi9Zu!iZL<$N-< zhZ!wp<=sI%eZy5$;P0?AeHsq2qQHW0HS6a{z6??VbF#D}4)dcJ&oVp$s`;_SIrO`H zo*e-X|5nRXD@OK-kS7AX`Vii$3a{RndQAqX_dj_EFCG`(OOZ9)X$}ojvZy$`}esQLo$I&&l)0#ndoUb&oODCNs7BRBlD;k!Ioi-XkG(`9-EiB$%CHJ!yV?8zhV*34GcVq#Hj!zY=D|8yV!-wC#-9pKNL9;^59 zak+!~q=(DZL9__r$|5ixBZ3clH_8oLMMizl_vZzRIRC7$Zscy0^rHiO_H9 z&_WPIrT0D@?Ot{`ECx-y*OK4(#2=)kG(l|CLWxg)r-BNe)&#< z;;X{ zr#L`ye4~=7+|im%^*TVFqeXX_KcVdXd`hm$h^vAYTm$w!>R%vh-=!l=w6iZ`;NUZ+ znHm&<;oWp6)8Ec4;J*kNB)>&#FUjVN&wPeSA{N;7Bxz{M{`u(T;FMz|wvYcvu z^recj#!HJU@zj3%WuKEDsN#)i&+8o?Yb>F-hRyoE$*(Ww|M&})@$L2YW-_v9X!wv`M)KM9SoO)}!4iTy zD`i{Tsx;T;$5q_s*ZWA)ZDvaA9cR(JPZt~ZJ%3Thyoo??)SU+d7hv`QD9sSfy!{yJ zMrYs4*0~9H7kd# zV1J0a=+%!`gAww$JnR+mQ?LB47tPu)DSEH3rW- zxjsl9)@gLFDC*|3Ov{~wb%YchmmTpgqHl?~9CVJuqh903uQ^bjBQzn2Oe`t;pj-+_ zd}#(TgRb7^joCp#WqeJ~1{KpI5ADm$k_LK8(CDWHDQ=GxdRYY}2S7?@|9I*>-$M#w z$;ItO6`5r!%*=4M)9J#gtk&lOIWr|K^xHL|idy^1)|B7B9}CJ{iT56V|Bd*sGwS?j zu|tOASKj?{XHY#hN43x+6@EwV`-wo%qY08((+st2oWN=|S(3I7ZeJ_!|tb(0O zn2nr<9e`&%#0nxe7)@OsDJPwg}fiOk&>wy-=D z@w*qCcl9#Y8ZPTv*di`&MZGmGkSkQ1<>Kd-*WFhsi}1-B7GENGnZ(UF*Oa`4zPa`8 zK?cZ>p25vJFXLnNLyKF%=(Bp^W1%+^6DfBGt6Z}y3JC{Nol*r7e@vnYiTJeN$TG(} z$qaQ7Z+xCqlghI%m%&%DZkTF~Bugt-o!IA=y1FUJv(c^Kv+3XezGz05zLZ6DBkFo1NL z2znorbrv5xYmYp!!yRY~YJSE3>TeWztwM2|$t|xdple=#$I01eoPKw|m?^+p;`QWm z*&>$K-KXW5aQZANKbdL;JduSe_fC42Q&3ZGZc((CpplQ=HK6gY6{Mi?`o2)d33~t3 zUS@mqbZUQ`-s;M;X*l^;-~`Ln!RMVn-78MXq35Ph|3vY2i63-pWi3SH&gz0-)bw?C zf`VjNXUHssDgEBVEN;bHfv7|;`QZAj*61;thYC#iIFY~91a69b=cfGFH;6Aj--wwu z8*RdA20^Zm~>Y!AtfLn`@2DN(xyX;!Ta+EK5F5V=>4 zFBDwR`+HjSo1Tnc0NWKLAw~uq3_p*)r^5~CH=UFK3wIFS&7Xe$v_aVL%ZRGube~X2 z8-FgME+Vd2rM{N;*faXT^Ib+GeF#&aZcO$*cUgF1Aj6#p{c?8GYrY$eUBJZ+ps$dw z9Pwm}qz_#XIElDpUJlj5w-a8(*>Tz{SucPq70?){JqTs)iP)N)05jEAZwE(J=Eh@W zI<(GR3oe1$X&H`9;pNkgo^!CUxMTez%op6>deo!c zE$w#(-B?%!lreR6Cr@I+Pk2+N+i4~pMRWP zU+*ip>I@TLc0!P*G2g!=)0;w55@4hWs7&u@UO(@4l(z=eo+;3K5mKMrHQf0V;9?^n zmKqu=|E3j+GV^N^+t4uu6!pnf1AfYH1F~Q0DVQ!5I3jH@o`xWiBH&BO!IrJ=4L*bJ zchK*jTM%Lq(Jz^Pel>1v^bD+q@PtnGr!S=l=;a7LUs=q;QZ3GThbqg$npEZ3R}%+p zNYD2za(S*j*Y_6CH}>oH$@*6$-erEEf;K&9w^B^g`<)d37WwV>LhqMBnt;?q7q3DA zWSQ-9=?s45KG0EGuyVFwgA-A^|eF>U2pgtQVkJaDl8 z5^G1Dk}g$M=Pdoae+zsZQgGGAS573{UQsy&PT7K?!F`kv0Rn9+|J*9~QV2K1*Dgiz z{~a;t!XS1vx54A1-1EKf9POiL+7?FdZV=_<{p`yOsu-YINY5?Mp#Rakd?V{k4)Bq- z|9-|IvKqRRLpO*%vVvMQZSHQ34?{=ynDZp&Js!MUA056H!kK!IfrXh^J*|rtbFRf% zI`6O6s~73z@vqE>iD@i`VpmbfwBJ6WJ}Vp8q`f)BH;eZtKxbkss~{Tj_&LMl3uZUJ z1u~Gk%v9D`JiW6)FZVZGBYXHtpoe8w4Ri0*J-R6gscI(-2obVIc z4hp+ah%9J74zwM43^earOc@^_zPeepusT#70Cfb<_r+8>3n^RVu4ZJE@uBb76`Y&7 zk8BAf-!9140N3>-N=R0jI_t84gF8$?hv#D*wdm3{=92oZ4dxSOi>Pf>I$*0&lK;11 zYt>fvOuKKpA3To420KCFXp6r`bkDBfX>oYZImdbu)HLGEsdeqPXAQb=&T=v8d{_)7 zo6akK#GkD_N4s62yP+c{JaqzZ@wP>7ilO*0glP;e)=nF!o zv+Z<7Us-o55Fa9#&Vm)Dv>l~?m%zI9HEG@6c4$hcbAwkmp*f6h8KNi#M`S(zaAFcR zn9YWzJn9eKYf7E-h};Fxm2GN{ZL(RQK_6=AWRD2N9inrzh#v+P5!Wd$rdYgKLA7lC zDy#IQ-%hX8E=C^0Tq^S%T9dA8yV|kNrG?mYkMl=XPuyr(v5|J*R!@Dj`O(Q%DKjMB zxAh$etbwRM2?T(PyH1{wMC)hEXE5#EiImafsfUL^1!U^A6#SO(W=iFece5=H<`=kuE%LDi+wvQ>{`<@(!FN2zY z-k9p09_vEiXu47S0eLZ4BN960?lD{$miu8==sdYH+@p51dGFxg$UzeQGwP_2A;aH7 zxG66&ZTyPv#0e!x6R{Ac3K0zi5t{#=8|_eAn?`IVW5R0$>$WS=b+5PydwTT;S7bMy z-F;$dh~K7oE98?gZ&-@e#}pqFvd8aekCeHVwR3=KP;bch>qCgFtxcLW%m#|{9A^y| z`d_6NmSSikq4Pib^A2UZco-+$=wi5Wc+iCNJ9GGTo(VN2%ghqpmZZ{o&~I`rt4Qzh zug^@Su2dafJoGpIr27s z!G@#=`_X!y-^m$jGUZsmigcxq_UANWC& zT?^ViZE(hHqjZe%?3DEu5CdO!7y5EAt_xkD6~MlK&=+4E>_Z?~Q5F;$#h!j6z7gg6 zm=HZ*+EH|9s`ov{S8b<8JG)tZdu1J>Ft*K0M@Y4$TZ@>}StD ztIN*z9{wAH#x1(v`aC?n&0}8tICZ`A<4^qN7M+*^wp^62J3je&ZUQowMJD`Rld49q z%#$5P=ai{r6^GYR=_3FV{q@`gX#tiBdPO^M>|c;Jh(kHdL&V_a#QiG z8lKyP)aEQszwA!`>YrV;h~%F|9!?C4ZzoMwTT@*L#eS^kmWJ4C}jY9RURU- zv_5ik{84)G^8;tPziM*by$NFcoOs~0Bvb{Be;r)K_ePh6#|cQ?xy`Ic_yHYQNfTJ zt2HM&jmlPA+zEb~VEyswGJCxB5c-Pw#mOA|`<#MR8WNE|ZURE4!Ft+^O}E*xF-ptX z75hT}X6i<;JWVe1vD1r;hvUJw-2AykXSY#!HUgttZgci(_b3&H`W} z>{@`qO-zr(!Ml})u1SOnlXn3l+)&i`=L@}4LpzguZ#!nazGddjgOK#liiUuIF}mFY zZLgNB^DXUE*W$)IGm*&S4s(gmVGFt?;|elU7+QL)*SrEZYm36yq7gwO`%G&`tjdXC zLX^F76<{i?yJ6@@pOnOc4G-FPzNV$og>IYxwve;OmLGSt*;sKke|KYH^EV_*oFO+Z zQMDH=dxy)%BH_y)N>(soppTLGbIN^GTBhX?ZKQu_k(3&N7dIqKC(EXMdi`kbO=jU3 zv}uds8+{6e`^QSNjGl3;-@UzzxW9E==rdm9v9nj_^+y)2$i=6g_{j}6z&W?b74MT-y6Y=<~B+6f6_+=g@ zEF-Ay*)&jt-yDm!heO_;b)iFGyf}?S`%Db#1)nNhrT=vW*c|GpMGQn3Y?m92A1zF6 z#DP=s!QA*<2z%0BhXzM2w9)&{VPQb&GSVvF`cvbDd3b!^ACI{^+UMsCcOj|j{UG3= z3?b_}Y4vvgdVQ0U{SgKB;n(d78ww&`iLv_o5?WBvTsuW+lttjX_%J)+Z^;^hfEN_y900L|i zmOX;p`19<14US4_9CPKQC$CbDbk?_uIi(4i)1Sp>y4c@u7nI|7u9{f-XSPL()lMGf z`=qY#sz&%|#PhQ*DPLJElGu4F9kIA^HzJFpPklkUDI zp;f6?_+rKLCxyyIx^f*k1gV9_JgNwbHW1*vrH4VAO+JliAd_fL{xkVR73$??xDkF_ zY4JS?E9Af2xIf6ah3wQgPi45GaWOWTkK5HYtPi}^#c*>56+;Jq;dPdaC$@LHdT+X^ zVCr3qa&Ng(O`aD|-;oSZS8(8+UEDsXC8${w?%3UIqB_!cG2`^GuDwQ&wKl!0y~cg( zZ$HSzFx1a-c?4GK;HBcljSJiJC>aM?0pk3)&ILPsMiwR14U2al=NSy9RUmv?4f~IV z*_jT=qzSd-4vV-Qqm708rX{1S8wdK}^1WuvtbONi;rkC<{3C;dq46K$XR{{r4P9 zX;ha~4ub)!h%;_NV6=^IPV1&TndltVfr(vFAK1`d-pgT0J_!DO-+Q2d8+JA7Tk7E$ z_+B&bR}h*gWV+dN9M#V+#=Q4TfiqYvhUhScbd;ZA^V&M9TM&tlbxj7PZ;+cnqrJIc z%|tq6bK#06!~R>=i$Q1qJ&rD*=hJdZoWK`stcCbTVA}40ndN7^`)5zqi>nsuJK`Fk z1@6+B%dY4Ni+EtL)~le+>*fSvN5fJuzxJ5o&J!rzuDzFE6Gv_G6Q92wTiOhIjnNAI zC#f6{{RsG`bxe>?>iHT%)Yc|;!?GSE0zA>5nx8wfg{g(vmfd5o1^=xfFN%Q*Exj}u z@v{@>8b&kZ=j4*NDsY%grPj1k)=@qhu03*2zulyMaylG9%4D-BwE^s~Gd~(NCH&E! zft#5oJ?Hc}oFu-{9ERwbag=lpg}}7q`4|M~CC#9{B}9K9@iw5kD(bh=j!XrRqC`IP z@yY2}8d+?k4xh@=^`i%kqOGfn2E#H2*6J}LY0fW^x3U`XI?|bC`iIns?Jl=oHh82D z9B3PKAY5sn1nOnvN6sFF^9|u+jI}`)u$*^6h|TRLW96}NcY(`tRN7}$g1tbIt%q97TB?qmZC zvkRV+e^rHSq~z5sfz0bl$PdgirP{xVHAV@|ykTkB}D7p!5X! zCt@ddyZ2fJ)S!dqJ^3u+W)t(|SZ5q@f}k2Xqqu?+FZ^B8d$y;&vSRygO5Y;@kzZ^C zd@%y`WW|qDb)duv!d?3&&{{SxAEb_!_c&u{zIIz5gYXvlS^VwZ^Wsv3V6QEvIzNFCofb}xWf*nXYH0@p5EF9EpQ~%B@ z9>DRBT2K)-h87m?9_$7AFX7%T*jDbSkoZLn_?NtII88MlSG0D-n~(2S(6X%H#T4@aOqrs!28aAtat5 zPcw?I`MO=X=e~QZC`7qfI)Wgl0paQgo*poFdB3Xeko|>I@3YOTeik2elxGB-@I4&@ z(xEZ8^1c(HKT2k8JJ2_fBf{*pe|!bhEi}(6>~Fnf0^zx3q5*h0@fv7(0~XMITrl;b zrO0LNsp~xre;`z2y;Oqt1L@xC!mB%NHbr<93wBSO{B7&h;1`PpvaVkg9MMEeq&oyZGv)pWpPnPf240=dnZZO;X_(9E)3zjl>h9(th##hxa z16$FIKwf{cZEI};^CRJSNoPj^$!wv~hA2;yF8`{B4rCPP2OGQhqL0odvu3Af%6 zN!o0|34AFw^~LQb798vgxBQ!E2$2qmK|h90NXrrC?W0+B;K;ef^KFXQDY$~OJ~7#)NF_1SXr5ZTqg=00qQ{H##YWaO}96d$|0QN_G}p zjSnud^0VOh^D1$t55KJ)rcqC8Yn7ZzECEZv3UKZL&8P0Ba;PRfNm*8zp5XHsEZwTR z-lwh!DL1IOhBEFg_%y1X@6lsjiw$nvmM4U{Iz;Ar2nDW%xnvp?IIWznrPY@PwwLq` zetS7)jyW_~P|~FN{bpsg=ci>KT&bdtwc%){e>Ci{0SuuH0UJUv=lV|)SmOjnbc=&k zww=q-b>QT=(SLu)u0QcKNa*_f9HWo%^ughBTI)~VY_D$BP4@yn3H7L^BF^LwTF<(? zi%y)ERcyRC*v1fWy2*}v&`MC#RD18bR&q{Vm8;tXF575`3DXFeONbeq*XPbLLpx>e z3fp_$Mu0Az>tHt?J(xUOhq%V3SyCzdS&7FE;gLrosUiya<-P11?@x6I>rp4SOB0^D zmh!TX^mYd-iW17x0>&JM3@cfm7^J|KRoNyK8j_(U@Qu8JrsrP2)QRJgI^7S zhUYHA7pY7jA~xAHw{;e&Zl@cJs23unM_%y>#mPc5lxVGW=1K{}D$`X34-fB0C&QU@eG~GcE<7jy9(AAZd2CE6WS!Nsxq_`0l7|`O63$b9VwHi^4BV<4gqP5VDW(n6r)tGPvQp|PS3?fDsnxmqxFb3iw^-oS$r6aLO8O6iklO#7 z=a~HZ6y=XJL*V#}hF}$&u{k$2W9LVwT9D^ArN$5-6k*3`Oxe5Gur^v%@#fB~|E~$`z z)7XG%PT&x9Wu6UBdFcuNVT9cgoS9n`{ZAwom+7am1>ZTWl!% zzduaHiy>;4@tR;qn>TXs z`zMZ$K8NwUNy($G_2Ps6=s|7dQR;9fCCw#Yjh^eI5aJs^_ zt6^j_i!5ZLFUPS$agl1#T^-24=a^!%XSFx94o)7EL@XmI!)(HU_A8V;5B0pMc{?9a z4c_yFE|`j-N~Hman4}*^Gw}-*WL% z#yv-|>QvBeG3Za}Nh|Hi=Zo@M*}exaFjbcRD=P~d)+rps&U^ldMHZ>FaxRwP$MPsw zcnd!*K?vWC{`|VturZ>*{0aPQx>Y?kKURzKln`CY%#l)Px${HeD0ZFs-{bX5 zRl|Zy6bdQ)d2S)uX#)tj^~({*-!LIhFeQy|5$cW841@K(TiEJXROM8_DqLW^vA(7X@>`Cq!7~dKHp9u z@ba^A9E;mzYWJ~H^n#Mtsu4j%igXF=Jj=ZCaedW@#3vIS)reruprD1z(MP-wSBYkr zL*+1`!u~u8ZkLo~IB$x0XU(8&Wee{LWN>3=+GmgU3pD#y-%LWcSg^#Gg3xE$@&!Js z5Eu8!Z7q`nZ$5ctz2yr^ZczFm4G9wy<6x=Q=wZb__-x#D;=(N;02FS!CV~+Qo zZX-6|f??gek_%Bhb!JlVVc$r8Akb%9?YON)kMPP<7_4_-@BS2r8jaMZJG zIHR(V9cM#z&kS#YRCiH`gG27)wKw90!fHp_cc(0q;PMTv7o(-0z?Kd~OpM?YC*_YQ z8b*p}htL(QF98lNrXP>3btAZJLCuwP(XoF^c>n0u6CK4P=c>E4+(Sjj3;s{g|2+WM zHzuJPU1HqFGqV^#;DjjwJKCe^-Bv)#n*WO|XZG1xr}vt2nwX!jWiqdLF^=>iuiUDO z-hSe2y!+*dqJS6fr8SL z&`D(QRYSm*K%+NuPAzFeC^;#fL_NLqZ*g63iK}5JH2M|I;aywHBHf%*^5+iOe)qrv zfOnVT?JC)}&8ph6*OAP~6KSH+ zV;z#xlp`Q{36UtnN5$fg~b zOkKLo4Z63uaB%$9d@(*0*!{}jo- z9BB$2Bl{^VW3aGoLXRyhTkRpc0(FQSRBq9iuoCROdg0(KF76_7Pvyj`;bnpbzf;#b zKKV$isTxfpHnW&*R*lY4n&9@*0gOvN&pwYmrXbU6iCDYBZBC7bVY`0t)=&67#uM=; zYwvRanYzz+KlkL@`yEs&usl`6>+FpsgEW(>Ph(KolLD=rc~4Ya}kKbF|5I~;s!^(Jj4yW%?l zeqThp{iH~UEupZBjAk7QV7ejPBkC?lUmk?gUYm*!A;qdst1pWhD4^aut``6D$ftp0 zB_DOB?=rFy_59HzopY5ty>YLTKL`bjJyd^a=-QuY>p2#!my}3w=V6e`yqZAM-W5x= zjLP@dRp8ve2cOj?8y60p63;DVuHRhnpiDVQ8m{Nps1eGl5Cci(-FpH0a+>EV{+CtQ zl6_L0EH65ZAwoOof^^Mvx`hucf{urueJjbm=|%Ii;QxOray`hdxp^TBx)f-v?J;H= z25BXlLgb3!dWW(m!`l4nmsKc>KUi{5#K9t5VEk=M)#`g#otNWBu?DV|KUPhe;v?La zQ3^|B%X)Fwyrl4VSiy%^iS;g4en>xq4>|N$=(j2i0e5|ns7Blfwx`rmO1u}EfQX7C zxZZNH`*g{ar0)4NOqZ}8l{{Ryr>8UecR9}XSC#1YahYT{i>1j{wcN8biR;{+54?ZB zt$xi4JbeyxclK$aPdCQU7vwO-cYChH?l`*;Tf<2aJtH?|O%oRQpuwrCXa!seaVO{Y zB$rYs?li>njd*-yQO4Wvq!|MgjY+KQsUX#Ll^ew&ugsI2dc?MnM1`G~?vJ6rGi@QC zcEOZm^*=gIUFw)n8%cK;yK7IP#&$i*muI?JGma~Bxe4S8gD;>W)5`4zE|8{hM>rn5 zzBBx!B_O%S0ZUtma}TXM)A!#JW}Pf;1Q*z6@U^K#MMheg=_HPg9aDWbL6XcH)NH(X zK3h?!Xfo~&P0E$nW1;qV0+E;{T$pIQ47aW}a|Celh3mgNimBuA68>ya4JcJ~T$Fp@R(>waW3!g5l$O*X0KZTQw z9)nT-8-W5GaK=l`)%9>RY^jVcd#pkZ@NB(SMVgjd(Cp86QJ-_%&9h{;k}Yh-#Ao=eEy{{FMd)s1N~z@>oQNTew8CS?46(iqD4%9$>p ze6GU(FSC%B_*a&RLA6*`>uW>k>xtjwkDOiD=`Q;IqIb6)P7N_mtyffTr56BJdbxHOSwZ z)HH8S$rg0~`!DXsk(ID+T{jxrjvVjSHd(kyht0c|ZwdH6BQ<54RD$74K@#zBO161= z<39(7JB1qm^IiNr;1bc!j>#fwe*hE=Ip|UL6Ro-ND#k{lHvWaX9{824i zZB;!{T0BU@V+lxUeRB~_j6A!2;TYRG4LDuU$Iz1LT`w`6S-e4gC+4*q)#Al!^rigYxmfOpiB1<1bGu3| ze%0Dw?l(mp!j8u%o-)+GZCSY3J?DlWN_By6RO>lMo9CV``631P2F9?2!0gq9^6!fi zIlLXqzY*exi8=paq;iWkW7U}>6D^qA;{}dlos#he0ynev$!fAWG)f;lNcmV7hxo)x z42SbRS3E78?o=bptgQfx$@BrNP%%S~IPrgrrBVEUa)5I}{vxoZQZtD(1V}D6KIM<% zY5yK8h6Y98NeltI|E2|BNNaiBy$Kcu4|K}X_>VgCANb!ffk^IbUx50y_pUVVlWyI- z2BQ}$BC62(FPJoO=Ao|@=?e}twoDIQbHg}+ZRn>Zavf}Y0LCpIvxqE5V2GBsX=Zz) ze08&l^?hE9$*8e6!55AlH+L+DsW%`ULO9(c6ksL1s=jVJ524#jij4kyqH6C)@bquLW_6m{ zfL9bnp;v7@Jq{IsEqHx@Y+>_sZ_E_v;xxxpZECuTg0@O2n6sYX_t9v0Q`<8-mEeBI z5~9r~AVb#dS%uRpPn@eI=f8a)zLLnz`sKnTqAebU>bE=9iYrwNyr)I}olFiEBTlZWp=$VY%W zY8~ZQQyvNcMPoh1B& zfD)LT)Z4?9Sz^m-q9>VQV4(d|)X2HQcC08vEsLM%(de*fnxP)g(U`yy1;gcibP}I& zsY4=x_xE!}IGiCiHF`+xo<2t{=Ni6IfxrGAS&ki%sr*WjJx=At+ka(f6FlR~6FhN7 z-u7Z^q2uL-*f-F{T69}Hvt~?tg+k12ee}EYzqbgll8TkIvh^_@VaZJ9*ji_XT7g23 zaJAxzrTaBWh@AXYS{JyorqzSDqC;~9_<9=>Wz16kcn*L0A7Wh1j&?mm9O1g1Em+#b zfm3*L%~36`V1|-{UA>ECHQS7i;$YRbylf|SN82i5$N5vrj&mJnHM&4?BCdl%YAD2q zqgqje>-vCK828NLrS{wr7zr!3i&$rt;H;IfRll7?C@1cI-4BZ0`b{kU@bwa%Kg|QO zd`pP>Qm8xzi9y>%>>xC+#*o-S`L1jHIBk~CL#YJkmy$fsOS6QSmer1sAHICrKtJ>T zY>fBx5o$U?wiqIwQ|C{Y+TZwVM_X4i%F}#}ZaZl$%BHS6^M@5vtbA%f;H@B|D`!W^ zB=M`9|BtozjA|lm+jd2yC?Z{u76hb-f`EWXi2{Oxh>A*+UZo?w1_UVzL`CTk0Rbx= zK`EgZ0Yi}zT7b}dsHro1c%JuL@3+?evw!TBKV;24Gj~GfF4uXS*Hz$>*3;Jxih{g#*#I9!cTI~%~cZ%9+TUCOFPfsz72%~ z_zdIJ<5M8~$V)Hjz@<%G7-$FBTz>5KD0yhqP5=5HMY1sZ2yg0 zYsd>|z+4*Oib`jN=*S17MokL`L|m#VYn^%Hh3nl_z^Nhz&E{);=zFXy)-Le^`CsAj zbsDrDyOQc#fo{PoFb==?pm((Mr^qx(ih%meFBBsCTzhN~|Gf&hV=m*fSEWIjmS$3; zsJUUX=4Y;gu50xb1p$$^!j>@6{C$taC4w6nO6kF+>Z#}XA;v#Rbg~L1rD#kNnlxU0 z7uF<8WpQK~XoGjQ4!i*j%OMduu$(f`iH~v{D?r zglFSwD-LxOAAHcAVr-buPiDEXe2B1(hJw&BC5RKJ)!k_F_%#@5{TJqXdAz^6z~{?S z6ocf2PvR6V!GSH<9Q)S-9GpzUn-AP0(-aG6dNg2Y3;k7HbvQL1vbwvJj$pRj%kU{- z=}yTf!$j@v7BTqIcKw@W*K#El^G(CLHO1aMJ2UUAzn-b-5;@O>Kb`7E+DV;_V= zS{zFb$Nmvz^tVK}BUOSjadUL`(YC?IH+ObnZ}o2+P2JaJ{q*O{nFGJ%Slv2mG6Xdd zr{D$e{)3NgA7|?=e*Eoel~H+p zs3Qlc+ZRNjDl}(wj;`*ipQhUK*FGqV_+IHankDRbn;j52#=q);L5}CBy z;i4k6ET_TfSG8Y1ab)DtQlR0^aJBY>k3#d3tPx@N!?Qv7iuGW0*fu?faGLl#+R3M! z$m7QT5+f+6D`@(!pI-dxNm=*O$|vdV8a>a$8u!7I@9=Z

HO&6>t^?T?L=$Xr-MU zrPfubzr_V=QJMI=16L6D#R#8uq=i=yw;beG5mmnenk5k{@a@};kTNTnpr^-ks1M4B z4kEP5Ye(P}h&$Ga0{-VBy8!x>s=(J>Kxbp90>9Ab@EN|*x;6#+x$9niPsy4KbN8Pt z${O`PfT|4q9=w%bEY?@xB!&J^#4c!N#&eOxbE+k!w)k~h%v~ZE#0WY3b0y*WUw`N6 zJ@p%+caG`IS zGn1Sesm*V11_pD27rAxBYz=u6a+9wTkjGg2Z=6VOdj;GrnU>8rI&^_3s#6RDw=WGH z_05nf_WxpRBnhQjH?MN`_(1^FZ7z_x{}`*z%XnE!%4y5OEOk%E?bSaDg3>Qy_v zE}4d14ynC%6*y)q3l%?#XaQyn(j&x+n_fHgZ_HVD^!y3B_3M)5YTw5?b;B;BUn`;p zXMb6exZh^{57Z;sU3#BAsZHZQ)K3hYj{jp6vtYsz7}a2@gW4QWPeAaT%71HG^jgPA z?>`$Z7m?$7cDE1`CjTGVtN(&g36B2(qx$5))Y)%xZy-ly%B5@Kh!H$YfYhBE$+eWA z=4KkM_cJp|7>U%$?F{DCzTE%HiKp81k73%$h~9e<(&lSYx#OJOMmm4e#Wz<*9A@D$ zC-THGSMu7NMUQ=-{tppER67gZY0moQ4YJCo6Ix%?TTiT@@b6~2fJeDkjrRQ^wfwoX zB^HK->D@R8{LIN*xO1B8XpK&}Q%v~8S#;~d%g=V3MzV!#zj?p3O2t%cA6?$=iJ-rO ze@3RKVc+k;zT@o55Vpj^#wCvPx#NAzX8(y%lB@g$3%iW}a78Z=DmeFmA;Td&V#LfT z5M7tF)!2YVP~&>G%WqE(S)zzAzcTD!wJZ&Q;L?7HjPfWI|0YZ*wHad1pr@%Vf8;A| z&7Hx2*fuH4Dkd*Jia52WUzhoG%5)Z1I#lc)@!U6D&D(e{_R3**WXzl@oa-ee^$yaVteV=k>?-wI_R5|VIpPctLQF#?oJe0ue`b5XJ)UqaEcp()bGz!s2&~W&KjGQhJN!g%pv5KBLZf21 z{2l*O#GJNxoUm&^GsC6W=>6|i14UUG8^4N^4(e*K+QLphK_DlFVGx|i0#Q5S0tkv) z^_2bTMVHDx_xiO{0z%|#l{kt5i6oQ#B4?$Y@Ooiq z{H~n)8p^T99m-kmq_*i2&~@HPPh+TKV%jmO6#ld5dsJBGv$u)x-znM12RX6e-7ZfP zjO(^_4-l#*0^zUd)E(?_6j0e1ypBF{T^o_*C?-iI>s~KKHP$?VoNuH!78XmyPWx>$ z`-L%opLlbZwbBS``CB%{`Hf^J@|y4*cf*a{sr$ptvLyZ0vYy23#Ff@5qi2s&Qf4Oq z$gzG)o;Q)YqAVX0&jHNA9!3AYOe=`xu3Nk}%!x#0(CjT~=P5(|Z)jh7U+C?kpoLs1 z6nsZ#pKnnpp=}|RCt{3ty|i_9OBZoFES77?_B!A)qw=Km0&`5SSAy&iwY4Wt}uY{{!F30C3&p-FXuI&-%Aq77h+)+Ciqk1VtQenM6z%X-dS+aGd5 z><3h0vRwl(q4%!ZDD7x0T^hA69B({0*?eJaDLoz0UpnxtxZ0#QU#E{o+seuzFjS($ zz9sbsQhwlst%}o&4$k$&mEEca034_U(VoeJ%eA8!8)f#bZsfnA)2UaZK)W?`iWv8J z16P9^Z9P0A#b*_fsm-rti7<5_%K_SPg<$h|nNNfN>-274l=gwq0dl>001GZ#;BCe)C5`A)wV>&T%B3R&-Xz0J# z)7w><8Beq}zbAz~x z0j3AV+C?ts?2g%snEjK}7M2E-dinJ^cxNA%!Xur}2AEI{u60$t$R#p~@K^FHa7aVx z44y$_q3!)15sDG{QNd8KkQlTqx>y14=7B~>>m{w+S2_Mm0gI?54NONVLQGh>Qv&j_ zV2LY1{1lBXWwqH8mGFot>=O(7=i0Z5!~Z5d`2K-$B=jT(Y#&!AKXBTj7v3TX2tdBH zK4B7bWuFS$U)n@*?*Vo~cgfZlEv(pt`dsvYGS=x%&11{R*&~=6=bTc<2#$fGuLZn* zf(3mdUev=aManK5X(VxfhfP*NZKun{^ruJ3d|K|U>ZqY0$7Ysi94IX}ojbKTKxpy( z#ewBfCOGpMG;V#kw49K$CAd&Z%)lxpE3_7iG42${XK|K0?HJ%zUHj|&)ySQ)_c&B8 z*ljLY2Ss6-G-pCI8@HLN7tRL@&qHepK(>Ua5fR?S#ynyT{wK*q>4&FtV|gx zSWG~XP`Dy1hSqY!@GA#Y_AZJ%rPH1vc#8RMZ$U)z3 z$h!N2-mi>9^bF?b-?Xrj?&Gq|9$s_lDx_)5^ACt#i>#zfo!&kFU#jL@owySqx!IZ`M>~R{Yh;boLBpZ06iZ`)g1b9{yHE zq3=;6efLk+6ZcMBz}a#FHaSF-#77TKUC|~p-FA8g$(!+!M}z60&0tYX!2ICN|HztX zTiI$Lr-}z+01-}TbW4ltf2O{`J0)-{@%O6g!9PC0sAw1VYIOB_;ENR^BFmvMXi*Z zv1MlN;J#92LwPq?0K*c$%Z)E{NoHy6d;dJ4>K#UzPzFruI{-DDJZ0gr^7PypWpW>` z&21NK>Nu>`TYUArjj&$XFvho^WJNJQR7;i%|ApO2>McQQlEDQz?6QK5@9t%%&v8Z{ zkx(Jm&usbM!v4*8Y#=X1vQneS20R z8CV?n1MB}LT*~u`>b89Dc`|o!CFdWn{RLv0#70>(_9n8N@RC%gdA_uFXbZ!gud=4xS!=vsemotV6VMkm35?b~9wX;3q4lTO9| zDS@j}tspl}{a;7t$J`$^$%BROqNwJ+TD%!mz;XRAuWyvjS%=d-TR=0tb-O`K__!VT zz70Nx{s$%--4t4Z-4Xpykk7)MZZGeykoVA1(bY5`4x912i?H|RBg2vu`fT67s97NJ zbkk{q-a&-Ab2PZ)N$qDW&42Bx8}qE;ikG)TpjZE^Qt^1TTlgXw2dO~oTOoc!2nRYT zq=1Bowwz?!0&a>3mX-g5QT2G{N9m|G%tRf1?#(%2qw zUU{dk&oh-YEY~+pJxlHln8xakbsgfv+{M9tH9MYKnq^;YUbu{d{M#M)X;oNkyuw>?*bHa#5=TCN>INBE{}&w04w`jd5o6H_sI0S18Ix(s zfT=qUsx*2QXv3TjUduFa2^-uuU4{Y8*0$&Y4QsRO%eS(*4xAC51&20b5rU(HdO`UR zCa84cD8BlzR5qjF>*=6Il=cW6Qn_wB-(~!-nrLED7-X4o+8l+STXM)=28F!27>0sd zBtm~gFW>+Zr_vh~uZDKF?J7(zBYaR83=%*Ei7J=WXt7IEky%{pcZ`%N@cOSPCcGv{ z>W}OEm;{pVKnw8!m7NCJ#SB(Z8k~cT6QAjgf1aG5tuVzUd?sD9=`vzgU%4{?J4HFE zJPD(!In|$J)LX!HOHM2sSa&4xW0XRteBR`p?z$$m(g-K44Ta7SZcw z!$^N-Qh7IqrW(9oCKOXcbiSOeCXqqjcIFNt*UokBR<4uLUx zP>8S_xS{&zdcnDS{7zKEO2B)n`DwosH}WmbzgF-yBYv(D-FWGe{ddBaStB%}YR!QP z$*`@ESJ8f-&@6Rn#l&-OrF&r~mhGQBab&Z@KmE48Ln_rDVekA8i1_3PLjxBpFJB z{$}@zK-#JL2QhEE$lK&!TI1&>X}*3(#xu`H72GaIdc-HKF`nEMY2h_pC-!I;r2CTo z7g^Lg+w8Zvb))}tDMFP58Z8E6F*!SO(U_4->;ox?8Eh+h!PCTk%jne8;J?|)S*_RN zZ~EC-VD^HVWpVC=PMHo`u>T0Dwi6qqsJ^f1KCrj*OLA?_hg^)<+e(_yAVAEm4L)R| zPG@kmFq`*m9$SVo@p=Ie))C~ z%*bGAzZB7;tAPwZR1PKe^iIdB2&;1#1@6|)N(g}6s4?Qt3DI83f5G%k%0U!5g=w2SUxTi%CVc$-cu7jvt!h3A(-#?-riSnaeS?*J35>THMKoDkDkY5 zvkVlLIAFlW*O&M*2;6F(%)xS=*Jp3$;tqifJ17B3Cav8-{_M<%teU+q^_u%STF5Ik z$YvhgLt~efN~2J$QLR6PPRaUkp=OyxakF~qgg%~8YiGQX9TzT3uW=~FBZmR>Vt@-H zTqfG!oMY(SDJMB#CsE=IWKzkRN#bJ*bsk&sSlfq-~WaJr@H)VT>%Eq;hk8t*7$7I8DdSg*x5so z`|P}K%!PNATPB88k}u3H+y4&_Tm3&g>{xXQW=Gcb>#wsVpxYP1Isyb1#5N(5Tz#>%DOc>zJYp7LjOKQAF5(b zlEPMjPPno~`EeC#s zIGb{u5qjraX0nXIb#9XNd1336Dvo!H?bT5HmF5-Q&|D&GaggiCASpvBp?o z71zY=fOx^$C|V9HNJ@RV+vF?k)tMwB*Sr`a94GifMSsSZPGxfDSwRh0U7Q%i#j19A zqW;c|(3U&-&~-9#TPCq5S?#xHGL-*utV%Lc5UHoB?v|XIFWk9YCP}uC7j3F9)de3I zsWKA9WOd(_SI$^-aQdIUcNb!>c?n#f^&7V(-P$2D**eAs$TP3Kd1zc-_3W0xyY*7~ zd{W$%-UvV_&Cr^_#&@umb}JvzQ{NZeAsTCow0B(@?mzEFjdwW%fAwDdM_A8%;JZ|t zbT>x*@ALF5OmxnX%QlRfEr{FopZEr5Ic%Yj(~;yYpqz)raK^svrw&~zUc{x#U6iQ; zBJ*8nH{GBO=6cKs>Xwj;;_E5F>z`i1IDQ<_X&>*--X22fa{hpGW|x)APb?pz3+8N2 zhgw0LP-gRxB?r~liPFvCr2b(;uI^!okQnVNSjcTQM{nl81ra1IHtr-|&$ zgkc@GYs7M$a^Gy(h+w>E%DFE(opzKb&Ma*GY|#zHe?bW14G>Sq`B`&cp;dBK#lV_I z#|{B`E5a#vs%CQsTf~*=v)X{ya!xud#+Xjw9N<=t zBdBdQ30lReeE}US2s@wKDQMNTW*^xr_J1thfm;J~3mg0YmHMdkJMGU|e@3%a@F3{#YHWnRo zqyikhrUYF8ff;LAyU=w73@4xQaJ{$;lb-Aw=XJ{J~taK-56A_ujBvKn0QB5*glGtLf9I3pJKZqRdNtjti2e5+AHwY2 zFCy>9Ay{ayP;(~_EqY~tj~VMtdUwp`8#i`ta|?+oY!*;KvYaCDA-Ye=^ZxGR)i5E- zeAGb*B^cjkbe>X_MIcjPznd0hytEr+|8)dSMEBTS!QEz;f7{AcdCRbC84hajW2Drl zHdQR=Q-~KPs)&`1o=-bvp3L;IQ!;7CYQvBM+g(lSW7%7_=N2raO%J7m=v!*Q;%?^- zSe{q#)8`444mg0Dfvc=<)f>`f-D&zzx`D=oyPv+Rt*5WoAU<(L8(4BaLu8EPZIqg? z6Fu!7J_5qI^ zJfd_I_3|V$eI=QP^^6`~YPk@g@PO)j0-}^n&og!FpqU&Yop|cE`^*880cVL!i1u2l zm|qc6)_nfUtrRVk`NyI{y#wc|3D{%{zcyDE@g%K(zg-L=@BMxD?ir`PKA4i={-!r) zQ2L>Gv-dq|K@MM0bV`<0H-^CZU#pGeWqSyK)q$Aho44IyMh)LYy=4; zi?};tUr1^3F2Duml*#zAMeFaPBlG0etJ{~q?00&GwpA-1qzJrl^VR{QO(Qm9r|B^4 zwPK#^+rQI8=Vll~J%vSoS%T6Z?i#nr$PoN0JW-&JqSn*f`l@eN$*2dL`SWt#_H>PR z;B()$M;)jJDP+L#Ek<3&WX@tPXeXmnTIw*4uuyhU(8Z_t5UD4UMj zZafk1K}|zEEBHnV_xlb%?|Sj?EubClQdom)B+X_`13JHMf~9Ri921^m2y<%X!I{?s zH<_|%Q#jpIXfvc&%fycwJ8cS^M->PVis%ktUWHPZm7ywpa~K2ppO40ZErCLUADOs0 zQA#F@tw~w#tnOUp0(Om$&GW30lsvgf6X^Z3K6Ngq0RQtbRruse4ox51FtuYn4MJfg zxB84ozM0F+VvUcZNOV6AJ107>OD3xb#tG=e?k7ILEPv5UZxBFP*RENbOo2}bI)zX| zM`>(PlFRVwAuHXQ(eoW5pdn!wW>YzD;bVYtt5p{axJ}& z{-X#UL+GZiGL0hmz=a&*g}iVrb$orwdce zz3Z&(9vP}J8M7E$j{Yfhk}d2bHhLh{qo!2_gHfaqJ2emoE?=qKfAuwFtnkT(dnU8{ z2k*;M@RIk@T?6gXM=o7(;Nt}|#h#6?Jb@%!e}`McNOLpK+2pl}6R zcz7HTTixd?2;}|1hSqhk-@41@32#Gp$E@uMv=BTe%%Rt*!_jPh!FV$T|1x4p{xTrV zQ=t`MBG00E(&zxq?ZrE@M*USz%RNkQm}Qj>tGZRMIlZu3$xyeMnV;FCuQG9ZUtM*) zP7otA6*H$YX}S~h=oHErkeldoH&&lTw6Z!;rYBHchekX>GC}8R1Ru49Zb4>JQdTq@ zifmFTm;16jxCZ-$TGiy?B$*SV!ac*t(ijyLzHF2E1XR}60AGdt)zR2;(m<6D=ubYK z*?%hN^Jk^R(4iQv*uM?v9t9IymOpTz~`wNxpPrJSL3#;bzBN%julH7CF zL%bK6wY(X3y|iu}25cZI8N%fc4Zw@ygXkD7a?SmH%y6}mEaP3W^=ZM#I~R&-`hS`B ziy!t|vK^E*n&T`6Yf7#*B3F5Ba@Fs;H(aWzad`f_x<)nh+7(OJCBtp<=ZO!f>Yb7vP~&KXn$IK9XeJD-9~hyh1YMo z1uT4U;*$$yRz5@#=gog^rSkSf-MkPb<~y^j*f@{zu549v52LXvhk%qmt6l#3m&daD z>-6fNxt)&n>obBpj{a0l>av#l7a)A{O>;q-QQpEXW7u4Su>06u5=H@<@H>!O zcwZ)g38E?{nV?rVJ^fu>V3A-s^A@Qb$_)@nkzy#PgCD-`r!`=-LqwLw3|&`FuzX34 zT>(7W%9jF>7u6LzrL$RoXCUFsRwlK9G}O97~Sn>|cp8 zA0p`;6c2t4adJoK-4VCmB?EBAji9~e7cXZGbeO%IZtL>qDn$`tvPGkljrklZsNoA&jf#Dzem=) zdh8#2c4U{a==qdFM9L>uC`$K#HMs$~?YjVWMs~;B$sgm7Y+pUV-HVS|dI0WJcFqui ztAC5<-!M5K1R1fgmSg#+yE(WC?{9ZGU{QApU2UsV~OPgQ2BKRzykRfxXg5wJAst%x(icgLqWM(8#1a5xZ&xG;qDLAoszGRt`I* zNg3yn=a)}UC){)#Q}n4dAU*7;+^ny}>Xm08ucje-<(+^HLRXzWCF^lv*uSQNKWMV6 z;)uVj=ZlY2J@}svBh|^>u^KeqsEKd4L~C3cvdIrGV*YxNWjBS zUfIDX@u1FVqRqLjbRfA$wgA&1L(+19i5s}uYE==vgoYp1S$MF5I{}dg7vJ>gF&DqaF)zo!or?4qa+95TGap2@u4?mOV^RIvO&cTV}B(Is+ zS5`oV^@-8`0O1rABecuk3dTepO!~UVz=gkUz$tA7FhqpJtbghLwJkJ)pDsAtO;Ay= z-b_TT%RBHN5UNq?hTRTkf|CJfS2^@IHz`pqe=3WA-TLN_9fOuc3hr^OIjObHTz`i? zJw}gM*YPOg^d7>4h&oo2IVA}sAjGu>wIOqZKzjVIOhk%J^q*huZvYP$a?SVW=QTb! zJZgM&qc%t7*}DhL1*b10g#59s?E_ZB8!%WdDA!Pw{y8;oDLxL`OAUo;{y!D38W(Fzx57k5)ooTZoaa-}r; z$OT=ydj{nXA|Se|I49Pm61;5X3usk_ACwakBy}069qLwCtE>vU(UIxE1{N>+rlAFN z>|{;j@H#`20^7RXnR$HHzA)jn^|8?R6pqH(h@I}B5J3Uz81q3i?m4(>(f|+;h@N#@ zHkM0oO;KH#SO-w3ew}@*SE}<9HL=tt8qDIsUiz!T8n%xX5uj7YYD6q+_ZV_KxY9l@ zChQqd+939>GyoLaBORa2ZZfHZ>I69*V&$y-k*=8vXhFdJ8TVSTHokKwQpnRI3hwRR!RdUIr!|{uq}6FF zxt&Iskbr$8GU%)CXhaktoT(88)^=u2J@@@oY1lx;Zi5#P{KT@7)@|vRm@vlr2KiMc zy$)EbA$?d~+X_lskPK}Xd2}o`{WpVmg^ULfRT!cLuv%I?s}+~Lrw=!TU+v%8pd}YvOIZ_Ke`0=LQy?}U#wO6?eS;x`Yii7w1af{j0kf(cW=629zR_y@f zay(iSdr)tqD(UHdfUgu(uO!?{D8Q57|BivN_N{$+E7miRlX;WsM*L(%rGI=Zc2Dl= z?KRlkoAC!%^28t?H}j>liL|NXIAD=+(dzKjYfQ1bHON>gsm@V~?PR392CwFRjLQ?S z$6rqldzDr2CuU_2@f^58L|U=q`1LN}F*d%uMgkM~GYLty2=k2UN`;-`_n0*}x8lY= zOkx|p8;28<=uxEp9_36}6hvsil6&b(^qEa3m44S(C1CF!}4TOCx z>%A>d8|I1kvenKJ7`^mo$R6hWbARLk7*~!(Pg(YDhVj1@FFR4@ygl~C;k#Q8vK?wGHWJYDQ-)q?XH&uy+ z%J|}g1@`?=Bc#F`54Pa=Bo!vF%}{HB@TIy!co(k6`sXE2^O?0Z=`?|*P&!zn9mvx; zPW(a$=~qpeG3E_jbkqO`y9E4Sz+xhPvB;&w{LT9Xh&ILygJyb03+#Ovl1&nohSCl2EOusduKX4MfxXMD%+A4EA8S2D0 z=Pldrxgo^rI$+masO^Xq1mB6T(XeOmWo7g_?u2^s-@!6;`EPf7A&p_8oCF~E`qw2U z4(;?wcVXD5tX`roZ`NV;_6JXy@}RDL?WuopJbdr=?^rF2cgZ{Pv0=djrVedXI|W3Bi^=Fj)~2cGJomQe?t8-Ck>}8)Mmq*k5#)k*ZCDt7+9G+#MRBAj z#uLDC8@1s1c+z+%`>4}$NIx%T950E0%$il(;`S)cej0P7snxzfUCUmR0nN5;!?`fD z;Iw~F3gd-@Z?W|P9O+>TJ<3{|@Hgu_zY<=tKAG6Hir2N(oC=@#;S{DpS_xulPPNc| zxooX=pU>+e9|%-xTr5>f+IqqIsGc!=Z?UGWk7+q&3yn*ZMAwW>uA5;h27F4R`MPeR zy$;E|%2+*t&RR3ORnCUE>aY(+$C42#NkMq}9=e19!#542vv0x^y=r&oHZF@LIv$|* zeTGQiw6tIPPLEkmFN?HICrrJ0VG%Q|jdQdddes3^w$ZLnu?74Pn$b*pslPEX9E2g9>oe ziWg9ub$Sb-LIYd_0m6slJstHVCz-03OsIS)LbCJ{%@alKoTU|ShA(TEKd_F{GOG1K z^R5Oz_=mT=p;0gq$>hIFk}DhN%+LMayC%c4L<}~^A&~KqK_LG1*d_Q3Y-i=r8cHlp zl}zn=M^vp+Y(kgir}6^H9}i%s;*PAGWzy*gEv31{8?3F+2s$9bxA0k*El^c~elp&W zdiER5GJ*6Zpgy23BsC25rPdczPYhQ|k>M?tq0wg*XI@Vht^VEM?_7TxY2>@^+a)k1 z6$VzB1blfQCN>$z6|5M}dco*z`Uf4>!rb953{|`4eM;CY+ZhM~gtD-t(9W56pJsLU zNDyr621=d-2B+gn7Ex8o$tNI(M?%*d>C>!!Kg8MQjyn4IJG!{yi~Lveks?T@72;4s z0xVE|j6$n5DZa}@Qks$FE(JIC@2aP!XB6v1h|U@-E)Hft&FArJ_z|U3iNSg)>G|Y) z)K3sj`GQ5t5=MsvRBi;%sBHr}M;zm<`mV)ZcEQi_Q}VYW$WP{OBEKP?iFsZ;%o?n* z+1(kp#&<9LaGzI^sIqp!PdpiCb_^5CbR1crGF{Sz|LRRFU0fR453Fx%$)(OROy1M4sY4YdddTW``~|T4#qCp4dHHp6U${?e^Yf>1H>h$0l6Gdh66UOx z52KSjb%erhsp0sWikC+BZ0h6yI#2!Dz78lMRzXr~!L4O62RM1bdBSWfwD)7to5bDH z()KK&zx0b^pKci*CQ0ftWw{i~q#r9jvGF)#CN`Q-Aw-#+}`!EQ_U zmWiCr`PwI>oR`9Q0fJa-5|4$L(I2DW`q{a62E7p>w?5&EkbjIYTuOs$jagphgg|r> zxxc8vN0DiwGmBxGVg;L!A8#0;2L4*`WA(1x7KKU?mjt#i0p=qekvSldT4C%h{*?Jh|(gtQQVznUr@` z0}8=QXJ(D(LzCZ0zPTvR{EF>AWDBlhZHUmkU3!LS1o?H~_1&mi38 z#X2S%3*IqM3pYIo%$vY9u&GYRB~yONn|t)8Ux{B^1Xnsk6@CscXl5_E(&VL|k+);d zBkaV`{=a`L{JthK9QXXL#+l*<*r6`0a8_4ycEiAsEwC+s^tC2L-2j&P9{p$d%RGJE z-{Ilr8VuN+&M=tPp**GezW)6CCcSJQw!gwS0Us0aSi?>&(G28zlZ<* z?_ok%xJ&s2D$Z8y0LLUyZn*{BDfKD1*0iLnX^lXzdfuCPP-m&p>f&^BFxDL*OlEQ( zM^Mo;e;U)%Z57Y&JDh88wC{a-W@Op8`a@W5AWqp2_yP+2qzw?=s4DZMz!D-(c?Y?0 z_pG9Ej>}iQT{iaRfpi%vy?1WiIAo4cw}7oK7)_1~9Oj24RNORS8WE$gZ?5(ksuvAl zEG|nu`}21Hr_N!ujr3a*`}fDqLqeCYK8{H9Xvx}Fj(W!y5BbbXA)ooI^~|$GvD=r$ z0mD_q`GKqVcD{1lBQvE#hby7Okaq#||EpUaz5?^tA*YFOD=M_#sT(|)p#$#0?BHTO}Kk^&99r8Tjp?8be>TcX`oSq)O=rOXulyW&YP?ao>J`VMj|s67{N1;? z??-)mdTlL<0D_;ts*Y+NehSB6DeLlE8kJuGecxn$km&Z~*T}f~KP|HntzNUodh?{s zmdSvU*CVR;?mMl1ACMe+hACa!rfT_$ysJ&FmIs<#M6=kl?k-yQFa4t-$_!o3ye8>g zT1N~zts(b2$!dEKUab%nuiUvLhs#f!)Y%;`*JkGD$)mz{_QSS^d&g&jNobo99KI zD4yS`x6+Y8e(^AaK1$v*GzrN>QhkR07{U6-#dpHO@3?G@kLflsJg>f&|EDLt{jfLT z2d&Jt{`>tq8zD@S&qwypO6H=SUNiotmIX)5x>DQ43KwCPIxg{&2k?{e`;xx~&JA`~ z6o4Z*tz)JMnzlYv4&h|@j#yzb+>aygeC}zc3P3_xDDO<_eqhUqwsY@TC{mG@Bi6IF zG#=;o=7=9?%ZHLvZz8V&pNhMzb;|;%8CRm3UdcBXJ-F6Hh~v%R9pCz7U8A5E2_BNy ze+GA%R2?MYM6O+zecIDjB=)bpg^IeO`kvaUkOiJ-E=3<_$QBRPum@{^%j%!hUhm_` za@ddFek&g{leJ#ye_&f&^SWY&@uc|6*wfTbljE33&pXU(FS?#fJ0VoX`b;3{{$!Os z^;tNFd21{K0~NC<+PnNU*73{SXOz$h`nz$pX%ZYIVYgfJ6;6!q3Lw0jsRkH+;DTN` z_%hiPaXhw}ZEdSb^y7+TC+ZA?LNc1=!ehFwAdv3LHk{T)P#Jb`c@_UK^DOM!U2J+j zbBI2yvvxBIcXcxg(pnbxvh$t>;AQa(8^!joPi>)fNZOsC>45p&PV_m}`3;aoWovqC z)H)gDI>%1?fk^1g`-f2dwGTi0z+3Pv4T&1To!wo0Q=JgEQ(yctbdk{{@Z4jm`D6Ma z$gwj8=3)-lQLl-VtLOk+CT7>yBN0u!qG+V7;Eg7F0flfj_dYlUwXX-*(DF2Ur9N4i zBDm5jrx2=uIBMQ&8dL?Y1NuH6%n`;O#xg+x)H^zYh>vOh+1(UaGWujLzSMM{xJLX| z`)#i!0Zlq9uWI$%EP;jtP|T*_cf_DZA#XCUU|}`_UjtvVgUL6)u4;-*N6-o^69?NQ z{=Sg@djHxn3M%MLDg1+yrg1^i?V{g2siH`vXp+DS$J3GIgfo*Xx0t3>8To{Ky_3&* zj8!Q#NGMh+?U-DFODTMoVISEw z))e#8&z}IaQjp&1f0?)7_24%2rj+C9TY*{oEkcU^#qO@ zF*h$K9yMe;$;2a`d>-ce>4C?uH{%LFh|w_&RBPHRw;YcV)bUa=JFpULc9ziWkN!=0 z6zi!Z1odwk3043?in{~1jXfy=4# z*2$@s)GmoB;_i`{3?HOj4%XBhIv*HPwnK;z{+GIHdG>;RVp;YqmF0Q}aD0@Qg}B2& zf%%xanw{rL2x^hgfMqzMqzK&p*murrm@1(%K%!a@{4HmOxSdd#Dt;$6R_C1I;ERV2 zsp0u;#{ZHUp>N8_`PIIPWe<6xx{Yk0@5d$`apfF8zwXsLufHC(O4&CNIl-L_Plit! z3nH}iKdTJBqc22<=LKbqprejIZ_;&3&0sJ+chjW;V2O!rA$!rba_wlvpJuLoB%^pu zoY&w{%b8v@nCS_rsc+G859YcFMk0c%#F{T&AF{xkTiP{X4{9XM&5p-8aCpvYw&;m) zQe$U-S=@JS zK&}W6UfJ|`y@fv{KdvK+KaUU$KK*Vr(~{U9F}&orgqoQ?*Rx9>fVX~sIX?ptrT5Vm zoESL0hFMub?+3n0{bssIgrsz$OtCk$uPG~JmT2Iy4d5W)Ri5ri17$^a|^X)3Cccm%{qeEDVPBdUC2+ zKR<~--vYQ0damCEw&;92TQ=N>E*HheE8c;lvKVokUq_6f5?E5=V|f<71Pl(lUt9f- z=JMmZ_YxJUCDsWSy>R0@{68(YULLjB`F7L-e$-+FH4`6y{LOc?!>eQEfH^`^_O2-B zllR5rcT6f~KUV_x`VFs{A)>`ZPu-{h-bo*yr!H?XqVuq9XmBJizDD;ew@gf7<`dVd zZ?OfNE-ThyJ}fSR!<6Dzgl!tr=_no1{GHqMVstNcM-Wgi(sOpW;%#pQe--+eiHK@% zT69?{-Io+ZN&9_<}i;EV2K|s%H>1VEs zTn<{Y4|8`GWY*RZ2k?RDuz32|vILzGRNJi}?z z)aazw`4?y!EGT~K02GhgPXEklMnUMc0Y*}1MuASUB-w6ix#w6LneFbUz_{Ds?yow& zB!T6_CZ$ajr!oClk-OK^!$xRUVtoBx^5^G{yH`^mKGM}#ZSl+8MNVg*h+|w!2?5Uh zKfJwpI92WYH(Z89nUgYYkq{XxGHsEe$xz8uHc6&r&TM6#i3l0C$St$XiFPvVWXiNn zvCS&uw!z^$8hb-TFYA3b)KK$JelWVajvQKw?yP&tAeZg z56?g>fs^&HmqrCrJMrQhO(uT=;yL=#e%^Br!*|u>gt>)yP2h8-zFoVO9?A&ZZ{zD7 z>GMS*zIIB`H!&dwr|b>e_S|Mx&As-wkKSoy1Hx>DXDl)bmHwDI!d*d_1r+AE?)R0q zI7)UrKs?Fs!*DPUalO7_dV2T0GVHZyU_a$?RA$9&rtXzp6{oWC3MwYt>FCYk_lbd{ z*>K^rup`G+Bl(DBo0B~X<1yKi#A@BE7bA=Ry|82^XggD3QMGKpntDHQ6o=Zc9Nh=% zX{fcT(Y38zv~Ku5LRp~Kq9^V0F-`T;D-;Fd65Sl0-_S4p;QaW{!Oge7o_Xg63f9q5+tAwYsrRiS9Qb4(};$M!U^60sXspA$m2ED*vvy#8FLnS&E$Vtcvv03R$ z5L=is5niLXeu<4J&s>y+pEJ&8u^CgVS%c@Yx257VdgE)$hiPl7Nw)@XOBLU?I1gtW zFUU>rm~My-)_w)~n~w^u8L#$y`};HU&QzyF@xi7dYD<6E!gRwc-yv#uoTAQ>*Z#^V z%Z?}N*P~Sa%=G9jhmGOLG&Omd?^QS0bM2i}$Ifpw2;3}Hv#o{C-Ph*6Klnv&?$DR~ zjqGOvuU*c`W(Yt!Uuf2M7$j01E$yIV515ka)x^h-1E1+gyDg;Mfx|9It{PD8A%nyf z_@pSGI4Odhgs5sV3lj$#n!lHdtemog z)pe01j^g6^_caTj9XIVX40?a$mae9LC>=EB{GRt1r?{tp) z7R(S>L{-5+=!|5lu>!k!Hm%q(g8<$%^oJVu!GvUq#0rd$(%W|0d@R!cWlA*_>pRY) zpB69^T}Iq{eY36AVb8*PWoPNeDSh#cEn2|X;yT$;mhxQyAj-#nz=Oz%*H@@`?M{j# zUHY;G9Cln~UaQ`RDaP~m;Liuy$h(-2ymr3Wwgb{PuX3idd7^d^f>$ePzVG0`s=uMJK^= z;d)vJp~OSgbAhB@1FtXN<8Q491H!=XGhNY!O6br34Ff(d5-sXaCDzG!)I=cvcNt4yL@-015?Ezv+#n zO9=wR>YxWa9iYmUZIV#8KQD2ugN!idEUnnWzg|4!D+udY9(@GLIT0{`b; zd5a+-PJVCotsFT>^i`NyS>o$_JV}z<=SZAdiW4t3E9vsx%0GP=?GkZu{yK=+K3dcg zsK9t7zl&bP^KiyL9p~0Lo8)j=G{?ovkBJ zlgNwzuyi9UF#SwGZR(lEoCVJDiOT9a~Tpql~+&xLfIAk=20LJVOd3*1CvHRmD(%Th+_|U^kq<;{~0k ze~0bRhMzneoVJ~5=36Qx)X2s=BqKK7UtJ~Cx?NVwBL@fv_=heUpUVQ4xo)aWKny3> zQhd_#YxgqXbCA&vQxtFNOnCfe4Nk{nnFj;gM5{(=j9kpQWJxclwR8t6e!C*I4(bPo zcZU*!d((P|@7x}e+Yv`xil?WXy<$%Bu3$0fy0Hkz5s=p4;F`hI_7wlkmeLp9 zEyH*x-jb&xlNF1`O79erf0Kh7EdN$=vP8OBDkgQwu4q(}>x^^l9}K$FdZM#Oq_O}; zUMk8qwd6$k;W!xikhz?DgC?Gkrke~zMJ)n`2^J@qvFNZKo-8}#f>wwKs$_9hGoK%S zHGJ|lOHu`sn z7@zB*jS>0Tx-n92U~$vXm0|UJjZmE6j3t{u zdeM2`D947!5m!hKF;B6;DeA3KMDIOSV!OjRa{s_Ebv?TaHClzZcosW7L;#A~Gy{pKByyYJS^FC|px4<1x=4Q}Q@=ivMI5?{w&sb81E# ziOBi27_#IupV||br40Ct&BLMb`ff;L9#tOfL>Tl-`Gk)3-5qN^5=3tu9J496pfsi^Ln9)wu$NJb5GV>r)`?v5qARNEX`11!d&LLbxx(Vln% z0&hx36Y<{aBb14H5V22=t_c3bsLTrmtl!@nr1>;vIruT$S|=xA$OcT$XF9;l_qlK@ zLU@TtMh3ARPfW&)+KAxromUeL$3)NL@O6`bI;aXC>(VuEFO;g`P+0Wi@Wm7lQJ!Rs z!86E|V~(Er@9Qpng7|_@24IsIb_Sdks-{R_A2zx5(#{jLsdmS(&UU-=aV@D?PH@3 zU;puZH+isXBG-)LbX0Yhv$fAwU?d7&z)$d^o@3Oadrf%N0_hm?H{~i?8e@4b@_h2e zAUJD@*+Dz1Q2ogmy)J78bWF7I*@qI{#k~H>y zy^~nWz$0wEuYk2=Cpd!O4uw zYJu`Jbfe%~e4>eGew3UQSDlOLcBRC*Udi_rmT(dShe_mE`2AjkLx{VCF~T^N^G-We zK1^XtA$uS6o_#&|>!pgN+*(T-yN5rtL2uf}FAI%;Iqn!W7>^t~BGO>Sq z8Q+we_q{7STdgQ=QQpIoChc{LAZhvm95!Q!vUF-uBy!H!)bs-gi%V54Zsx9^2v*Q)qgO9;k7#@U{zRcy4RcdOWsCyRhTNQu0 z{M=>uBzr>^}_U44rqd_GH z2m0NhJt6X+q<3ESwn>jczF5OP%;PBXc*uKd63xNL(h`auoP>^?j1o!vo`;~9I>Z+X z@@L98(*=K7z`c5401c&N_v?NJ|8-}iAiL!UkW==?E_|*>&cB7-`Y}3KN=|MK_+0aM zXdPGn`I=gg9Suh_vTT}X!^;?DG%AsLVrOHt0gCY&mNG`s*OIeAw(_0KJ#skwgm=(o z?zZ+(S1p*YRmz#G zP&8``RX+z{&*YR>uFCr-0ju1~3nl{n$OT*I*a_T{VLTW5QBt!#-7?|3)G?Vg`B>er z$DP8B7fepXo9`iEx##P-v)4>t@s%|~mDewQ-@`m&J!AJ=n>UyzrVyS2e%9c+`<7|q z(ma~PL=9UQtraV`TIEjGff5ZaK!FG+5MYdXY~rt$=q|JTdI#qSp)GBk&%;! zZ*N~0iVxqFw5|D96nZ`Ro#E)?sIuyd^vOlz(v^(7BC>A;eLz|Kw&`xhbLh<%qCA%Z zxPoLovtI*MMo^@B&BZo(YfSmHB#sx&kq`>1Aa67y7nHelE519!A*yTxaxD>Artbj3 z;usDGlJjfTEF-dXxkH+By<-*1>dj(B`z5jOLjf-3=khTg;@Uc=ey3joBb_dAGK!j( zzDR8R?kEF8YXN1^NH^%P8n*9@DMjxVusB zcFhMQ^eq8?%1wW!)ZLQS6_4M8iWi*r$sY=dfA-MQzs5|}JH@kJpc$t+jTx4mrazBN zKNsfsTwonq_C$5`a~~oiHZ`k*$Y0Tox^)TCW`(hENuhSir)t6=f#s+wc>l-l0aRQV zzK@y(B=}k(CXQJcKKo6FM^W9kFsJ!X2xpIj*}uQ7QXO7N*Z#r!)a#u|e)Sn@Xg4pj zKbq9>jHoIO6!#5&)IchzfLA9}*$4XJZZT|U+9MYH)31Z?b)+qp$5G60P{4J&aA7GD z`Jfy>!>PA-8*SNB)8yMn^e?;A6;l*v+-xC1k`%wxTU$^ff`b7r9#BE<-Koij9>@!Y zpRIj#>8_cSW)orm)G8`C+F-nK;z4rnQd^?9ycN9(YptH#AF;;?qx=>M&p6vme>prJ zMRsfEkjtxA2l-+w_m45U3JxgaRKFkR}nCdxpX>F@d zVNKp?h|YSNY^-YL_bLAabT};8T-x}YGhgVia4`<8(@&{e;@|buTslW;{zNFdGy~Qro+;ef8_s#B`y)&x06N>%Z43~3r{5R zR{W`lC*e;^MU>Q+Vf;(}8g6g7xw}s{VKm>$eB&6UV>a9>+jfP6s*%+cSsZHXkI3&` zmPl$K?cmjHIF}(@oiVImrl{_3|npi=rs+6oCh7_Q978XSi%OHJ+-_P^o z3_A5nGTk#pWo9GjoZ9Z*rEBM$cPacFtDf}5YN7sw6RrX|v3C7y)(@BK3tR%8dJ8eN z14`Ef|CW0tzPOt-YLJzK?x(#mcQ%?MCjpK+F9&IBRs5%^_%9t4UD;n}BKUsGFJB7G ze58%DzTubgg!gY9$Z{e~)F|q)frn^Pi&RZl%@gV$2b%h`??NkDsOufa+wG!ukY8%8=)l^1rMuGQQF+czm{gf-PJ=c!WkEH?ve9x=5L=s;@SzihCN&9PsHb*K*yxQ z;obnR)S|zpM0@`R-kAUXn2$On^oxw&m*4GrXVowXp{cpJXMwL8Kla?V4r(V-srf9? zkY`A9W6sRGhE*gopb1SYj~b!;{+zKwdGr-N`^{CFUcs>^;Hp-Wn|t(bi!`VSF?*#9 zCunmrm^xQ)Y>Q}sU6&RD+dv($PM|G(L0&NLCOswLYY*`l>!LfDdhG8HxH3`oasv)O zFPp)8C!TtvPN3N=LYu2Zr#7VsyqE=qF&Hssbe5Z_ABGmLYJFOHJ9yvrW%Ax*-J*^9 z8v@F27Sa<6KBVu2d(|{w%oQ`+70X*uiy-ss?uH+1afq3AFjEv>;Z&=X;9dL3Kr*r` zDUP@1fhi4@S{i?ClyXxc+*(!7>X+;*R##$m0nBiA(FE5mw=kIx?!;|u0B|+t4rfuR z#B`$vbr7Ev@41=@=t$5ya6Q!{V^e-dqPVk{qI)s?iYogNR`KFj)xH{%f^XpMJ`kp( z`?D$dO{y>Nc=CB@76Nf1`y`we)uwXhmwx5L)fPAJhVsVeShy(<3EQ+0G%N1tHq99- zwG(Ew9ON+gvU>G8B)7VofeHSl>sW=v>{?eo4E%(X``qm29yDz@Ez#tLq_}drZ>)W; zaszfstb(xWRAG7LRRXxEW26MFhS-;cRmz$L3Fkk8&U-v)zzZVvoDyN_`~_WRzWBexGd`3ybk?qs#-!;BsdU!)t{ zIs2fPdUL9Kk|M6ix-xh1)I}4XG|L!o>BtZNU|B(NfrDd~9qBI`oz(0uV|RGGlOY z{C-S}cp00?Mii~fjc8sSm$UVBS$7N~W>&V*bq$Cn5~t?@;&^);nLcAY*Ng-6waP4x4^UBRHW?4zWD+elvF7UsC4S0<2|KTyqq9pvn z^ljgEBKdjeBz}O?zXgkuC1*(4~yr1sV{=$xiDVtY-%sFw|+CrwppKSMDcImz4I6Hp&JK zqS^?|^&2x=vr~gm@HB17xN)x^&x3SFGyn-?4C!Xg(~%OJJvY#ECZKC~qFD}Ey%|fw zKgz3}qe^nvKt3^E@sUi9wTzVBodLy$%9f{@Ax>&46m_{iqU4{|%iw9?j7JpBo>m?` zDR(Tp2^2a)Z;zn0_I-X~HRSBf!uq@mB=KOcrJ1of(AVM@c9ei81mLG7wHE%=TTbVV zp%j-Cu)?6BJ7&{3SjUw=648IteBr>b`~)2nO?dOqI5vk%efE)`1~T5`zP zZ8Ox&)T{jTHK5nJd)FOY7__D?B1?m=-WhL%C@S+mt2b`6Ld`|FbsaPES5FV3CwwBS zI(M<*NB8m-!ESW23mEm_w#hmMe6%h88_>Q(STUT1mLA6Z1qmY*o};secg!m8DCBSy z-Tx~t{9&){(MWQar1}nX|KF0}Il*5k&lR8e3M{(5s6zvaX>83FFuwS&%7bO+8C~z% zH)GK!u_gr`v@6pjjP@O7Eb3qlB!c{bb8xu5MmN}0c4^`c;?9M!hj}{7wc5Sg5r|;q z^5`ru^jd~e#?`m824FyZ66TD$*iTa&n*Vc614MRZ#a?(eGzF2y8r!}boKSiN%}{46 z2`J&fE%m&UT6<;YZ?_<)X?{$z+q2+B+@@D1?(@!Dx1fgdWIEmzYHUEj1vTlh3EN4K zyZeSXvQ}Rv1%iLGF**2bS`dE}A@^l)%>Ln52u-Q0Y+G0+y=O|L9B<6MNsQ(W;2j_WF| zF@Bg5x9X6*Mlxx-sap#{{os_)K1&$DkxGu$h|s9jimQY4r_);dS4heyvPdE%9PQy& ze4{Fyw~cStk*b6B6344TE+p;~6lYH^5<-1DX;MTlek!Mc&o9n_`t~B9BO!Zp}?M@pT|t<+G)Y=+H!+? zw*2o3Uhk2G?ru!eWaPF7wlQ${)l-)p5K&Zc)TtIhQ-NM6zYw07g5bRWfHKR3Q?LqC zLz@G>Y@Tl3HZPYE>z5LP>2y6k+I6!+^}0zj_L0b+JI}W;cgz{MFd%Vl%j+l>|5j30 zd`{vAJ3NJ7C|r$e7E-S#uy7Ix=lL>I_K_H7{AUcCNUgc-Sk=S!OMPW)Mob;X^n0!l z9(uYF0iCqhqn2n^Hk8cIZy;!)uH$WT>#E>!PA>xSZz_}~pUgr&1});B!ga!&a$FyN zYVd!FdZodK_W|Y;ruH6gx#VmG`nbw}m1o*k^Pg)6s9}=u5<=$;)=n1YgAMIv`~_Ed z28;u4Ol#2Qhr&7Mnopc)(N=aP)IA^bQe={)%tqK{)w$2wHtw z#uf^E(>ldjjwU4jZebh4(~@wZaO=!HOOnY~`sPR@uvgm{ux?+Z6urXEfC+6~6n0Bs zA?L8zwxR;{Z_TV_k^Aw9gv%J3Oy$hJmO=qk+6mNmj{o=(xC^Ri-Qs51;zXOzeF&Lj zoQ~!kV`bP-^Or~d?I6AAl5gq{*|NsHf&2Tq&?F-IT~Xz@xGrTtI&{}A!FT2-?a3wN zo4J(^sPGb-F*y|{0w$SG?J)AD1m8;L6&>@1xtF!8>pa4_#~`K0O6a=zuTa^3>M-)l z5^0NsDFWutFKe869!h65|4ooDUeFoBYc8P2(dBtp7hVvNjgpI3Bjj4D+R`T9xZcdf za!XotA8OtxO9;g6M)%!A|H^X6HQ0Chx$(UZ_syQyw~&10Dk4w4|JRQT`M@Db82RB& zlwa9Xd*ZqTuY`y=p3hAB#}6N~Hy4EIT@|Jx<*x_a;n%CoarWP^LeBTB<=!>kItUcM zF=7#FjG#P5x^ly1jbyzr#C-sErFxiLT{yJ~Gr+f2eDo!Dk}=7cv$2cv+Aj=_A7(WP ztiK{l;0I#zg8po8z&m!;`a-h~Nb{MbGAsBt6&I`(q@i%@G=!gij5Rt=VW>kBxBTBfx!Y1OO{SpA$frnhC7 z@K6YeIc)Iev_ob?zY2({6ox#ef6nt-Q48f=Hbyqz_zF><`MKuE)4t+9Zy&Rwu0y%5 ze|s?}5K>Nv6auOIh%p7kfm@2Rdx8qHF-VOSMC|s0$1VSY&YOL^DMB{`Xft>N0i5-$ z5^<0rv{iSYKWOOZ)4$c34bdN)a&uS+t7umegGW3kVREGkolnBA*+_M4|D)2tekHtU zvD&wYo9NbDrBP#(gQw5nsyxSjUsmWAEIyspYE?&x}|<%XveZ>7^G;DQ2CrdQ*i-XvM$ zAt~!;?%p{=iTf!mTSLE1Q(UHv>7$M0L9}M~7+AimLGh5QMTgDgStc5hc|?wGm}5n2tG5}UH1x2LvBjok#3cmGjoeBNta$fyo$ z;@1KDTlpSz-;@AJh532PUJq@VX>^faK+UK*^LeJRbIxFQguh4&6ccJT{b+M9TBWgH zP;tNVS(LLWnGyM8${IYA6SCHH$?NPt4k)cJaHaV#fwPxc7!Y2exkFCW7~lm;nY7b- zIrJC9a%(c>2P?}k?bjDs^*sV-3)s2)XLL)W8oiJ3i@5uk@=!14KxbK0y@g-HXFmcn zS+1MU$sKZhWJtP`AI;g*G%SW#d#~@^00U#m3#SQI1wpZ-CQB?*wgz6!;>2I5PS95G zU#wD=JI$|wsydn3eYRAKJvr*yyVt?J5L3xhMpLbIwBV zfb=)AY4f_v&Id%_JDoe?M45-T+l7B#4}JT)EDr^6X{{OW*q_m8XgV!)qc8wkrMnZ( zmZsKQ{Epv^sX30n4uv~Mz1q!G6;_~Y4!D*TzriU~8W=(gl7sC(1sAspjXB4S%K!<^ zuyK2l>C1t!#vN|*T@fpR6V%sD6*f@KVKo#t_@fRFX;wz=*Cd>L+x`? z9Jlc~_*%!LP>Qj=#p(vT_9r>iS7P#THgztNvUOVLU1cTSmnuGb zki0yOVAa!kMdqtwC4G@2=#7}EFB+3N-HGzw>R;G-NWpqhh);VDlg`UyUo1l{M$TuS z`m~lw_InTWcb6)*~t;kqdK zgYaVy4N3njL>o~+gFlAXW2FX}|DOmeX45OLZa!nM}g`!_=@z{Hd!@k}Vko7<_#A+Y?J^br%0wzZ6#0Mtb9 zjl>%O_XZxW9@VT43306sHFI?jadq7aFb);Z8>#Kb%!ln>eD0gWeS75Rif^_1hFSGf z>%^uZ|3Cg$v}zQDCI3Z{1Z<|X45x|I@Zh|JPm4Ybq_9L zPPi$YO5wE8+FafO<#wh&g$jU??G3dyDlJ7AvwkK7+x&;FKsSmC!R`mmG)Hpa%}xG~ z7ptc7wn0VPi<+mQi+Gm@Z!|G*P~9T=uB3w239=wqxsDo#)fE9UKa2zbUYJP={cXH3 zIoJ)w?<1H;ug;+~zNM^Pml(Kdy~SdZUryB-cl%CnmiQ9Gwy$v@tnEK(RFP!slO?Iz zL6+vaJKp8%ANs|P_KFuyz6UAx^o?*sw{%r8By(y_p@w-$>~=1@fiV`%+H#oJD=hUu zU<+&ktMjhfw%u#brrucA4+q zmmW%|JXH-;18@CcefJ-gEQ>DCt(>*+?@mfHpz{MfK&(}aze z`3%t(?r0ReH_I<@-B2UFIK&w;VztJ3S8HVA802zM@>D$)A^FQ-pz-77DisbAV!F&G zd$Rk;mcOLU$NXr}7(W=F*?I|V485a0Qa_G)Yr}1yd{b`7tUem~uqNO~#C8EqZw7~W z+7phH_`&IulJmn?Ij`)BP0)`BV!Qf!9PnIVRm0`{Er%Px)y#L-8*SqQ`Mk z+Bg{*-gyugKgiNM`_xMei;mRy_bE8PNE+e3E_Vf#eJ{oW>w95>&x>nekU-LZP~y=_ z<{dB5(>PDTVm5578sW-E-PEKKFQ}m)yMl(`{DN&ixq!Tn=-P-O^ly+0*r6`&ei{5Y z<0YbCh2#i60EBj8e|=ik)SL?=sc*Uu-&$ld;Zxot?`5HU*BkO7zfeg$;2;Rj_&o>s zL(j}@?thCR)wpkY@zbOa1^2?-jr+7b6;Qn)BIKHn`6sz6Yjw$gf6jk*gKXUH&k6WX zYk`Eykl&e!VCJ-69oMEsZ%#H&t8 zH+3W0F8N-xLy4sBj2aA5Q@;O>JU4VNB9{#+5Bf`_bpwTW(^#WLSzxU(h1IOT8WX8k za{Rbm9eDQa>Mej<1xLs<_ZV1_KjKdx%z{2%NH=H?YvJU!cGK}9U9)y8(z^Y9GxpWL zBJ0@5)rmfReS26o!ReP1*oxl0pfgM$W`;5o>tCGv16wk0tExD_nl>wEu8 z=$wo^5K7DbUlB_8AsOnm8IK5kjSn!}Uw@pn*3tKhJpMmY?iHA{Dmcu@Vb|4K7W5yf zw{sU#^Ve9GQz3Wq2@vgX@Y6rMhMd23#;c?Usj1BusL0fv+uO$oxjxV>YU)?tw#~vO zLd#?iz?gO=5|>D)Y`;Nw|IS*)Aizy(Q*hAp0F@Q`SNW$oKsg$XUaj6tATPhqHFle< z8cbP-D5%P|8j&hIxM=<9H(bMgpI}StcmHmRVF4n*F48&bE8u;DfmW@X2xZK9RXaE0 zx+UczLQJG>bZW~^17r`pOq<ZP|xcS<{oxRZga`d>t@+U|U zS56pYet!XbZ!tgbxOYfafLao{2-BT&F0VEpExMd)$XXrs0oU`d7Ipij7jINXm2-`` z*&7_?W>YPg-Ym*(0ZY#(Zgd$2sBuHd%P~_wu03RUvS@t5^VYkZT4C9G!=I5hK4~BB z;PQ?_LpI{Mav)h0)#j>ULxMHi&ustIm0!^!oL+q2@k>qCVdP&~`EtGe<>mC+hf9~L z?+l`j=1x)!Ryn)|ef}IzfOohrC&_ttVEIq=wa3?4)6a?=NXmKgf*KuLoGmJlVNo^r zTwpoYB&2igwOm+Qr@f^e++vW)NE_D0C<(WenoYq1dtdGUYQ_^W1oU+k|6x|;TZLTC zCj&gMmy*qoPC&2IwXZK(-cNs+A$HPE2tJ!VtEA#|KF?E-#91lUR);78=L05=ZzFY@ zBey&5^hXvCKSH=>o=YDMurz&cCNfM$M90)1LW)|c{H+BISR}__MoEyEfGjwJZ-SBf z<)&Ws`!d>rDo<)f@4lnO`QmD(kL4rC9D6V$<7x~9PP{I{(bnrLR9V4tGMW^L+YB5( z1yF}G>-TQ&R&MQPt?iDhV2m$9TeQm}aL8N>{54QCnAw%FdHY~`mRPqFYDP7$E>)Vk zPgHoYdw%gUs^hXRR1v%^Q+T}|v#Oc7k8rCC6fx12aY*>KPt|v>y7tz*%i|K%iz;om z9%;LV%suH{=f@t4Iv4x1*9Hf}K#pwyZh@A7iRC=E9y+bH0xa-)w@r{FEUFEHN1Uqw zqfGBz88-U~pI0|Z%B|m+42%BzpJDv_bkJv{=ALp=@hFj>L2?e^I&Q)Ip0`$WTDcoj zWOe#%SaQw1sq!RvI=m_Lw1(gFuoy zE$qwuB0^+oan;k~pBO0ZATdq>rK84nY5D^|_ptL4W;&_;H-!fhHwsOvrL-xN2$+ z2|cG>s?J6p+7?F+!?3g0^v?8DuoG1eHQI=vn8XkGaNw5U)^X?0MS)LEN_Ag#{VXTT zQ(|%XyA?PX^v;E_p~pGXfccBFP*=rZkd5nLfYG!TF?&M>TtDp=1O$&jv-=ysvCo=^ zkc`l8baHv=v5(8x^W&F|+ndJm^i3SNjG9`ZZYRNy9;Ml3=Gd`R%w&IuNQvVd1*8oU z-o~spZ1kdLr!KgkUUAxAl7ty!g<0x$-R_A0q@{N5Eg^@D;Y&&v@{AiTV5!v3~A5FD9WN#b+0PRsM(zJIZcnRln$= zx-v9Ut2KDx&i8+vF0r;SWOK6R5k2g#1z_;_0KC&HIB<2!y9s$42nhBoR5s-%?EL&n z9Tc&ANr1^E<1I`Zv?!efMM8fYLn!hE@@fF7V*--c-%;|e`G#M?g?R?exLODOPMBW7FH%$?mH90k zazW5aJQ|`IS88rJyLE8hAN40Q6wBQw4*G9{0UH&%=XRvl`_Q7JM@QD6pn01rzlRlr z$fo`g^hv~@A8n)W1==WLb<2!(H#Oy;F|UTQsQRkHXG=|`6` zVH;H5ic@9K3XU!S~3 zl$y3_aZ0vV;ihzb2gc>q;I%D9L!VI5R6&3$DxkoRBW@eMh;o=^{R0VBg@D4 z1ca>efr0>b6uT@`Bqk$E)7akDWxl&$>AgTo0sjbJ zS-T}QtcI|aQMm$nm0gtK^y%+9_8DSVI4MCxW>!>ph#h|!D#C4srZmE(Wk)`I^ryw3 zXYrvM9aceOp#^2o)ZIU=`d!g#kv2UD=Ugo8g@+aS0n%fYxFx=#&Q?U(3Q839Wwq# zqiX$_a;Lv`DQjng;3pNQV9J>`-gqfadAlT-{C@hdH5;g|u!LBM$mQXp|Ba;C? zis{0*Zm5J7xyiR_2H90Cx+QQj*WdKGAhe}M(mtG$+`;#2P_dOBVmcq}8>}5+@Du+B ztoz3e60TZR0eJVlV4bMIjtwa+a@r=GGekqs8^vIf%wSqAK5m#O{jlr<>lstQrRD-=re!CXDGlyQO2X zog~fyj@JXK8>twULv*FMG>+bq5BNv9o+mQC;}(R6VK3g@i!_lhq@^@pRaHCP~~CPqbbr` zWs;Do$IDtVTz1d2beA7q+iG4;>MF@>DyoBQM|E7BEE39#Wkg$7BQKFoK^SkrO- z?xDv%XMq3Up2fNU#4mMUY}^a$^ikH>v;B}7cpHIey|`FQP`6c&$fAY zH{iB3n3rFW{9G)aqE$49h#pL#QD#y4F2vU7!9FcGXAKh3Qq6@__~b9RKNH+pyubGG zgAolEDyxH*`o|q>8oB`oi^+Zqx@;=08}@{^kxp2dO1R?gO~xf&=x2u&Kvo7@o7V86 zB7fv@GLG--R?Gs$i0`+EuDLQTK>eh1b!e)~gJQGN!aEC?&p^^5*>4;*A$kcc-*=UK z8ja@+)oX~q0aumcX@8UnSDY4K1cIO4)C}maBZO%P+d#SvAgMv5!h%x!+9OTHP^{cl zcWAe|N$50g=FHJ{7Bc=)ef^vDF_ou5_xk=B$p$w zSNG!Io_?CCr5s-Fvj5%yE}#z|oI>pc$D>g`Z)MG&(CGCsbkiU$M5h2m42SH-p--V3 zh+P)4`Ug)33qHoJj!;P1`T3?h{7nd?{bg;74)e4gOjx|-4gM6Psy;gG^W0tCLlZj- z=E_MwPhy426QOy3Xm#SwwdTmWs>GbRovOd*%ryeva{_5+u(4>nM_LKJpK0|mE0)LA zCX2d$4-6VziC&eLO#~F2uh+#n_Qso86OG*$^4~!Gt1l-YwqsQM_Yy-6D&i{Z?++jo za_1sJ?Dzo>EwprR4}JEw*Ed2*jR%cYf;*uQ_7^HxGAOUJ-h;TB_F<(Nv)Q+l;ge4c z@^)1UShZh7?Phx1qG`8N)D=!A>)J+Zdd)(CkW9ExE2#q^^y6*$iI_I%oDvSeV*b9I zN4)ELN9>Jv(zt~K)-V$u+ka7?Zx8=DUkUx5j*$ilR@B`p+%PEl1o0G7*Oy!>k=vQm zQ$}Wp_ud6(n8s@8tt8pSlp}A=X8n=-XG-$R(Z{DSWTIMtL_Wqu0n~FQrX&-_tUPy&tH$FpPD3}&o^4$MK#*54t&*mw_@vns}<&4lz*yb z!Ft>@v8*%0vElenhVCflmCi!llZzU~TzO*m z6}A3x^W*>8t-2A9S(UiidCCkMZdsf*c=N9c2urap{IP2v8@Xr<`=>l<{r(5GD%3pR zED(L($MLU2xX z2uyHC211BmQ&6~+M*rDIv!I^?8S$Y3MlD*_`mYr!1`mNM9ZKb|xcZ9pw<8;>&mfwRIdvy;?uhvvPuB#G_~z)iaXhQiYu=cOp?=dq^Ev@uN3K1PZ`(j zr07ajAYE;I!Vl$a=f2uP8f50z!He`=yMxdA-seYXV!^-&ZoY&983J-RH4Q!o zikJR8)g0~Qs03`S!NPbiTaN*npv45FbN)c*)0H?ce9l}cbwSI6o)z?h)Jd3|{x1sqlC8cs^+&8P?2eJX$+);095e71s5`S*l( z_2{JN3JZwkWHH6?yb5E6i4_p0`_`U*+sFd=l<$MFJZj8map2Li1P(k}JXgoU`0{aR zG43>7m_{$zwWqs;h$Hx`5&1`t8ATkr8vjuzddL`OX2_rWG z7o8$jf0|ssKQHEW>!9}18>2^k*Wk;0splhb=&)q-0FV4@-vg88!4YY=eug(0Q<^>t z$ciT}R*Ok3u;gJ0e*auE=Q+^r6_&{MD3XnMuGu{)$v$6!xfzumc3^$WTtCH;x#h}$etphxHha?fAGWo@G&^MzIH?&-6UHIj{O{OQVCrlv02YR1WGKWJbj(fp`Oj;7 z{69cO{5R~(_S}rpr`7GL&{;fVaL-56r*X-}!7 z6@+IE#LVUj`49ONNRVI%IOwgQxI)51ZLE zgnK98Kt+9se5s&bz^WCeYYIo3lc&T{CrQ+O3DfTN7DwP#^Hx=XFjnE4xU zoltU*_Dag+%H}gsklrI<&G{~P zrbFnHS>o<Vr^lsViccGs?UWn8JSugm*L!4!gMM-4+NWEL7~~gvUcR`; zFrGoJCL{7p(i?LXE$hYVbgT;`AW(*wyq)$D%rG|=i+J5L7S?ql@R&h5T^Myyuw74y z?>^xZn8L+)y9}{%o?{ zYJ(@@%M3+jXljDcwHF;HW;Q-w_LpC>)Zp-z0|CA`4AcIfH{5gd`VE*fo^v80k7_)p zS3SSek|X;sA#Ppfx{q8+RBh4enZ;6D%uBt_1`6Qx-3eoYEn#x>c=`A&+`cEi@>k7g z0{6Va041s^fS7y>Q4en^j0v#Qe>5FO9h1=G4)r`<*dtVRGFcrsD2*4MZ@)PWjn9Y= z%Y?Pr)du7yqfM8V@8q#0H3Yh8kX`75Tf-L(p0xTI1n-AK!X z$A=FQah!?k7qhO6(CN&aaTDPDG-gV8xp}1$q6tc)PBWHrh9^eBj zttx}n$LzNk)AA>dpT_M;-DoayxAu^1ciOdSOJ*$xC+5q!+m}>vk)Sb;1Y2$7l<2;! z>>r-_{9gR<=-x6FJ7k}QRlvs&!7p(QHgwaCWMoB3fx=xSg73=Y%43i*%5YE?o_+Ad z`Q-s?%?cP6hZo6x_x&E`7Zj}0{yt44wT23@nqazS^vaoFwF`Gl-QW`dNlL#>9O%D% z`V>B=h1Z1s`L1#a+?Ni$69}gGE^co zRN=NO3*T{g66(&5M6)}?$U3w8AoG1zQE30ehfy@X;BFha{P4@X^Q6QE@~H)PZYL~h#>mHOa*y0Ky|n)* zCdZBhZx#nBaVvaZO*mFS;=VPCOhJrx1zOSfPQ#B|pAh0BmLw7$u%1qaT&>Jb-( zkTLQxrXd#jL#E$~M3 zmp53ZiP(d$m&E&`AY(oyf&(Ue*oElM-FkBh8uEszN3^R6gEcP{U`xk;jiPk7xQ}lK zh9(&m1pibK+POGnSb*5!Mcv!$cX=9j zbxObX*@CUc^>9BLj0kvA2?h@u*Tr=lR-tW z%Uvgqw)>(3#HcvJW84h4(N~*BndI6VjHY%*M2N&7-jieJD>lz>h~baEHnA6lQNASG}^!b^3YFUma=2{O<8;dkEf93=E-dUqx9Mmv$VPssSnGe-IN;#E1i1bPG8$~OO zce?(Td9_2Xb2dRuU-6YIegkTZne~=PNFjJ}1H>P$8|}WIeK^8HBtIiM6vh~H+Sgxc zFG?F*_$zuogksItiMZ`Er>V3x62rYmzv zyS<%=LfbvR4pVP%tY#xrD2ai?YI23KOED1+DUu0RZ&d?dRXSH8;4 z-!eA~HPMiS2UxrWh`fo7Rt$yrq}UuGy$4ZzL_$@odY6&JADTQCE@y63(Oo)3Y+syu zuNM@y8RxPDvjq~l=+)ByK>pgUxa=SOh) z_na+}y^-nWO3*(El{v<0EUt9P6Q-u#Bg#nJ!8WPn4OooL$WYx%@Y;wpn0E@-y+5iP zylXXTo$C*>Ws20m74~?d=40{;ODMd~TS5hs0HI_pLUsh@7;{Yg^5XX*DBAYQ`5-&> zS6zpQ**9__CN~v*8@M1Ys5w$e9lndg4_xL*!W?uY*Ncb_3yhuCDx}T+(>e231J6qF zm~Pc-Z7YoO-ymCp5LF_crwm|av;(z_s z)15sN2^}9G>s=2|<17so&Q^O3W^#lffsom%bI~7eo`$2zDL{?xjTjCKwtwU^<;s$X z73jIxFn0u;X-f}=P6$s_AK+D$LhRslrVbfy`o|jjK&0tj9&G^LOhRhXJbB>h!ZI#g z@6ttDEMZNOZk5n(wC3fYdb~&F;B;ljIG_6V6ARw~wcbLia4wabTlCi2=I&wo%RUSa z=%vv`_Zt`t3?80q?zuMBp|Y1Knp+PezoO>172w4L6Nm9|+GeP6N2FNRFag~e3-eHw zLE+K}ud9Du@(vyta^TW_^`L1-1XhEq*1egB8w?)d4YrFFo)ezhZ6S6q*-trO4TNlf zw;kd2Ahzwt=rq0-ifo5bI%I2*-4NvP>jz7SH+Bk&Y1L)SQKWwTMs1d89gyT-+%K(a zbGfL|qVFp{aK-TMqnag}kJTI+S60spZ|=)BN363y#r=Wh?nge)4bP9V_?#k8HF&#U z%0Nvn^J)Gw9*~~sp*ozqIhB0Eee@8Oa_STu&2)yrYi8cdvzH5?j{0%==MP7=LQ?#) zMg<%N|D~L6A=E@p17d28<1O@d_8}_(O`YTa6}V$PHbilO&VLXjexL%tNP|tKc(A)n&J^{r{?%f^AEO`vgj80ekiu?qBXg-{+oHh|&D#h2*Z*=* zU0b|xOUbCYy9u}ClizUDuVFhRaJjA^Dfe~Y)dPI1&|YGi<%wyEAU%WODN|-y%`pQO zS7&gNUWsPHBp`nJ_3tQ@4C7EgGJWtuLK%0m1=%?moaAm&>n2+regs+KLV6qe;Izy} zK1}z32_HW{!Ch%-D;G@hwz6_^ajq#Ww=Ur{qD9s!R47Dg?eSmR?>+c!1C7wtD9pe@ z;0VC0{v5XZ1(}!s8ajdy@AV9LIyoU3>vwc~q!6Q20r@67%)+@DY0a=xnldA+Fg@k_1D9oai$C{?<&$8D>X6Aww45}9 zF-+yqE+(OeKVWyW3Vv_)U$bm|b(_RoyFGz9Z9=-zZB*Hsid;rgu`-BICEl z9_$Dol1(?2GPHVo{M?N4nxBrIZbOmDfRqC>mQCr)B=?87FM+MBwL~o@^}yAbE#wXF zn&Yhl#Q4eb!O@8sR+56y4p#6?LB68I%ri2tey-7`Kya^G-L<<9%U*rRR(^irqKjK@ zY2=|tr-BH}S&Jhs^Gj8@eF6MWe-{PBQz^;eJ(15Lp*uzq*?>j^Slb>;`$gwVjK+c- zOviE{F#D=Gz0mK_>@xfD45I`~os9N*dFF!oJou9jM7-%QkeV4;;EYbcVpZm=unx=$ zApNVN{UW_Tv?C&&90lJT@?qK{x=vhxGXJU`xiWZzWV#Ep7IxyWkua(-=+7vDSeU9k z1TQzdary*_oIiy4TP}VCeN|{fK^SB;is&>$LrbR^UJ+!}Q7>D!Ek({;{ShM48A`=W zek!zV_z_VT4qmz{AMDw6*yh0jS@gYXXd;o$`i3DbO0Wb-DL zl8km9r5McBNT+HK3Q6Yi62I;trf%&qB)eiFW10`c5n;t3+AOB)4gAHzQCNTQnaE0e zo8kYkO&vjB(Wr<@^S}Sc!f}&pL$JD~OvRu7DA#0RzN?11B{|&KK0<&V%v18=GW= zj5_eHWchN$F)`4bTQD$p4uGMn(PMj$?%TZjA_3mL=lfg^Hapk>CTu9 z>!c{y8;#wHOB2EUFLrErCtjSNlJ|_6b4a+ov&h+;H*8}6fy-L_h0DV0mpc~eI@in7 zi)(+er`U3YqTBrx7@(|Jal@)dIk#gfM~3kq($)Jf(sgB0uENx!#oKxB_UH{Qr;~J8 zHf8G0|2zHg{yaL2

O>)MF_K~!1hc`RXy?GlpD7CObKo>Ef(|3bI+brQDtj=mZ^ zTvzm#kicHX;je&-J!qj`V}_P)s=X~atVn{Ug56v5bfTwjYMsX-N_w_EFkP%J$){(` z&dg9Onh6&DiHXluLYMx@%-}#jw&6@b;9o&u54u6^t|^SKeViejMlw}Zxc=_iW&BrY zZGHke+alC+TM~TVdZP%t!^?xHt!fTk^wcalh1rrgSly|)pAbK^{q z7Ui4xZBMoM4<4I+`|UC-!`+-8Ri_MY`}JCg?z2-LsBr!x?BB=Gzy{jOxbb9-z!RO+ z^%8M3a!YodUe&Hk=g!6p<*HkDn3kO<4Rgm6S>bUn9U%dAs(2k)RNY&JIlC&zKv76B z{3i3Eei`B{%bL-Vgrn>h0rPcaI`TL9^baY#LXN}6tE0u*9e4RtF#vC5m!;P3ACRt-1g3JR2R%Lnmh!zq}QQs zUZl3CB7YX~X3eelB?+|OQF8-x0(n)T{8x?oUXXPR#rdAO6|>;6)c!CZAw$EpP}qi| z5bQqH5PeB{KA<;(+Ge=v+Dl# zS&b<*PS_S4M$&u)MbvYm{6(-ggj9~Br;yx2i~&adcod|4mY9G{KLr@xC9{7{u=+(J z2y@HXOJpk>;v-z>D=uo<6WCwUqzz^`!5~1klolsmKUu=ppC!j+pwO|!jg#ktt)uQw z;=JZr`;uWMXm3-=e6G`8B3@kLTSfbxmI#%)ja#dZ}q*IJ-^AfQ4?KS61$U&Zt9Kwa-OMa`5I2&Eck#TT_2wMFL4%RjWeNeZafiq zBq#^t+AfUnta^M+kYmwYlag7z80+EPz9aH*wWFhi3(#TXB0lXYc=9l$=OO;y-0bo9 zC~1MVByqvGd82|PW2PruKqzYhT9f7`-1)L0eq0{3j+mRkCNnBVr_W;;=-N@5VbRBX z1Ec8sNv%7P(vQ<=JQ&y>b!-7o92ZPSrVApGP&ACRUmHcA(r>pA8UFsz$EZqyHmxw4 zw(Ycb`XI=+`g(%M;q^gb+TtXAYl~$sqn3$;Dptqqb|~fFYtaKs!x->GoQc_dfW9x) z#0BQ+)yGSy*L(QOQA9zv;)Ezp|I~SmQi4rFz@g-!E`-+Fg)V4o0S8B3g)_7@^KkJQ!KVW~LjJp|!3_7f7jh?<`YW?HV>#gsVQfOvI zhss9py3e6}Yh$s5`?BVHQKngP!sW~NLHE1R(ni%$ozT>}GS41r3wMN{Cy-YFSjoDr zTbSpLCx~bJuo6``f*^{v2)K1_uLto7K6c>%Z`^Da1lr<7Sb7ETcWX{S8JFJ#BwzKmQCyO=OyUXKIHJMeK1q{@W7<7IJzZS%H8OVzpvx@URDXY#mLS6 zf^msZ502)~Wl;kWtB;oxvmDN0kd|x^bE$FJ?dc_+O13^NTMNrE$j}I)K-#+!#3afP z;wXMiVFaA5y%CW8kTMhw$py&IunNCsy8g{+$O@p0f!9jG!@7lp0|mTrkh3DK)#df% z(~_s=`L&89iqRRdNuijcz8#TMC5EP*+&13>w{sI!_T=4!Lxf?w!ZQ1LIfR2#VrJ!` zLTi7*o;LU1m#iJaoof%hKp!CNMuCpfP!ukzibpPuuZM zOFbs^=s#|Gd&6qfjP`{Cw3HYdFLNDXM??ji>}trw>9nvAMU#MF=Vm;2it^Zh@yUzj z(N;sRy2sl`NM^g@`yJdrh%7guD(d9(N!&sA`TiPZaG-6!|O2< zgoC!}m)AoFXOG3rAJv-Vq#F-2ZXlxAc}mfLp6pI&hd7L)L^Yuye#g*2SQ`G$s0Mng zDjOZ%d~?UkBwFkmq9AKDiRjkY@mZ=pqx>r}0{5!>R!yVLmP#-8dW>GZGW_v1UJ57= zjhClYWE+#%>nXa-oW+O*K1Y*XZMz2jQIZ1^u3=t)`S7X$WP0&>Qhc;iP!#jqD?%(` zujRT<7*)cG=L9Ejap;`+^aJ8g82EDj8pp}R_ItChkW%&$z*h!b>%c^AQDumhf4SDF zW`bap{Ql=x38D>!I}#qtZk@bOA2m^U8HE~Us9rY=wz`h^xcreiamUE~pPE!37A%}n zS--mkhY#TBHm}3RBUv$u{w;?_>nR-SG*u#Sz^A4arExKQzwo|zVX?gYFfLKWL7^e+ zGga;6{lhBoLiBN{(-P~d&!<%S1oRS`ZI;k(G6f`&;-9jyn{flL63=MPnY<}>efQ3f zkKL>uqKIRN+erbMl4dq6s#UcF*Dxj{JZMk>!q7pr2Za;IC>}h<0!q|0&LXBMarCgY zWql(QFqTPnXIy{0R%(wT6b;)D~k+BEjb z@(|#MubE=5MfMZOO;2dXjrx{S6Z;bqJ5DAb*1Fq`aQ-BeI@nI?8I#eW?%0jVF4%WT zu$6c$Oug&_F=xuL+;?H$fLa3u(mvnCT|4Z=I-#!2*&oHAMp#q z*t=6U?qhUk_F`80fzig8&%%3!l&<&?`4sGJUVa%Ug?R~i^lDRKH}fhC&*i+~0&!Y0 z1?pUMA+XN~rk0@>HFV0*u!N)HKmnF;RKpSudpekl>y2l(G#^qm{mT9(Z19Adi;5do zjM$_1Z`19Lg`o627mtgiBlz`ukt<=n4KF5C z=!CU^nPtB!;1Y-nNEt1#0lZRG&rNY3L8+JgReT%-nyZSnm?45P+$Wl8$6LQ)HpVm@ zXQ+i14#YM8pE{y(F%CkadCNE>&Pp8oZJ?bVg2My%r+HWbFN- zbMKe*C%>k3t9h5r5N{>0)h`3Iiz~nJ?D@NN6#`7Jdejzck3kes2QET=c|wEcYhG$q zQ?ysXbIHuNxqwFIQ$xx2fqssUCOt*k7f-`9l9L)1b!=|S;5RZY|5nS&(h5-JI?aiE z(Snmof~`V&Bgp?v+XX+|EI}o3ee61+o=cK;f1~vNSzr6}mZ#71c9|b6j1{ch%=q<%^%FECrNLl;2)5En| zc1nw1=*q}B^(lep`P+H^jw=@XH||uN)e#jft}P7CC zPj=-Fm`XHTptAIUZ%{1fRTEB1mEoa*rC8eQyk)!28*q3g6#!GvkIp%jp#{lkFNQOb zt)>~gIWz&T7VLD8Ih& zqUoQ^nLICftC&u_dq8UKUH*BTK|9;#MXR8o=$oXQDKiZPqJ;%1>=l?B%j-UU1c9k; zg_fr~{NS_Er_FS;tF0{ZvfjZYG=EGKl8I6tG-OP9sDOLSEYDQ+y1=e3Z9f}eNLu3b%Dm-Li&ou zD*jB%diFBi(@tu{s|{(Znezx1VBUqD&7(w%B5U z=`%IKZo3-8IrM#G0NZta&KvV*Y)@Q zbi@@Z>ICd2d_2y46Akn#82|SIEdF1i61W1QASqoD^oSV)DzS*CP(m*Pq^a z7d~1gE@SbhlfFf?{TZPSKw4h<&d_d8IJ_Ow_WI8F(a_jk>%8B6;mIXG>L`9adg6ki zc;U5}sJq`+muRb+U>@(hM-9_WWD_Xm(A!eNir5yk>~*6Kkb`cOt}-}WPQ1xx@P1zb zG<2^m1?c(?WaEe0Cy9E-@FWfZ4}vajdBMC$a5p@&t7hS?6Q zehL8OXfaL! z<(~Fw1KWIwc~0|A6@|0#>|3=rcZ?1dZV36_I3(ZkqLyr)z?`$XZ3B$P@lEuBn2SwL6vT~8`(YNR}8iK_B02mhQ7 zWFPbwK~ucM&Cs`R=og9j?}DM>{*?>R6-}%6qd_q5LZ*eJ^=Tf79%2z znv-qmMY=iiP3p-$`&1#uQ0sJ8=B^AxB!89@7{A4tTdI^3KPE{Q|L36L?*xuOCH)wW zst)@YtI6#<53dc6tvC9~*qGPO6__j+Bie5RyUN&cXyLlR4qDVMLK24*pHh0VyH8hv zbcf8STYxdN*(;-KB9ox^w^TwmcoSE?13AsjLEfLBKmio>o9psdw6hT!Z7deEH;2aU z`mA_DN_QTQKC=8Y@L?@i)?~#~@kCT<8S5_FJ8kT!dwiB+C-Jiw?s6-;y#Rso63PjzewX)*Ruu_JEycV=3B+f|`(Z#d?&7vyaVNt(C zzlnlIC!9L5z#M8_sx=@e*x)4f<(EXY6aQo2sQ{`egStSw^O1kfuiqbAv+jQPw%(>y zwmvKT%(B~Y3jJxziA(C@ONZ}&9!l;lGyGP+52`hPWC9yoPH=RNxyR(TQ4#Bj&gy5b zk`o+m7j$mgIA`_fK^<~!XU_SwyoDRNNeP}2jfl?>~^5Ue_ zxF+E%_T5eml&Ta9D@Id><1sT~SGn%Q9Txwc z6U~)X#NaZEYJX8>ExZ7Qa<-nwLSb_YSYZB|8+Fv3R9C)dx=FeDENvWlVUP7@_4UY4x~xIn z4qObc+Z~%Fz6N7&3&<2_A_aQX3*|FW<&_YEgj~2 zMT6drf|c8pOEcAn@DZ+S)?GU{!UWb!C>qNBwgRZvv04OD*azh&_@xdp}Yh z;e~E*t9N@oCzI@w1)B-@275t0iddz4=uw{zc0(P|=j-r~(h7dobTtI;wLPbXLM#BG zW}eo&h|@<6+=JCWMyoFMxf42;Qi~f63vRelj=Has(|_^_nnRSSR6r|u&jh}Ty#PoZ z!*j+4$8sH5!G?;^w(T)p2ug;|&}wf0>zVIFZNKE~!9c}HC#Z2v4+}=3led78skcMc z4c4mH^se_B$$g)$PPt%fpqNtS#tkButGh=-|0G2f*zL z1gnUP+R3P^4n#w}ir(iDo^2Pcj`Fj$u`Cs@2km|_8SC8rfq|TV!f#U^+OJV%4XYmd zNxo8nPUgASgzus#*Dkp(6WQHpvfw+wFgsNoYQT0Iq154+`k#%VMuTdZ-karz>6jPT;GJHSC3M_`Cyl}^-z#WYa&laQejL{A^up=#wI z45i9F@^~Rnkr)cKUf)Cq(%85d6t=KBHGWGxMDTehaeRmGnVhGqxV-Q8o_-)qD}h_R z*HLaF;nqRM;rE02cTl{FOjW>jAiZP-D}JDcLGHhYkN6epQcK~9BIcWW{1%1wn{%Nb zU!J%$Qtxm743n=u1HXGX@#?ydcz%cHn(k81#@teVviWb53C*CwkAmGtpS7tCmIcf` zu4OS?b$5^6v|AM%9qUW{F^t?Zf|H?2-@|}y0X7jZgz;^wo1+uLLK3H!&nBAH)uPgw zIzZIWezn{}_T6Y*Keb8b8aDR)V3 zbiThe#8xOIIff)=>L(H9tfOj@_Y0|zpL7)g&)mR5Gwog5yB=d)RE3>jF=CLXHe5Ph zI+o^MI!K%`lF-j8KHhK>Bvwh}BJkv}Z7=lJ>3yYWv;eqX#NlJ`Qf&MX z^^1nuFyLcY9}i=EH+TRb*cUI1$K!;cCqZ~5zUIbf8n5h3<~FT4k;{+7tZWCIIUj45 zr=zO2=5G_Ulp#UGAg)sK=+fAgM-+R@c{2}AZt$~Cll;U-YduKVesYbAX2%QFoBJN# z@*?i&LU36%1>~gNMRZbPdC1j31zQzSNF4e;d)XIMtX-DMLq7%|scKf*=_vt??dO^D z5V<1O{vbtoKj^gv;=YL`Sa$&BJQTT8;3^+isZS8W`t(UnKeFw6*6>a`EBN%siuGWU z)CE1DZ@DmDL9I+DD8Nu)V}DF;WwT`9g~YF zO1P|{AU`)QvhM%eg$X?;>Ehr%+%rNaC%Qc&j&*OUyR>w>HsbJJNXD11!@*@0y}ImnE|t4-?KpptK?GL6SLof$=5Zy8E1-MSgv1vG1k*jY3dtOZas)!C zcXVD}gdIkv{1ouR8@020Y$5xZNgeRIOQJT@sEgAQgc$qJClX&|+(m@zJiH0IXfkuW zY^PWND5Bx*i|(tmW=prL#xCLv&5me+EFWYRMUuGySeU;G35ZhxBjKs;egND3q69O* z$RG5Hrr-5-AnN2w_f;t7B9n3=y$(bfy$?Ma28zUP*2$lEeIvc3@JI*sP+Adcxtlc9yp59xcZ6xeo-3r(8*3=b6N!leD;trQg z_vgWOncH++aa?_LYWyInq~>|z)K-%2s$?!w*QT;|3P*sHw7+0d=Cf`^t6$%N)1KS0CL)PA4b6fkjPtb4Z9?sA&H`WNm zLXMk2T{32}`&p~Q?{6KC=Itu7`DPbQqPLk4t)|@H#3SE4$PJUZh*v^mtKH+1qrWls z8fiQ-ic+3`#{#JLeTtCA$z zCuY}HHA={IO|&FsGC&=G>&)ubxS)jgyw&YfkM`%PoKcMQJq@bHmTLx*GJX@W&P3eU{$5|9b)Zf=!u)3Wa zNoXay!PRar0Y(D}!Rub03>nIDIO>>t9}Zd9MotY+YTg~-8*$?2cJLFVqHMvdUYV8}~zefDhR!XchObnQ{N&4_6fU;N_Sw1@SBxW2g*Nch0g4fk0bmOF;LgiF;M&^hE#q+~@~=xr zS57<<92>*FXydF|ej#rvh*0$X(K##+KOgk-9KY}&61H;+k6C6-*0%D|&nHK$jvO^T z>+>?ev#&yK3uPri0t~x=r)rA^Xaqm#I5WojEii_A zcss9!2~2MQ`nOu*$URH$W|!}|%3UW~*$fgnmmfh!u!v>OZ zzH$RS#v(R`6yS#qr66Z9dJtLqV*mCs;yhoP%=681^TF=P&^spngC5Jj?W~+>;}!r; z7VNZrhERv#7!O;Dt3T7~jKPodo|;wY5VPRamySyfIwwCXb%3ra33|6Ypcq}5MkO>?!%h zAi4*F92Yb>N<=e3#dEB^&=yC-p*MrJbp?WSnd=-~0y#7XcN|jBPfgyBm`Nxyec&tM z_3MtHmF7xc`C#Tbq0C-^l#jx`NO7r6`y=i>2dqlzY(KMsV&c+8Bh~M$a_+ zl_P!y)*YdzI*>Uz1A!NPcY`$X$14#I=NF4Id@oFhd^jt1F~3#9ZRaOGKD_x7zB4;2 zUe+<5HOV0(de@{|X$LbP=BOhxhJ5(W@NSau3&vZ6X!#4$Rde*bh4Fl=N%40^Izg^H zwTPrHFjwXoZxZ1q;L4>+v$_RmZ=?wNy5!H|#3j0PohB4IAw|jXb7Df!*`{BbDyN@S zHnnX89>bByI1lhfJT-5zDLS6EjDo5l&tb%!#?K#kJ}2s}i7*nw1E5N-JjloFY2qTp zNgXVJL(cQz6by@p{fkGDRPhZ?1?nD>vuj4*gDyGLOdTqefqgQ#&}fk zW17Nd%B?H01a0a^{a0;j;-tsf*ZAR$Bd>(SM^!GZ+O?mt5uQpWd3@&>?uWBggih^%An zxfOh-)@)tn%rzT`v~-TY;!6Aa{bETYaM3!@`a3veT0Y%*5t_*VWX3uJgxv3HT6BZv z&|B~U%kS5LOf9$fBcOVbT5&=wh#}0geE9rWgf*>QQsfD#`k2Gm@y!U)IJ#$6N)!9Z zv_CaoWtNKvtw_qgMU-L(?(Hd4(H6py*ze*9gJum!Kt=OJ{d_RlhsM|*JzzB+|FPNR zE$_^(gf+c77SX^D2|$b^gNJACoXfMi?LRL$h&4LVyP^X`YgjU^zcHN;Vh!I#G-!d5 zTvpz2Kol|mvMglvkx?g$yi_&UZ*Cu>T-YiQs*jox}#nZ(U^>>dj<0{h#^aZglz|8%fVZY$Z z_IqK7fiM2Y%B$De{F1TNt<&)7lyp}zyNwV{V(K0k(X-U8fZRK2UKg-M? zV(3=q+G?ciKmMXQ2eJ9Hbi2{>QLhxf^)$YYd?ZdNr;8+o&CULwMTu1LjH7qk?W_0h*fWZum1aj@3a-8^J{@HID&ZueYiA0rDILO!hE-vz`ID)la_B#FLN5`^PYF7Qe>VAj! z{Zx2kFGft1BNW5Oo=nVfXDe2H*9kKd{{u3V+wi8uiU{KV@3i?AVXmRZ3YZC!-c$Vo}|!a{2L zc=jx>47k-)J+5Y;Ss%pe{SCMj@y&iucM6>=wE<)M&N_$=`R0XV}_N7V+1foKbd;ucb&;nG-IKfY7M_tH|uvyOl4J?I~iA)x% zA6E~`)X6RLJT2_BZk)_t((*e#D)~AZ5Q^i!!4n^I!7u3Kfjow<53%HAy60mZj`)(~ z;xi~fUG^nf_78u+2!49>a*Yg|Bt9G%G&oU^&^)1&E=IUn@bRkhO0%*!h3s#`oUi-sQTy6`cG=0=on4%Ss!8A0>Z~J=*z&f?g7|@ z(&+vWl}ArUpwPVkb1&m%E9Gu^2h`%A)oM7*7cr#ZX} zk2zc!TV3}wc^zoWLqzZ)TEO^$e`3KM^zslsUbFCl$?Gs6lu?2K8jO-iA&`p12^8NP zGZ-pUH(vriYhw&uykcdC&-w`NNU7H7;HfR504H@Pg_=zEB3Y=mw-e-u)xZ?$-vl^v zZBzCY;yd_Py&kH;%Ya6>=7IBb=8`Sdq6}s@XUt!ppqZ7eLYg*+Y~~wC2FXNd^#H8a zk7MnKPVi1{F#`66sn5Tl9-MA@X#F=3>jpVN+S&7S1E_is0D`X%f9iv2hOz z<`HN4W)J@xdwVzJPwOCEJ+H04M)ylQ0fVR;U2&yUu3wWK;u98AZ?e3~uoAqT2}jze zOZuA*7cU%U@?}OO_Mzt=%ckrir1P1TC8VIWh@!p7J|u6`5VeRFnG)~^AMkn)KFb&h zKH0E)eSH;+fqmsCriCO&#V1G{|DE#NaA@QNdD1ta*&p?AjRktrjt6`m##4H!&nGc7H{3ox zOTL3KDY!}|uced2|HKi<(Hai07b3_Ihe*@JUyk9T3cLvfhwitVB!Ok|0{SHQ4j+d1 zCFq>O!P8;#bHVViy=bmo{axLYvvTL&$Sa;n9@QXy?*!x&|EXJ-vLe<=@^H;HT1awn z1ZYF|$)(v$MiwYy{0+=2@0?V-ndc!m2(w1o_N+dc5A}2o%dyWN(Onn7J%Kw(gBzo& zgtd)Z+zVd}!`kDbKX@l6;8?x-nfnja@8(~>JE9>Kb$liv@|H(LjoWy_zL%M~l2IA{ z_nC%oU3NkdAs_e272$#eX&z#vjb51ed9ieLAm@Jjk7N`pqi;$`q46z=3!SdtD2)_m zjJX13CGS{K8=ZgT-f5VPusf>~zp76Ks0f0|+^TF#5eAy!4y}$gDX=%$5Y@=M>h&@T zZ=_3_Rw(Zzya*5_Y6KB=xd2%d9Xav_762d5q%9%q=Z;^AX_$d7ZI!j zMy^vdNh`<+UE~rME=q^*u@jzo@*ked8f8#M6ssN{fI?qglz>8%mEG8Cj0cbCG-2^UoWihVd&l1b7*XR2W7B4>;Yp;Y?uSzCutq zK5&7nx06_~lHXtMiBQOFyfoWGT?fB6cm_!Z*eZ1n zCkc;>_x~-m3r`JgotD}gtalSK-s0{0P_<=Pho@mcl|J76=~;P4AU<+`83uKotTl`c*E*CTt5HSY`KiqFo1oa96yH|(Wg}; z8|^tm4qieLCB3Ri2`0Z#e(jcrnWEibw%1M09VyS%lnGS5j*|}+RMdoUG&`mSv}aN# zpx*5Y@J|FDkWHJAtInboAHj-v_pc%QsPlm4YmY%Asfs-SrJza^DwMoKa&S3L}RYu zH`m!!cgpc5(I3BDV6wiV{Z6YI8{PxQ((NR00fCkAZ%wB(O@U!>noI*Ew`@>X5N|?i zTcQTtJKhNqBpbWMBM3hUF znxRvzGMNRO<8eI`At(Mqs*__3IYz^%5%aBC@ehr{YU00=xW+MsQH^(5|BE+O&kh+N zDh1&vrhxX-5j&Vy56P^O^!bCrM8@IP>{*<)J4pu*5ihhXYT5sWgjB){gQ z9asFhT8|luJ4retZ0m`{uvz<8KK~TkiO4ZM=HUb9(8tTpbk+!B94lP8+M%=H#VI?V z%)^<65C4a@HxGv@{{Q}?BH4FY8aqXnvL|CJL?$gFX>3IyTe3SOd$tG>Q;14s-(|9s zeaW7&PqG`^Z0G!rKA-RX`{VxOzW=$exvs7**D+_t%z3|Gujljmc%8-8KJyuUFYPzn zyC_!63`ZHIil>6l#}X7}#rSLEEM^o7?`(zlVEI3v|7Wr&(2t`-gsEpdrVI^%Fae%? zH@%#nOkXb$vCIk=Vy6cYA+u_C*?c863!QAvQ8?d(M!${%da`>6_?WA&bppl_Gwutr zPRAJYOqjlPf_wIM5~bLO39ItbbNM&+UX_13;cEmOiM-JM{NeS`6b87Zx3}GTWMe07 z1Y}&o=OTTRy&cDW#|xd*ig`e?>kUdWH4b&=Z30z?GWon~bI)t-#Hw3657+*~S@!M( zBW2`AVYroe++55cGf0%;qW$w$3Er2OzjZ-*N0>|;pn_l6IH`;SXNGti;vf%8icjy6 zUX}Y1O4FV4v?S?M?o;1(ZaJz>wCbC!<&N59O(Fs1iL`!7=!2u{pL!+jS~^QBUdO5N zdz&7(Z6kFwg+;$HS8uAO^T)TmRI@av82e7l9l3!&6qpo5@cNu$u8*#i7pkZ&C(I7H zi(A0I9^rVaD0J=ZDhj@g!s4p0mD}c>>xp{yiw^&(_o01dtWK`~CHKQ8K#n6^^!E(d zZd}>BJp7tR5Vqz6imhMr{`#usG_1bdxAhA{F>TmG`t=!7Zlcy{5I5aXP`pU{$E;LvT2jTw#Y1xR$rMk0bMHuq5n|;0 zn|-&nfK%Of=MPNj8e zmXd@)34JpDlV3n!G~G&T4G^^GSLl z#OQKTWZ6wu1iGMR%M!_xi$^toXjBL)F$a6}s6AXie}qJuPUxP2OPXV~gbh zzU19%Dg$gvT~X8rk?+;~K%5$C;*Ks{F};r~g?8AEw|15RR_|AlRd2KnA=3u%jRfJl z)suiz&-Wm;RqXd96#eUq>Ejcq$$rzclz`^}h!1;d?Uc)dE`7FN2an^n`66E|D~>A| z%h`aieeD!H%Njq4gwrQdREDhAV8p0~31pBqKXzUW-Zu^(g@?7BKR7@@YbmWzJ`nB* z{|K`?FIv`gocX@w`drke=NLpsHPMKyD6qd`q5{=CZ<;{5!SBRNSwXzShn+iJ&t7AkeH! zlBY%*9Vq^k%@IoH9a+9w1n{XG=i*4)AYjLL%*f~-g+4@cIB|C2kM z&oL*RV0@o-*e}O0W?Oi*d7Y@2yDORk3ALMLA$9MXm^>G0OAs|RzdbCtE$ZB^m9!cE zt2Scm%I`ZAPAKL~DYl$<98sZ9I(juFJSF>vYjkoipx+|wd3_|k6B3UT%(P){nx6qY z639jO!$vKWrME6y7O&@0QHFwfZR={Q(9(TUF#gGCwlh=}4+VJjIZhbO@*r-#=YMBN zkvhQ=zKZTcKRDwLUj5!r;H7@f#%f2GCN9wDXh*48|HT(&9az5DBR-4ume01pmEqZZ z;dd`hh5G@xC#6q#rS4T%UE?Uu?KAw09pfb1x&3`@KzE3Ef?EKUuirayHz&YCqtm;R zkn6TPO5tPn%C}@sT@H zd`T{MFPqyFAHG~gk&=%wJl6jF0m;%?%z>iQHZLxPHop#w*j`Q|BIBnP7IwcQI-uvmo9AZHvhNA)`tzj# z6^FV~6ZvdDxDygWK)i5x%C$Lj=|Ol&(n06TdslfZ%E#%f&72ntR>=pqu>-xxXih}? zvd-Kxe4oK4il0<4b%;c{$g6Fa%$kb(A+Z$`@rAH4&uek~lT0?X2hwe7hp}URv=WZJTja;;FWCUb^ZS<{wHoDCJ*yzbSqw2aje1--eu)&N5YS?65 z0%9Cw1oX5i7tcWlJ^C@KyUzz(x1*R1>uOCwc{YZp5C8D7OCS2PWezy!W zkDshHr3mzYedqb|M9P5_*vP=rHBQ1~t@$Gx&;M^PEE*A7dL@!d8FE{&zl-@Xm!aSm zOZ@fqebAkE5>K8e+dONp)~c@3=D#UeygbsI>1h#YEW>Y5f9gqh!u0CjYh`sy^!!J} z{)D}S1qr9qGht}+nFmUu&`E5cL>SVG=LE;Mnm%jdGg zu?(e_+>oCyC@2w*d#@l%vv4}tXeJ_b4+SVaITQA+^ZZE^l(wGc$&V%-5SM791QEl1 z+sWp%SRy0>KUG>GR3aoA{Q-LK){;9c$2kb;im%=!VRKHl_SzpC1aw8I*n7J;r>h)p zabV^XovygIy?tM;kks)lYoLB(`tP7%+r(e1Z5NvdVwQ0O!;l-F6vM2dm4m&$U~Gx9 zEw8*rJF%{cIX*Vb@8yy0@x=^;&6ot`D7{!DAuKTp2sLJZ@xwu}vD8J}fF?s>MUi&= z(4cnm`DZYjlITOsoh(AFTTVVQzvIc(?Js{t806;C4zoD_&mrGpJ@U1`n6@s8d+^J* z1?lwN`!BUAGSVS*B@QmH@4I=={XcflFV;}XUN4GPc~PIx@wlwO@!dJ>5MaEEf=iZG zKv)<;Xnwq~YyK80-YR*rhTI268Y*IWHL9OCEbO43}3k#+n)W6^o_ zmRZ^fp#8bUc^-&^W*nEnk(#^pMFw6a85@&8E|iAZr_dWINyaY^C1L|@Qk@Dz*MRQ` zH(b$fJk{t)7B%5b(p6AIB^$r|y_fUM+Clx<{W}ioGV1Ox8)q%LsvoQK*gq@o9J!oo zkM|B=^0{{+{=a8+xB$EE2U4_ihvyXY8{x*=0*WE5?ImzteGVfgOt#8=vB=T`KDSFN zq_)Bl0C?eXC>X(f`;og=0<0m~5<%BwDmn9DXXD8cIK)^9hZq~EQ=MZihTqZ}^+uh{ z@wEruSx>MxF*%Nw$#%hLOhx1+0886^N(q_72KLz4iGo7k;7HJ%v;~4J5Ge}BZ{3oG z4l^{i+1Na6CQm*A(Wh~E$z+a*{44473PY4l}_r*$6sYbn!wnhpYJs-U)7)M&1>GiIWC8fFKtN5Ki2gycJ`+W zU;NW*!XX@hF|=`e>=+I&kS%I1WVg!8SNBGv4j0N!I@pgN{Ul8NMP$`K@!%yUK<4$1 z=l(l$u56`Absvk=H%G;YxQNSORr&*l^G1905r+n~hHtXGzE#qy+YHx?xa1pk1vo_` zJBTTjHP*k2M=ZUqgWk3bjI5v<)#;nV<{PmG zkEml!)jm}jBGvOBQ>=u#1S)xqcq}wuT}x8bb;i3rdBCTKa5^PM_16M?wU*xBLy14<42^^yhf(^hn0YP| zwvqA1B6?Rlp%8^<>}>Gg3bRC;E&`Rc&9@vNqGnHXmyAk-syv6@fz@t;dGXhk$hG-F zQs;{w@Bv6!13WY}3=c(v+GuAwz){~0Fpl$)^6BN>G+i~9$!`W*kqQkp4WEbP=9A5X>K?BbPpT_gL<_^?!qdVqI%{iYqC}Z^`D4m?{ zszyoodawvu3yng;a2Oif-^EnyUB{8Zgh*rQm-|EVh(Af`@_d_z{;TJb6U z2~|3-4Ie*1zso3f_nW{TY>;8b^eh?o!EEAl1=RnM?bCb?>?tJAvi z`ksYK@GFs%^LpsiUX`G{W+k&f4^~bhYGY0Q_B{%II!cLu9wrTUVyd5PhZNwr|Meu0 z?by3e=S@+^*}at#JRV7xF2lL??Ev>{0;d; zDn^N%;3`C}Pk>eup1R;j1>#ZA1i&Aajg8V*f0@YM_sp8>FOjjBk0;t}K5R~Ur_{Su zOZEY9?9bddqWW%Nh0x5)G0j<&lH6&~)SorN+5vRg+Q*FG!>+lA&>62Zr}}f~P<|g9 zvB|C>c1*gZS{7RPh8w=S$uEC*YltPga%OWWfxL4j&^_}Wff4xEU%#Ka3am8S(nM2O zzx%6qEyTcxiuTeng_ zjEdiIimbIs4_99JbB20U-#Z#uxRNAMQc{2CcoAEv1b-9|Py4){R>~nHkAiuBJ%ztJ z2~@ktX9#q}i5lzL`xTw6J2cAOwMb!Fb#v%BfusWibL&7*eGb-9RM`??w&OX=AZBwj zj5S-|U1bA#`JxSjxHqW66?3D7tN54(OVaT{gxyUW2_@LPpZWKq2BZ>@Z)I0jp$1Z>HUOAXGa%c~FMDo1mEZ8y!a{9d3bvD1SC? zu3>((SN-)bRxcoM8(&3?e z5Zfisw-blS-Q~V#E3Brrd%r7vBPHWk@D#usC#|;GJ0#nciU|=>+=N-C+0KW5dl0E({>l03mtG* zj0LZxUaJFC=TPgp;tTv|z)LyOas6~#jXA+ycBY|s?`0sbv%#JFYM+GdAA_Z9;(5rg zZwGOkcUn*Gd!Md$dj%=hd8qw(v(Te)wb||~1%8L{!|msD0X?G^u`l5N%b$7=t1SqR zi9rIk@Oc<})Sq2my*vtiztQh|1}U!y0n~QEiUax+4**k{ag2u!>1gcOcu4~Bo(om6 zE*U)H3Y#DFSLo#o0ovpCFLtDskF2;ej=wze8f{iMipV<8aGOw|Q%f90Q`*PB7Ba(y z!$ok7)HWV`kzVtJ4)e*D_})3oSd}E4Y}_={7EL?li=9aeO4I*SLa-VEv!wD5oElWK zc~Xi^ku26F9aRuK&YB-3nswsA)|boJ4op7mYQF-n!nwjnVuNUSA;!5gX+T4DKOvG1 zO{1o*@K-{+X6m)4{9k2%f6+f$_Y^$zz>5@EpyJ>6>zz)0abI}_)?OD*7kyq1mm!no z>8nna4$BWE_{Y58Qu<7@dOYy(Dt_o88K3!cVs7S+;dJK!k_augR2o8C1(5~sdFws^ zoT;5G+FV5V98a0#dx-7XPTS`0RRuLkLSWFNb?@n!hNwP++?+@uOoLUAR}>+^@69k} z%6yhpZejfGP6w^blbn~ehS|!{{w|X%Qv@S0i+P$1G8-p>N12vU6Kyth$%E)SraqEy zM3qzps;rV>bw;sP`^h2@@w)>$!bHSet8usF)8v5@G;hls4gg7sk4A>`qv`RTMi<>zX+k#aRjdcJT z8Rkzsy77!td4=sY5wkW>HZ(q&W3u~0(1>Epc8&vj&OL`7zdp{q_Mq)!H{|M7ift1e zfF1ln44qWEIQ5!XsUb<=Z6f?N2xIt~`38M?IiUaJgUet+iD8kwS-#|Tnh!gp?uJkE zu#M5pgXo4Nc`(Vg6w4#E%nDb-v@d3!@2Gk>y^+QW{d)!2o~wA+JU=`GvusidRU-P( zrI%q@q3Df71NwG_YkkTeThCNWQq|2#H<4dA>h9cc^_0sdmYpoA_^#lLCu>dSGIxg1 zOBfUG#HeK64zIB6Q4q-Pqtv4g!%5ilWsl6Ch$@-DA*F(Z^Yfo*?>I3Aocf{7TDl(09OXX2*xwT*8K0*2_E!>r2O0maZeDJ%Kk$VYewdDYO@( z0c0;zIx!<@kaa-%Ba}aQ0j`y^QluL%eF8@tqRdyg;bLnXm<&UF9n>)WOikMgBl&}` z;9?phX*O-P2aI0=CRMjZES}l6C*Io-bRKB><|I(m?5UdjFhcNxLFN6-uxHt>%0Sa( zzgk|4!X`KPLxI<)<%{}!!n3A(7`8Ec)W-XL7eISGEGMSsB;lfK0rYj8f} zNz)Ls8dqfY%UJV~Ce9aAcT1C(P-91~+iN5@A=sh;2LrZ<8rz#@vlfb{-~iF!KWCdH zASntZ-=O*6bnT515uq&(={-AUn=X*S!5zKaJx=TCqdR){ViwNM&iE>Zk-s;HxS$Wa+tdk28O#+`*X=P;93?nc5&pcs`IEdtrA zU6&H!So3nc3-XSQUG~*9nqn(^UUC?*QbMMCKC`%j$0P2&!BVe zak9uXPMhK!n9c#cJWCg`Dlco;hx-val9Vp|3KXXcZS5RD{|7?XbqaO+zK5t29No#y z(o!e86{f%ze?bX!6HJ~cKBmM5g&uQHq|$R-j@Fi7oJD5HSbp3`?trMRFexU-LqH*c zvyoPcZk(ppL}>ZD-n#z(%mUbH9mX9P>h^KxU2f=-TcR-Nnp|8OrZ^@*a_#~c*-Wj^eQ*{?C*aRHgJY*>F|wS{#I!c^IGkyvs%uiy zz}n59@S0rRy?t2(P7zztU%_KF->9)^A|riYti24Y=DihQo^WHYU|nH^C~tPiy5|4p zJNGkNR)@SLxjSYRZTrZJEzh27ui&?0Dpleq4BJ}`UZEIU)h_Hg%HJ4&+XQ{wWF6-M zn{e#a3wyu?N+GsZ1i~;as`R+zH^;!-zu*ojji8IiDX?ZiLtQHr9RAw%W*Z|1O&u!J zJj43EpQFH@3-GwXGvM{6z-RK}THK9kMfLnI5-{@OYdy+m6xCDzF_SK4^u$j<$>|gzPW-;!?P8AKz>hWy$ z_rz?DvX2z(@RvR|!gi@;SRjc&WD|S2aFXmXfk%Y|rn^rc!R)oB#GQ>NMpLH@I9H_L z+jaksTMqZ>^mju~ByH*pYD-4kfe|Euq zZ^1UjF^FECRy;g{;wAl7hetg51Y$N?MJgLO00;PKw&{;_K zy911W10DzvE1mte%DB}A1g}55OdZloT^c`m?VM1g@SZ(WS5(9kF&^LUzqbQz2}Kc~ zUmhARh;E%dUSUJvs^DQ}U?w7bc{1_idA$`A(CuhGzLI3?ddNo0HM*c<^_6~C{#ZuEPqF0N?#bETI`cukxao4l7@Cy86 zw!7rrsawJibV;0J78Zg0>3mlQF%SFTOzO15Z%dW?duRDszJx=;nOrc$Ie+sc-X36s zDre5uQ}11*B~O+dZ~boRdyHUzqlfkE31BN>Lcyge2Ikztk7!6;13my@E8EC3M~-sB zT>P%ve6%EZ_qmuwLwWeoMVJULi%S!U^VyCK(Yi_}SS54m*Ld7BMYCJvKgt&Py*_l@ zo`e!$GaeG~xh$H`qtwOqfmr;NEKV?!f?{KhigG?GM`6y75K|O0P349-vMbd>hSg!Q zN#3MGeW1nADuaFPGqWkl{y<5Pfyblt%j~IN&Ndfe$`QVdY{Z^$y7E0TzK~;G(dhTN zY&?CN&XBa%by*6B;{>XR5{^RNpMhcX#7L^5nmfgYLAG7>Q} zbf@eNe9s4VE*OicPCT?NkJXZX1@d1xstFs`rFSWeE>Vd|?a7TFqVL>@W7cr~^HVX{QEBHwS@%6YC{S;zuM_vIQdL)uJTLsI7T4v5J#z4?9XAL5YvBLx$*Y_eY#(0S^=Z21sz zKd}#6x7C%Te%6Br^plmYd!P#!{+x;a1g}^&~^NPovmOQf(91VD!UJB`NPLwxXw$< z26iZEtUWLoPT@YW`7ZrshYMkT6%AO%wY==b6cX_3>)BbCCG(G6J-CQyMMlC4%ikAw zdiWVrFV_=E8$VyAm*ltOchPl^ws5Ld`+=EzxYTddQLyfzI#c#zE0`LGzuYd?b6B@0 z8^qK<2;MJLWGZR((=V0bP<)`5sdlHr4PyI<)iM^|c-e65xr>zSmBia2X6Pz_0CSDNw(u*eK z4MZkd{(Y2%Z0jZloC z3JL*90>lA{?xL6d{F9u+;`VlPrOOjNeb?rF>}wQM&51-(iX+0o>&ChJ#OI>5_ve2ZKAev zqmplPD@WFMOq8I+>+LJP+18uwb3-jyyO>q^waa@3=aRq>jt3$i3T|1=vJ=F~LI0`bA`2N?lh%N%nXJH1Kwsyf z=_$gSXE&-_>2H6eljeXJp6iK_M+tWRmGmBaqDG1sAYO76+}DWg`Q3hNdre&fs zFjGYyRlHVpm9>NR2QMwn0RE2=nO?-k^z(1u$p7+)$u@Jw5MFFFFAvtSfCw8Bc9 zE7`%nod#0irnoEEJYMqy{@4l%D%XN~k0*{Tge|JN8`bBCtWe1C9h7wbGPpHIjGO~s zonrjbDbk%#Cs@@`OVh*pl9K}lUTQuhvK@Z!XPVf|{?XpG{D-vnMVR*f2UT}MG-qI^ zJLwP@7u!B`x^3?qI^9SZ7*F15|L1fMq7}kJ4pG)aOTGS|rGE8)EOl|JcYLBW-JscW z5w3~7cO$Q#Vo(V+;n>(9(=KH2g`~M_-`_ng(X=}{`0ZOV#Hdf3bxPyPv8#t(xnLPh zj^0B*T~$SIy~^j`I3c>HG9LxSXeGdC!2Nl%XC+8LRd(tl0+t~@n?G&A9#hm_I@f+*IEvV)tEyFsEU6@=Y6@748=BiI?Lq==M!LS;JmY-q?RGBL##&>0J-~I>u=zRI zfDZ|LRs7VT4f#lKX%hi_H63gBu$iI9 zVXOBp?56DCZcsVll4T2Vq64C5{cG}_8;^7e{9;A!qv9$y9NKFR7BCQM32YZ2PJN%c zquKT~R#x@&-se*s_fNk*f!ew$`|Ew3XVl*|Bfi|GB0?szZiMg6$BP3LeewMl)fc`! zp2bGAc#8McigsB(I$~bRaeG*z30DFm>aAFP?Y*k}7mH4sD<|PqQrA5X*_v#6>+w6T z2MZkbuw2~Eo(O)s*5s*_?HvwRv=_5X33(@mF6XhCbPvHtRDJ(w{U7YxviB;BpuM}= z$oX$~<6F~(TuT4LiN7Zb@jkcz?^(xR!VWwb9ddAZPQz^PR=+Vc(@Zlw2>^B&@FzOJ z5lPz`y)x{(vBk##90!0G9_09^-2$0Sr+`&7ne|v6FzEn+l3LU{fsj3@dJPUVx*>he zt0=x&d9c~9objthF4($Ya$~sQkL5^SI&Rc)&9)C$^uwL52POUjxR9P61w=Z-@gTL6 zBkf?@dqX&iJLwP!=V`ps@$#_V6~0VqFN=ql>b`vM|CDt7XKk3PKc<_lKQJHuJFwg{ zima7shod1J+sGbaJ5LJ|ISn9!iMS+OWP3&SD|5!)|LPQuaXx z%N=QulVJPkCu|^c6XqdE>n`&86Lfe4UVr>^8Pl);Q6q6saISuqi#*m>DS21mpBa z3y$DJMveiSkIlZm~optzI~3281zqwcLgt4gq1_v;IofCBsirUnHXPdeJslb7sx*!r=I2&GA#>wnou6Lvjgs=G;jz)efhX zjV)z$j3<<1#@fe)B)@~Sd?zJP2zqKXW+dUS^YEAO;g#nY+IJ|a!_l6n8aBqc}o+(&`DsZ|+6p;sW2 zLmu+U6Bsdv3RT$= z2IY4xK_+78fhej~I^=w1`?{p-jM2uh)Rry%=OES9xJu1Cr7VXY9OxoMEgc3psafG4=~G(giBnZngxXo{!+1 zeu7SqLZf%+eGP%mlcW!o>3-|lYacr=M84*Wnme=0@f8yRB3o7Oz#SOH@sOWi` zq%#+3^@sVyg-T&IemNocW(V7i<%A3QtooS=7vJ$V{>-Pdn~f6nWHNF){}=|})HTAH zM7W{1igO5fpv(}lpytRtypU~$B%cvt4!NOcq4jtXpcNpZEp8AZQZ%kYj2Y`mh4t^} zi=fQ*)-9)%do3|Rzj*4YuA_3SK#N$aDwcTdexx=Dckjq)gZEHhaIaUY!B~Ulwi)Jo z;5K!=YU=5HT=nZ}7hE887Pl|3m}^eTOsC298bajwbCLVLUj9O?;f#)f-5Ioi;bI;T zqD#WPFkI^=PWqX3zsozlNEFD-BPM^IN@PR=8{0pS@fXdw5ZB64r+Yd*gJT~Cq~zRY zr#rJTB2nnVLr!R0*z~G*5{T=45IkjY@31jtm_zfoWU#=K6JeG->)uMu4(}lQ69i0i&a7k=51QcR-$$NcBJ&}5#6RNz>4lW~SabT{SSN}c$d^X(M?afWy*2QZ*bY$I7X>h+Yu@e{|FdM(koX^<8|9BdkemCF`mOxT)J)U0 zWr`O|5t82b#PaB1|5tIftaH6i5HTrLYz(Tum<7?i*!0nOs|KwEtQcR&9x}Bp$*kn} z`-*!lQ_WWFr?#c5Q>7bU{lqRtqpsBAfKzJAEiF&M|3wkgs}67iPCYC!nmis63a9o^ z3n$+!VUt<7t0oq!z3bY38wmX;qreMi6bc*s)Ci^C;jELFRq*OPIMokCPR3bmM&rxG;!ob6_?}qra{JH=B5qkZ<2UJ|7(B z5r<3Fm3}qhi0~a=ccx&AE2PQv^4TNla{DR0SPkfvFdSc)%?}l>;O_hu+tjs7Ngyx8 zCI|Ng%~!mW)6m*Wxb#DAYA(h~7U9*NV7cyF0RIz+zbL;CwgkwdH;*S+-rt_%l}D1x zpildqJj9hHw_@XyzQUwX^vcZP=XkD=tyrGgLG@;2iSSi-sUa7k_b82lRaX_FV6 zjgxx~m#PkeEY*`XfLh_G-|vn}*A@Ju1XiVX4=OxeF0eTIT#Ics%4y72mh)el!S_>< zZ|bULVaR)3@qTf%rY^FKC4ogs<@cObio^aMD57-ULGz&`aqiDX#iDyh%B_UEK-pQJ|~ z_K010)M9hxSxqTQp@*o8AL2Aj{DVSk{gwo&lgF3G3OK)D?eSs9?*e0q_knPo772KB zPw)ocsiayYN~g6zvH^Rc$FbqLZ_s35mJA%{4}9({i|09#eR95*WKR=*Sn=0iud<}akuYtHC&H6sbOUG`d# z;)tq0Zi+Zh-2k9%8SJ)9m-Bt{#9~EJP|tS5e!Wi({<~H%u}S{m`Jw=JSs*n5J1QmX z=1)M#At`#u1tb-EgV}x4r5cn>r2vmr*SF8P`E_k$sr{^S2(SYw2Ui6NmJJ&Ln5wDH z;GiQuKtJUUI%s!02HN26IulSrGE49u-u6cCd(;0IT7k0Cuy(|vBL838ZK@3(HA~k< z5B!nMBjk7V6aEOWc#(kMT!|akcuT;Nt$|uf3yN~E9SQ8DF1i8~DX@TL-9N2Pjt?K( z@FzAE{ngs=<_@(%#;^zx|4r%Zy;v zH^2GF;i%Sq%J(4Kn!j56mJRrT21jBCGSI;fBw(w-3!dk|+6Wq*EGEasfc-fEEwHaSO(dMTS!SLTR_{By4D zn|-F``AW@LN7t(OL0~H!H*bw2l{&uvEMG`xHWy0MDtH{(g8MKK>g^Iawk`#aM6jAU zxG?!H(A%xX7#qver<8l<^6Hl307u0m1rf%nIE|8!m>f_vA5Kqem#E!p}V~UK<1XAK0ar`}8HobX^kl_|xFhyeucQj8>;uLjA(0 zo9A+pb{y~WhrR+$ZaqKWLZ_bRaX1D>hF?vF?U>c83&|S4MEQ+?ra`dPevALtr$zWC z)-Lo*;$t|W!;*6YT;B+4M{GqI90=5Uk##uMp=1nYVX**MD)`P{^c&njk`ZYDwFtrw zXd3RkXAN@lgYS+POeA93GrQ(X-8k3B1bUppSGpP~U1_}3VaBwJQ?v2n_P4tG`LVMZ zs*e*EQh=u)j*P3*T+76^2H+qNQ&~^!al6vRPnH7y z7NojSNJT}#ML}yTsgarsjfE=H>S2*uJMX(z?(b_L=7x}K;GPc6%ScOClh_&vL8`-A zW!uezlvtp_?QDDcKw41i9oakB&G0U`w5FGjPx91^&EBW>S!I)_@Nx)Zhk@GVr&Y_< z(|x6^()HV~i%Pd|!j~XnC>0UV$oPfBxtwOMaw(;7ClZ=Pq)tx#HNU-ARf)p6_iMdx z<%8euT-Ry?I%Pj=|KbP)Ow!AOk@7^24X;J+J92R6gO*JIDD!D|uq0*WG^tgsqM*si z)WUTiWM7-1`sqmPeRs`H>_@xiHf&uni-h4ktx92sF}wcOqqE~bKleQZD^%Q{wxm4- ztAF2%bxdsgK$LF<=|`Gy#3h?3IB;^9S(a$lVPw=hcjH3-(_67lPgOvv+HRf1Hg6B& z{-5>au*s>ZqDy6a(9yLN3!2R~+)U52=XUqGPw5^`dGD6 zg4!LRNujg6v||$=zN6latc$n$?*Na(QA&B(WuZeA*&7UjM|50fTN1ee4-m!fQWOzg z`QF;9qL_bUH`21PJCqOjA|;kffY365ciO~yZ%(Sa`K5tf51y=fJjBF4XPsvOB6q=G5Ym2W9&{>>gdchO5Cn}wiRwk<*jW6 zc;DrA0PT*uON@br;Zt)82U~PZDUj5E`f-?z)eK+>;G+gMnnUc-mG&E2o7HmI9cSM9B^ap zce&>L8|aze3Ia+}W|a5Dkf$8|Hp4bupXDACxJ&x9=e0L#?c=IhH|`8zf+4azc4sh6 zZlfGC3y>>DNdnXR_4x1g2>$QXH8*qvM4ycy#%*2+xOwmicQCxz4DVtDmZ7QUVA0da zHwmmK12^`AH$001rpx@P;~2;<|7OuSpJJ$Qx9!Gf=pij|rBSL z&pn?kyuuU}Mv~If8g|lBSKO*?YkEf_Fb&wq-(qOGR)1h7WH-30s!g83M&=1Xo|KpO zABaueZ-9b8X9g@p8kNUwy(}QB&K1>f$2{;+y)R3hD57eRjmR5!8&)o2o&Cm|Pp4iC zvr@0nB&+vXHjpo3hyFHr5lx*5Ihb1insBN(QXW^}?1#J9RX`4!)cU>>u{beRP=_O& z9?>lJCs$OW`l?v3e)-68D&yD)wyLU%vgZapTZ6>l`Zmoi@b67nW|@hUttv+!t=|3W z)?-yY&Ntu6)cymSNEOsC2qXHG8Sm4zKc>pu-yxNQ4&j&Sw{nvvFoPgM3`XJiImfO# zKvI*S4%rW!6c62lwsDZm&jFVhKGU!FAL<03<@8f24E7rJczh-7p6cf@Zthh9W?9Fz zxgq~?Rw3W4KFY5&hQlPyao8|3kjsNdhU8%%ay{X771ZG%V(21FUs#;deWauG(fj4k z`mx&}8Z5YGLRx7%^vv|Gxz3+hOo4NN9;P9}Ml9k(ML!`G9!=+4UkC;|I)9Ix^DHoR zoBRv3)ul?|cMF8621wGtV$enqW&?^-oi3l+{I-&YBqmo8amqj#mHHH`easo8_61CP zE}kO@%o@exNIkfF!IL0?)YPsuGCdr;>^V6(oqD;8`mOdcvAz$gU$kWn8r$ymX&CB4 zZ$3v@H!MCvm8sUR#m(CS*qvBt)RVe9dU1bda6NdxPQHlr{f^xJ_rBZHECm8m)$}ECfQQ%aNx(~IiIH{H;1$(0+(8d+Tg7Q- zOg?aRyQ^|~a>fcj?dMoi^mSh`!LNIpjr=C~3vVkjW;ZzY>;7rq+`uj6hK-=mi)}&h zuHoaiJ*uhRZb6ycJdlL2L8J?-VXST!j1q_&`+eZ}_QqPA|2@P(CaY|bUypNOpzQ2; zU+{=$m(^b1pMYGZ;O4zQs_j~r_DaxW>S#YObB1mOZ~}xTE+?Ov*M!`7tH_bxS^-w&830Q@0Y&TbkzHS$ zQ*0JzVKU)+mm*_mDD>{~dUcZ6;jHHCH!}nqg_hk;G99_<9`#z}>8qX5#T#UF#&WcM z8$^0UttOW6NV$Puw7Cl+%RBZjmEFE}ex}9ck>H@at4iU?LiB%9hl*LupAS7vc-)vy zdg+|Ru`94hmo@!}<1*N8@?bIR9Og)Pz*iTGaoF|q1_y{m@ijNir>-slx-fA+dh%kl zC}b~~1ieX-j+B9%yNjKfO`T5aW+9(~YD2^@*d(T<_N}aC;`XcYztid_tEQ14=Xyop zh-2``_AfN`mvMdchI!e&l7PkSAtd2K7bj*Xuc9gA(j|gr-_^2GOb~YWYbvH3OOJD# zD`>Rb@$ey6FRJ$in~n=Mct9BVAxN|ZkOve*`T5|wU`L`l`58F~**V?P{`vqn%Tk5D zzgL=eE&+UN5a7{m=cf6Fwfod}!FikGE2UpD2GWt{Q-pvr572+FjOb&VwrGB*&EGAI z^_5c)cjqz> zg1nEmE0FYwGou3EoWnI%^OQ4b9|>-WKxfH7gzh!Qmff~ z!T5x>iKpXN|5y6|rN0StTvLY;5E?;YwRUvR)g#^VGyr}6JMHI?wI3RG56fW=_lA!N zqa@@eq6m#IPu;lLAp{H8|H*U4=^TPwuKaRmh7n^QZT#7k9DH#3Ddch3Kw-8-Y-&;E-pJz_u0HobUqLV(9vg+~WLh4X-knmb*4} zLZtPm-n)byv%P_Y312t=o4eTk???~gR3G(TbInG~0dBpr$|`twG}ybQPwKbSq?G`9 zG&h$oyPTZkNmcF1m1)<<4b*z6L-00)Jm+>mZNN%t4vdg--MioLz+iI2Q)S3|w7qY_ zegs8KLFp{;HgxZbL9+&_SW4G^0QiLpJz-(X6|4@vMa`Wh!OdQy8@w{jGN3H9JXVM9 zAt!+4LNSO>UnhcQ-Rr@5yKaT6eOX4TUzyefrW z*|mXN{Po3>735=zGV6~Fzl_d~sN9VzKN-jkrymi{)y%lBOBd-y6&YOI!opYon$ z9ufwU(I>*F1<0-)OCr|EDuW|x$v5eck^4n#Ue(7V4=>R{;!x0u8kKx%v2euCb?LyI=hdOzOB0vCL35rv}`fcqorGiiaHX=8kT zWz|?_2*|X;J*=7hy<2i#G~ygyG$>NEnn4`11pWdBwg^oH%KqG z?w7@K(#ybbDLDDl|5VV=268M(>JRusJ@sfM18<;B&I@~QdXwHutw}zfSC%TT_yh#~ z@n_tjxW~ECK0yv75$YRAe+_d`nNH0n`~?x?H0&#;t7h~S{YBk!oxX4LozU{Ee8asD zeb-|xbJu59fK#2qvkBd}LbDiqdBm&vX7l3|Wr0AW9rW%X6_)%9Z8?MmGrI>+vP>j| z;O~GtUucCg%_=s`xlx?OKcAl4*C3tVo`!4sz~WMwJ4}rKAvB&~uuv6cXs5UbrOud+ zWqz#XZVTJbZebG_3$UH=`DV~@QRPQ?0g~?1UIb;ZU)O);-*{tgigPy zj_VV~UY)J3E^x8ZfBzG@LWy7b%y*emE#Uu=qkwGobvxM#+ACX@2E6;JC8}EjKfk;I zc3Ao%VEHVV`!VPeVeg*8x?0>6e~0wl^g~rY&`-FQ3<^7KBL{yUdXK5FrpPw7`xt3j z!TZr~r6MX}ZKgyaX@z=beWp6PqzC1AQvqOkQ_lH~IeCCxyDw-WPs{u23ldH@Ky55A z+;#6~sekn1aOJPPZEnmKch}%&V%g#+XV9v8+=7totXy7;$D_@HHks;?H=Y(id-unN5 zzNJxze9~3aTw(%oRCQ?@0BoB&cmwe(rz?$)i7kePN|jIu8|&5j#oue{;%x8y zNl0gX5!BNq_pFaPAvKFO=YUDJ!nTFaal3C;5MJT1(BARGSgAFv0RuM}!e66el|BBH zM)V*WT9wpzF>{RlcJ{vDLUr^D=8VHsEd`+AyeG(T{Hx^N+vlRqB_q@CyZ8px1MHp| zLdvVk`xWS6RNaB@c-uQXx(W(k9IwZZCy!5Ucn=Fo=De_$%i9!$v75!g4d@&39YTGw zsU39npyUK2p7zH{jm_I1V2RG3d6hg=z#EG7gf}>X^0qaCxZHHXdE=Uz2Q?=vxM!aU){Yc2GL||t^JZ{PQR0kEmxJ; zB~xV#N+fd>QOf;vSiGw%rl51&6imRx29nEsUB&$U_<33_o}-0shWZtyyFUXH-&usU2Rd)ruxIl7K@TKae@6`Be8 zPXg1*>nAoI4$THF#=>DTlSt`>+f)p-_fG9QNa*9C{<_2KotaC9)eSrQjYqn`r@-OD z_`Y+PzlW}clRpiLCO2OMg3`}Yo9A^hXvPPS6gVekd2zchT)Y^-I$SjTr1EyBkF#*K=j}PCMS#9HMgx z|1j_U<-dAjJv9q@O13fYb*!y;c1oWCe_r_C?M92<r|d;JqR8nGUCDJNz=p(W-_^D&OoyU!Dh?a|&l%-V&hkAJ>Zrm0UY{RAl6KEbhfo`^5ZLNVJJ*rM zI6YJR;Ko+|_YMCfAVMHIndgUYpsn~tX}|Og)7V@EoU2%L$vUtwa>Qq2X&;N8Szl_) z*|{jWNA;zVMVFCld(MXfS(yCtH)IjYv;jpbxBd&RIJ6U1M}gJI=!=zNOVi#^-wT`h z>JP)U*bHiK#~O2z<(l(dM!Zh{Tlv$kp-+^j{GVEESKZ*+NMB*TuWXDCku&W2B{+lm z^1$>DmM0M*a~*vH9Vzdh{I8JQZkr14%$Z63%M_1eaOyHFMhBns<4Xr;xR^A~Gv`oj zE^@v1f6rNK|Nl7$2iX&J8sMme$E<~#&k>f%;6E9cN`qy>7@4xbec=l_H#(&5Nng-8 zmrJcSww+_sc_eIGbYV|D((vpVG_Q@#qjT)N4mi`2fDOyoEQGS_HfAY`Ef7(UMfsgJ|*x63`s#Hce0P1{W7>^^& zXY>kIDjtz++>So0TNfJHQNC(gM9g&=(0HVcBQL&EiDS79gNZ1L7erZk7)I>g>2XFK zyb3ThKaCHXrc4$WJ}K`E3VLLR0LQVeveWC+z6AEyKmBrNcaZq`aLk{2%2cImY4@!y zZF)oedMR?`dEMT5!;h?xOpjc4x)@RsJzc~pf$Ywt5lVZNf4w4NPJ1>UUH<3W)VvJr z*Gv!Alm8YkQI)eQSE-cNy|BBKaaN#e(tW2}ekfqaESn{>kf$Exy??U!pb6;`&g&jA zTcGW^_jtPL=u-T6!6xE<%O126F8hx+sP%LHb|AqYTY=AaEyTRt13+Ta&xf>HCTF`R zA3A$Y)plkvPo$Haa3=F4iy7PMw^x#LFk-I{=j|3ml{-RM4BTiT+1h;^f@4kkWL z;YKhXWO=usmRZXM#8xh?deu~mV#wtH?E6wB5LUKS+%Y*#qKMRXx^Q)HHIr&1;Bs_4 zYR)>xp6DLhbo$Za%5%P)vtE?pAik&k%at!~&Qcr3`T4lmNdHIt~Mn$9Hu~Hy`~z3reIbN zmUf0eA$9~%5jaGAx2xBscTv=0sZ|)ow@*O_^;Y=Q??~O7)_M8zWxf(gQ`bAcb9K^l zPzPT&I92rzMDMNP7kO57qd0d;@k9(*M%zMQr*0`?R^SG8aQs2J$UncX8v~O)lrqDv_dC7Cp(D7;j;xeAlenMh`hLkcn{4KYl+<o18RIs0Y6Y5g zXEA%U0&oLd5I0Z_&4AGqUj0P+l@qQ?ZY#z^|8KDyGCe@XX(nVtt>QpcY- z&eE{Kod?caJ}==~ICbt$S2lZ#8wXjCfIl1`c0NykH()^!k%YAXY}=ps&tvLK3eBsp z$F96LeCXKgyL18{PQ<(Ei?y+gSWHh*ruHjr<7g!B4C+J;6uY(V^hxUFY;?eM_Dw$= zbqU)0>b>0|Qi&^qLdvIo=UKdLtJ`%SS;W9?^ww8{wyn!>ShQ_ z*%Z|jDZ#*t%ZT|-pbQBC9kk&&iUaKYXQM4e2UjnFQDf@w$Ooi0p6P?!pt@T&&betX zwG?Mxvya6hsn++&)De`_2t~PW%69SPB4KEGIw1u;ft^dj{|IGuY(oG6l(mt1r_EM6 z*U5CBa@pEhFTCP3=$+Q>;@{j$b-6*wU-ep(#qY!$B3E+~cR0^TRxbTs2nzS7>!A+*@1bJGTAqE+P zJoN%^7;-@QpHMN&|BJ1Kw1DX8vIC)#hvjQLhb>uJ)@pql9*N~J2RV!>qj zMv?d4ez6m$g|B|CxKH}{^SVM;sp;Q@@Cp6SW#Nn4Kf)`}|0=uuyWZ2)!`~`*F0QMf zJ*xjkkbS6}TKMK1f||ipj=qmf8eWbg1n&Ef?>waP0UM*?F8=sxV@hVgckr#^aO64r zqV&ZDy9C7Lc~a$sw#XH*7a|Q1hb!xEQu2OzPXj>Rlqz10x?fzMZ{geDLYk!ujCLKV zp?QC$K%Goyl!+yAEe(eFnNU`j@QW`U%u2dDiRHzZ@gnkP>iZzx@#i&#vk(mb{kc3d z4O1ujPbaV*nebaySk5UcGA}m(d4F%<$DE5C2e<7{KBsixw;51DuTVW(U!;<}UUr#X zD_DGs1MxqaJkt~xMX1G&oE8GhN`j}(w7FJxD7a3KXf8vCARF;JmOR9jU9JnP6%b#G zC^xRzQt|W^Hm{8g^+ZEyHhOT z2g$4wb-_faD*+1c@;955O#_C-BtXZ7q6%{XOf)ytsjSg(?RLd;Y6`3tda-$XR`P+G z_fXx(Bs-DZ<&DVuF3VzG5>b6)1(S?=k2H5RS=jeBZvEhBOIxA)X$=GuBjV>KUdd2N@eT7?|gEW6P9Q zjIPW}%}ebgCV;(b0v9tC^>zwZEem^^rVhb~dv|4Dz0I9jQpKAG)Xnq`vc@EcXT$DN$!xrIzz7>|JjIk;S%uDjh&zR=ys>xya=dTt9t1+hST<=`O%VA&4EjL`uxz* zB%w7ZrJUCFNfL)MexmC}t>h$a{BQ|(=28D1-R2%SRG>F$%IL#kYDQ2roMm%A=trjq z#3}H0d6Bt{reQvu=K{4E+0H2Z4mf!nFU6ewbQgK@*!sOI@9e~wN{`XrxTWu(d>}m~^sxv!B%4n9>_4U#Z2LQ&x zyWbw?ySUPyIq1ZBr`3ml+iq=O*6i`r978 z`^`Hi4F@}1QQjLP3orKyf2EJZTO*?GYn@vCU%-UAj*WWJ+K!>-h*U}ei(>eH*gf7h zkN7baJf7ei9uP6dvCSXly7UK{2FIa6qvmiH&p(g1?}7`mX%qFkn4b^h0?o}zjf$jN z3wCt7e(@G9OCTZWuZLH9r|op#h%>P+1U39Aoou2kTJ&4dM29`K&;GAxFxPptcsCZu z-nY^(o&_?qM_wfRHY$AeK5eU2M@asIG+ZvQ;{iWoP*Re`A9sAjBe2&(;c_=j7fwdL zXev}6lqdoCrZwI%;EQdoGva|xR^G7Ivvboky#lLh?;!RT0CEgO$?n_ywZyYmO6r+a z7@YSk@8aJU1Xj;gA8?Z#vyh+c#rw-+)8C#AWiCCpoR=yGT^f$<9b98OEfRJvw}o_s zeoiWF(icnw9OZXLf0p%w9E>ClSjmFf`o76KwOu6Ku+L88CbJt%4EPy$hWTRxKJdkZiSIWltA$|{4`P_oixFHLXM(wYVF`1*SPC zIA7hLChxBRmjJ~;r_?JHZ&Jl~qwYbCW040QwS*a?(2ygGLq< zCZ)7_|4~HE9;`f*$WRB9zrr^Y5&7FRvzb9oo1?$3x*ll2{ zKnLN>jCy8Et!%W5YPYrgO2EB}CqctdLfp%z1ZIAHWS7(c9rBv zJ;HkX2cK`lo$V}(9%n%U1Z*c_MybZ_<%O1YV_;4r{XQ1>tnjt__7ZmK)A#X8Y+z== zVENC7`w3oSe@8lcctp5K*PM~RRyDRoE<7)}S?-n22J#%MxqHMPB=qePr>`@y1%OKbGv(m?VEEK~W zcb@C2J_GZFlce)xWy*%ci2MFX9A!G`pU819S=i4a3DfGg(e+Kwyx3TD2f4S(HXxW$ zd*lLVjs0Xj>cACs5~*o$HqK*mH4=o^cAyS9KjWje20QX-DC82w)V0K zCL&CsYG|t<8V`{bxt2$~M#{BURNmVfWk_;dm68}OmUwj3S^rn`yH(Tc;gBNXr^g`R z=W-dG$5n#TM_co82oY&}un>UK?O__^dJ$0OPzyf)d1?E&0EZMG7$^A_*Sp6V#A+g) zSEU~hy-Jw1xi2Rwm?kU)k~%cBC>SmZ0b8>0U_Cjqe8~Jje$KA!>I?r=keLIlHxzyP zj3yo7xr1Fe8gkm8Q@LR8=Bj#6vAw~srHLm$UR*o6ECRNkW4n98p!3Z=VIc@E$&Lh?-PJKWNL0ddx8K} zL+2EZACJfl0CSQDl$LG$z)WPUi$*sW6cS~dcx4`Qx8JmOajW;h_*F!#4#*Lc7+`P- zI{yg>lO&~F&Zbp=^g^Wc`G)4yxpvVou@6C^_O391Z2!v5jTZ8HJb6aHucM;n97K)| zP*Re&Hd9S257IJX9jY5PqP?UIupbARbM{t*%5T_DI3oE-#O6BJnfW-?Be&1yiCzOQ z3-k!JV(1@RStUFFJ~(oJ7y;Z>y-uARo>i^ONo)AmPqbV>X*91aS#isKiU)cUS(Ko2 zEWzyu%T)S{mbfc;sS-EKeh9vdEqoa=LggoTAaafV9vM+7HIp!W{wNu9ui3A>8q^?P zNxmKtd!)7l8##-?`VgkRP}u5TQ-gRSq{e$;HW&wBWUCFB}Hd#7p z*!|?jy#{z}SsSHY^4M4WOj|y@uX$TpRqlfFlA3IE>^8wdkEPCs`N@wF!-25gvx2FFLpQVOO89$MBe0w&Yn~|Vj}S01-5nR zFJ$mD$Ok7S_ouZ8 zksG+6^4I_5{0y+JTKxt@x~6wN1NW)PjL%sVlWY4-Q`Hn zyzfMZ9%XaCaTaR+#`*J+o;&xCdvJO!{WkjkiEkmdn~sfPW*OX8fOjHw@n2rO>uL~! zAuxD%rK1u0F6SJZ=f2ek4M4*ggMRN`uL|hh*ZI>qbJo373C$Ok6V}rd7N33Q2!)I5 zbYXibhb_Mb(?=K4cNY{`dE~Hgw~P(q&-m3W&N?-mmEU4zY{&8o4jIR;Hzo=}8z;z}*TC=)tk z*&yds(xrE|=-$ukRj&$}?TIboIeFS))Y6oLa{ZUguzc!? zIkpdKO-R24DWs3V`Tppp(wnBH+Iu%w=CIHE_j0!lnd7|-h6uQ9ugqlvb2D=q;yxSj zqk=HofqhdBjA8K2mqY&cLv$y|2r{O`>u^FO7eJZG=jT@d_&tNW1|GJC^TCYstn;i@ z!OIE}T#~ZI{8<}HFa4-gk*4S8xquMid0l(9Yup3~vAr1%n-$}Q_zKDf{)`5n9phz) z@z~gO-5l_2o#bVodp4db2sr>sOSw9Et{=Xg^Y0Ki83U=j+JZLbvCeeTb{A*_B|hLr z1O1D+;+P{Bo_uBoJE8+f)l7;6le=BDFZf0d{%{Yo}YPi@1mw^ zZ`BdgM@!vYfTDN|wnO|%ux1(3?tX^#$CPy0txE>?%CN5zLuHnqBb$Y?p;fCsD928- zv1dtg z0eJT4(-WgxjA{$_vr@7x`n@{j3NJJx6OtI>b0EVW#tX@Q|2{U;MDV?fcD|3< zwjV{=b-y;dj{Va_;&W-D7uKS3&@c0nG$=IIJeNrqlH4Ja_c5z^D**E~@^p{T0dK1D29QwuS>FrOu3e#dNH7dRucmi)z6t@}C|^VU?O!_IPA0Foj5 zXY?4&q2=O%XVLqj&<&O2J&kg?2&ZP?`y4?kX%*;zu4#{cdBiCW_JOO=3KVVktx)a8 zQ>u*kh7M##`y_Q_e!4Ca`xDS6VWhUhE61<9)EP$k{T_08qQ7}tH?u6O+nv-bc%#1s zh=Y^F9loVJPTyhO(qm)SV7hOpXm0rg77WVfdIxkc>qQZlsiMl^Ak#wdDl_iTbYInY zFAyR9Q`)kH%aZ=^}hqnBS@U?KovzHU&698VFuv{AK9R?Ky+ zi^ew8o1SO6^`f>$NF}@k;61a9731+n7%v;I&S-5sP;9oXZbq5k96{W_m&cZJqS*25 zy7Z8dIa}1#i24bDOCgfy(jq8W#h}6kW3`H$?|FBzs<)H1sVr>`dobsoFj$o$R;*~T zfY5O%wQJBS%rBmCHajLR@zFhnIiJ4*5gU02=Uar+O1+M}Z)Jb|gPW&9+H{5hc$aaz zw@QU_D)*8@A9Vb9>MRs{zfqVB9JTrLO(kNhE!YZ%Tdb*Gn1%yIVv^-Dh`g0|H_>vS zePGMr*xy@IP(!hIOn~lk#jzDGAb2od_id7`1_hV=S6j8#0bpMMJkK;q^ns4h@wA(t z2cU1U;>Hl7cqkdaw?pE@1M=fYk}fEP1a)l<#ZzGI-P*lkT(}Ug`067U2T#9wP-A!Q z2gm-;%2nO#ReeZL;l)1g-B?zp4=$rtp9U;#I(U9Cl+h?wBJFhX2MX(+5$43 zj+XKjLIqJrOwzZWoUbgF*~cc%LF=#rs_ndXUjKN*pN$>nSToyOYmLhZo0Px*Ri<=Rl3H^Rl0BC$;zS4d3H zFPk!Oe=|$m(K2+%XCT>&I-Cuk&*4xT7&+_NbSD%yiJir64HuK;} zpyYG)W;{@aBqR1PW@mxJm!8Bzuh)-5&uSFxE>(I;wfnCv!X8#}Hvl%$mKoQPE#18f zgBPdI9+(b7hfN9%71vmio~L3n4Lvr27lfz#$$p$?bo!Sn)7WRA-}|d}GcQ*lU#N|O zYz6j?ca1|*(>P3QO1(J{8FulHlFF0#-TRTgkrFCN&+!LHSCMOSrU?#{Psi4BcP%g?-GZBhRG?z`u5e#zern_&DxgNFJ@bLO{0YaPdZ+-k)B&5q|B%eRfU{$dO<6 z+8UAlT4aE}5|j}8&PQXBa_MGzjKq42E>pI&!C4R|l z&g{1AoyzvBMTPnEdnH5uf`3`%TJ1Rk1p?;!b8eA;|G|D9oT0)`{;7FAk8Ii-X9J@? zNR%t=mC)ZvCwaD$FI@iyS)z2aueRTTwBg2c>n_vR{l*^<)Xm ze%^Iefv+(BEzjdSrw={1Ij}3X>0(9ifvUFs8&ghNvHVG$eD?n9G$|T7SIthtUi*Gq zZ2qOT%Pp916REh5iBK6pMv}2^N0khht-@<+zsy5ikTaW^I*5XHpWoblDoaDX`SMeh zfBPl;&9K?JQYb<3lN^WA_+DE3Ik5ivu);&PhhdcLfxd7FOc3v%(%_vGKm7%&iP2oK zc|@K3${B_;K3VT?Nj<*-E$@0xuP}>%&BzsX$XNUhh-%D$PK=BGflDon+|E6kC`$jS zrrY1d_7E%ol~-d&z2e<;kZpAhjKOCazTOJoY8$j6_B{$E$W*-z2E5LY@p;)p|_?Q(DZ*fg{!FAuZ{=%t8jHDg2P#RZ_>5wQ0w zFY;P2Lvt*)lG)HT2Iw34T*q4{6m5Xu5=;Q-vz{(sN?%d6d9-Rm>W0jD=24aMW2iQP zzC>}YT_hi`{aG5CE8*!bw&%p5!DoMWKUE*`^o;+bZ#5jPn73;8_&Ty4g}95Z0!PDD z!od;w5A2KS70ghmxp^%KlI=|1Zyf+{_V;WxaskEtMp`j8SVGI z-r>5n@2A@{WVH-hi_gXW{dvdpn9QS=?vIG&4j5V@r93n0?tH))rU~!1t7aeFjWg@b z+okVmtTkTlDC2l6ar*S(uQ_ZUXLqVjv?6>`Ua{_D?R*TPVvtU^k9glt-x^^Kr?75d z-yi>r;*u7a1rsT~EO_MBAUOuxbleahxev@e-~z;i24*Gy9*Cjtb>W1d4qyzZWEy|+ z#!LJ$MR(qKZ3m3rird0|x~Hx&24&Si$>@deP485VA4E4LseX5ApZx{$p8vioakJ%v z^v~T3tYf&oMJhcyy!F!2u(wiM#=PxFRr;lsA<-)?Cnqv#v9b9hAsn#4f& z3OH{ayDff5(JC@{z~xb0ecrk*)aUX)CrK5Jv;%6P z)LdH4;~2xy8!q7>di~K$K9hg-s`NOWV`A4lV^)E4Oz<8XLW#?KQ57kQ{4JgBCx3w9 zs2XL~sz}toa$D>D-LDD@WAZZz7!j#jTsz|Bbzs=h9DAxX&%4OCL&k z{356^^x%}lDPxs0+fwTs#PwTt{W4LwO2GJj>Q~xPhD)4qJeF^d^goF51mp%SOEd7l zf^RbNk1z~8h4GuX4ADgPF~3UYK%Sl(71I&Y28*B}rd52Iw(pWU%wsYIh>z0RSY)UA zHq0v}Z^xN^Ftxn$`>eg1bN5@t3O*l`QmY-x=m!^T+VpH-zU7$~pKI(O2-XV~;MNthjmi){{b^lDM3aKKs5 z4S@Y5409V^?*k_;SBye=sJ6ys-BSD!>E{y5BjAxd{m4vBh_^ge2&F){4ebf`R9CFAl5!hXvV_t#Z&D_8Z;&t$o;tZ%7 zp@&#ul4*#4r>%cO)(OhZ@3|)S0&0UjJZSC3VPr7$aHa$lwn=S96M#ST?Zy+~c|uwmvL!{@#&r6&j}OutuP(B9TOITKfM)iLELdv5*T z_^NZXM#Y5%RE9d}Re!~cF?k{JTm!=P@qpz2jxW@wy#uQfOT`5kt{oTe5V;bJY|YQH zIR7AxY@|A8u**b0(2VjgSOh<}jVD-3J?0!t0^i$DzJYAr~= z`$D-%xGP5s5F;qI`(yTvR+_acwD_PU84TV^ixY3B#5BpW&=62DxtpCj=2|(iE-2wn z#qOjNn5*3XJ~rlLQH!4#I>in>x?15_3(^6+M)Zdj*mv!m~ z&T357?%PZkUd@2p=)YyXmtE6PZ2GSiuD&(SIF@|X?S%?^mEiufRnC*QUR{occ(GB6IHYYMz{B$OSwMMDnVf^bVg$Vw4@j+_7eUclpD6E(Rn=AIN-E^S>dh zQ(!ccDgMmdoON& zMMgrI3T%A9BzEn{4588kXA(5#%4FI|@Zzz!qabuzAR#|HVY1(aVO#hA{`DK0kci<~ zs4xOsR4YL-dIJ1s{gyM3Dmkc9_&A~lWn7$M#$VT&X=bEm0Dh6+{8+Z4%i7rNrTZn- z$S|q-)eoa%;ZCVx`-xY{tV0C`OWTsevb~zlJRblsME-EE8Cv_Z;Q;K^oFHQ|i@!jz zDvxc>!q68?K3GMzQLrx8P*JZbQvYK?MpJY*utLI{pe=aKarWo>&1$G#gE&pD&0p+| zxUeF`50VO81xd>@C-Q!B-(Iu7*sn}mElryqL$Mw)$1bgh6-zZ$jt63KMceA{{h2=t4++L7{Fa{g7qr^-7?zOf^ZwszGeB$oC@b|xM{ZMbD}B4jgM2JE=n5|;0h zk??C6`Wy{pbCST1Pv|UHQmAFOzH9a+iY61qSW35rrMo4+C9Wwg0iIWBsTncADGq2|GreFWibHICqXz$#u2(+b8|9Vm zNl>xKbo}A%t!DX=?2!i)#2D`Or8mc4M*GwDNacW_`9sRZ9ppt_kv9;nyiV#1rY;$_ zfspHC_!jQ#sFQUqu8`BNUsxH`kY!d|*`hq6j=^YO$rL<8}zKpf(F^e z3csgEKX056O8%5EF5)Kr1ntQbAM>KKs_%VYcEoQ-7b-RIR?M(a8miJoR$N?U_;Nk za{dgyH&2z~@jRHHA&O%%`6`OJsG%UGZsFyWhEN%jz9g8*GdhB3Ka@$6mu+C9Z;5)Y zog{V4*lM`T(p`y=eRd2JY|?P#m)bpw%$m3aW$=5y#-j!3_a-pLtp2Xi?T7Hb(gr&I zCC?UuZb{POQWnJRv1GZ6T;i;lqnZQT*ugjoYtx)%88C*xqvgjiLBSG5sg$^fbZs`* zsc`{#O)Qsa?HlT(UqcLi+!*VKyaO z)BQ=fP(7Y51ALBq_%y|?)a=dwu+lUjRmeG$aqIP#ed8Q*9=+~V^&fw+cVbSAy9!zY zQz_^TY&HDB|N91ZYm#FFqRoybPeLob&XlHGIZvi{X@n$W^C`>yUnAw132%SJ&5R@C z9U_>g^JxHh6kkx<}5nDzL|DRF%NJYf0nvhF=FXoCKwlBWmfmnABj?OmH(RibzB=o9H=1FQ|`Ow zTlg*Rk7DvA5j7EcYQgNqxQkWz^$a_qyt5U&Rl{t2q~=6dgfD1XEGf8`U{UwgP8|0y zR1H&}l3$Uev6CGru3p!}%uU`c_0)t9{(Am?>9h@tt@PUGk+QePB~tcG{l7W&D(Z_e zYpfX?*zlh#{Jjk=VpPj(piPBtYd{Nx~&JRF&$MybF@8KypWAMGD&wHZ<>c0@!S z91&{E$)3{YDD(?C{bu$)-@hoTNv`vYAhitmC_TD&>6L@)|4ap|7kUW0$N<*gGnZY8| zIrduZ2fOZ{0aV@)$;|?yDl}0 zOJ?JzP`);lnUD5MRQdS4MUz)etGU?kg{Gyg3alCzdTRP`Xp!06wAGv*pqX8_#JNi4 zGs6V( z+kdyfJAt9heApiUCEsqJUe|P+!+Q~k5)BjnZ;TAvr{##(rIIpPuf4I{%i?{Ln z6$;Q%dzwDC>b`h=lgbqy{f34-4dwIMmiA!c z&C=pjep4`uZcsM%xM^UP+KFwT>j1%1Ea4A^qqH3GJ&GgcxnQT^7zVp10ScH8YZMN? zH4&Rs5870(1it@8B!ff6R2i?Fm!*!v0kWqcN=XR2--tKl(kYXVcI%oP3 zL6z1yB{U$XX!|(bc2YrpKlgTd-Cp4~{cG+d=jG}hVC4^MXEk!1;s}?quRi!4y*N%` zb%j~P_01TFT$s8`jh}IQPt|H0l?Q`vf`hV0LNt^TXhl$9+pO5KEIv~wb zU|fl!J1LelKFsrxy#t@Z9j&&s3?JFW_t|>?k!ja%d))fZ z@|Dgmix5rzRW<-l42r*ovf&`I4$kJkfX*QVyhr!_n|#}DyznyEG&>7sPp!%ee=_=( z<-9)>-I=?`>C`tZ9!4rC5GdQ@pr4!&Dv@Q4e-D2&I@0^~lYKx?<{x#KxTQ7Iz(h{1wSz zK?Vvqz_i2!A-6|s%xearG~MO=JO6g-ua4oB~5eCJd?nn==sLqm5N3cUuV_sw%yK;rwe5AJT5<0 z7U$tCmyA(G9I>srr?|S=q}<2dTkP>gc-eSn9r5EbcqHujExC52+OgaEV^9B`bbNg5 z!`~Yvw?>9MgW;&xW?t0*c#K)P)ptOi@p<*VCxt2?$B)~s{oo4l=hiPgb3rs{{6IZ6 zr|l+rnqD@jsb<+@A-ep z4%(^sJpq$x)3s3X_z97;cx~b`MjI~ik!@6xm+=%{M zynqwT)6v?#Ltl4GENrbT;QmbUViS|d?HyU2ELV}w2UMb%uAi17r4*Ai_MX~7C%-=k z8T7QE8ooc+12v2A>i{`T=j$}H|8Mr7lE4Q=_LRThUB%N^GeS>4#~w^W{*;dx`Ck@6 zEsNfk=tNcl>J)T+cTI!OY)qd&b>CXdlL$8YQsA5^AZ4QI>_ z4c&Egn6lY{CRJ(rRH2ytAub~e(e^81#wn+wqjhwO%4*JFC2=!>d+f0ne}3uk?bncb7}+N@ z%%#vC5HqDJ%Dph~u-~?)O4*8#N|VQm-_5d?y>Rlgb_B3|z3Q+0LbEW8m|2TFWXWx5 zFNQHv@k#FqY=pyWb{cl>9)zP>QT1XoTdLccfbk-8G&mIdjjtpI6=BW?4K zu1G&f7zhRJ4XAN4lyKRJQDxs&Zldd!%w?#7+g{6bM`nx(xp_wn+C$Nj3f3p~WV=M? zT)$R2+Ee$Y+oRopNQ92--+b5cj@-}ryA7h-EM6q_deH`G!h2kIOpVj>vVi?~G-~@@ zMUyxAEbmzaem`h`e}NXCQoh&)ZGBm>ZsLJD_jz1T1D8KqO7ebIhNF}i$?0dQ9QY%^ zPDnsxarWvBI3q(#S#AahpJa00E~G^74Luetp8db5d+(qq!mr;GBuYj=P{cukps0XE zK^@5ml2M`zQ9wX4D5(jGB*}_O7|B6$&O?q$R-%ApM8XgUW~Tc#{@%Us)>duZx>Z}d zwbg$>6%Rd~rl0ei^ZkAftcb6AUuE%r|JC?@ove;1PfyCM#CT1YW^ zT3H|AMS5&(<`59&srcNFsm?E_EJk!>PitD9qt2Hd?M1p0BlhX&?j6i6RxBbmgdipz z#Rg+98dCUl=foiuh((sHldiXXoe@Q1y&uaU!C%TKIthQwRshjCU@v)j27RA{7JKyD zZR>=VRJyqCeH@xC>-_YCLV>D1qcu{EE zXLj{Rmbr{JYpT%|X32#d1A&Qe$DSWXsx$|W+2|LxQ@G*p;6FhL(+{b(J8E_0%n52u zk_{}wV{E=!UFb7U-#%0eWez;Y-v!StwNQv$UQnW&D9`C2?{>G&*#t26!MH^RJflCB zY|(zr_fP7J6rrwqr+Z6tEK9fK`*XE`TFdP^+#+MK3d+}wi0`96|5EQKDpdOt+aU31 z8q>7KFRUr$K*9C((kU8qv)Voy`hgJ>ep?)o^Y*e=*bfaEMz}qT?VQ)a5AY0h;!q;> z^d3vjb0x=u7dnjQc>NhA%KJn<_2fi$-^eeBka>*5BUa<&F8wG6OZ2k z%RC>Z52iD)Cn2SAXD|1^flL9pK!O0on|G5FLg0m`AnPdrh$|5iWOyVMd7<*J9?kXA z<<#Vl<&i&*)}NUHW=KY9#eNALuDQdL9B=W;JjY%_{kx;#MLKB=*&_l473_plfyfvi zLb8JjDsZSecXI-+yO3_T5tFX8oYq==WIKpbNAP?{oz#N|JkQPm&PBwUw|onj2e^%U z2}WKH_`en2_>d&b-RJQ|wovA-D0ref9mAnf0ehootd4%^5~DpgEU(W2Tkuj#1N!CC zPJdg=0m5@Z4NdbD0@a%}b7uSX(4F((FVeJh)VEbaDSUjy3Vfco@6uYL(E_lwA{ z5`w(!Y#yS+R|n>N-2)Uz%wcURR!gu4IfWoFxLtXRRj8GNGAVCAP#||B5xSXqCsIj! zTBGnRQR`H{s{T*EstjIxKqjupO;tK_`*VdMdDX8DSXX@ZO$O^k95Cto%oB3*J*zEl958_Gfe!2Q|S;AT3buU(YhYEqqO;mR&Q?rwqUYvU@h&UJ>Vs)cdP z_Xu2l0>jO@oDv*h0Nx;BKq$YLk7rS3?aBESf#kF%;!FfCkhD*Qakd~% zK7PZed#mbHnX(L^;mgqM7;->m-GPF|$-lGCJTq4Ke@gspgn)AP$Q(I>a6+bJ;;h5I z)BwVG++Aex>bLE=9+hnltwG5Tje}o=zWP`_P4LaksW`dDAZwbVlo*>!XLY3}V7C6w zp2<-PA8iUMP$x{5M~8B&hyHT@U}gA=so4;mfp!+%PJwi*k`<|}?0&IQB#u_UPK(p(`dTc&#=0x~ zJ-mc441lsU`*AE{D@p|qjJMh;q0pdQt$6Y z?ttdOtiJsGQYNa^tE$|bJ8+}dk=W~Bq`=1{lGLI*$Vw@QL z=E*j*y9g+814(}wg`;jgzxlIG~nY>4HCiURRoMh@BlcknwURusNXlHge*n~1R zXGNXdh+EUMgq51=Q$niD zM+qy;7f<1S>67xS$h#!>dN&*U2vgUT%Q6y0&E6qjUDh44z^2cVG2DMC(w2bUUpmAC zJ8l=O?iMH%C8ayLGpeql5k)fBw~^Lw&ct32^6+3aXg5@3b4ctS#NaK>=@yU(D2M@m z3B&Rbu$!n>!vKr5=X_;AJ97nCOznY`cy$Vn_^oJp_U2x>GlLZ`4-e|^&|cRt|5q34 zwoqwJQwF# z!EB!-rg!B9a!^8#S`dhar$q%wcIWE~_lWusq-JVKh~v1P$g%SpJI<8X5INjsb@_EI zq&mn$v4uR%(TCt;_gz1Q)k|Z6d`B1TFUv67oezaEW@^rY=Sb3GVWr-1QqP zgbPaG-sbLhfI+H>NyXyl!%u?&IsODIU~WJbGTDp}DsgP+9~?w@O^{6~EamP4_`1P& z&I|a@G24*s^7PipJ9Xcf60)xMmOg_UKQbv+^){AIGL0}e{bTPPQmTy1&fNZ6n6YO+ z?7v=CR!U1|TF(^y7v>1P5D{sB!}3ua?i`565my&m3FP!FikaG)w56K=_)D`WN}Co}jwW17~Ae zm}PHT5jxMGkv-qp1M9M1$Wx~F`^IQn6GT-{%}4bm7KF1}ZKomwqkY(N|@+f>^N`*L`KrZAWDsR61&mwRg9B^wXhJZGIT_-ZEur|AHnZY`%g3=xTV0lwe#yK#|itl6NQxTdj~kogJkEO*T?|WY&i(OU4_(M2rQxMgGHn`xL}p5z)YP^ zFqkxmoR?s>dCd_i2wvB4eYQ1w0jifedYMaI=jhsnA!rDCN*#RMeQM9%dwisRuFU^= zYw}YwuCLR@2{dug#+p zhng8b_vBA?>MG?{P56?JW1*wA8PYd#hU-PwL%Or=OwbS;Xu&}jcAUGkgx>GhDt8}o zbv_wBMXYEhub$wk10q~AruMd0Q1b^UwLP-OTgaf-I&lP+cy`(i$()sEOz)pep%b3@ zgYb2aDwyUV0OC8KH5Pz-h1_#)v?DhU4OvS!X zz2>YVTX5En1C|hrXy14B=r7Q)p%V1j($L4@dm(FMpGdi@VebG(RaD`>Mz^TUJ?PWJR-#)#xk^9HtS}SJm=D@Dr zhS$}5W$qexhcI&>J9Xz9j31Gj zOUp+INk2%NzQMxX*F58zuLpIU?a9MYT)GhR#H!}Or~wFgj9A!QzA%1b z7@TtxUW#Ew{fw`<2P|lTbw$~qPLqtM`m2sBB4C@)%51tqc|IP&6@h2uXFEcLo;gC# z0PfIt;*sDtH~0s1(Iz4c47vqRW2Qb9c~>p=HA=(4(JX|iFbbX zZgrf&``0Vm23&mP=9~bCW!W4)HNpS29enZCpd?xkd%QX0N3#AzQ{N{04;!-@)n>)- zRkYWuKa~By$u7I?v~KbXzk28k<6-bb3ZkM9LW8U7zEfWc$>3WgLG zdF>6J1&sYRV$5_VbZZvwDYL7!HuCOf-KbI}yr!ApHsy$?C{|9@wEuJ$%4{BCL>?8+ zeuw3k7-7FoM~A-v>=eM~GVFVJh#sIX>(*olyEqL$5g_q+%!Bl1OFX1afaUx??Y|se zC`OhuTdQ*f-`hkV9EaQEl&qGQ;9b@*T4OO>e8y$ zRX?o6W(#1YJYtyu7vA1;L#L|w>|i{bMk{N==Y$KYijHk*SFi15*+X4lP_uTldJ(^VCit)p>53lSYuz~H_{HogV zK0)i==Gb9@K~sIv3?=T5=eO$qlG;Z9LwDO6_krUiuiu^Q3|+8={pr-ub6@c;Ft;EvTdZ0={5;o|zeo&&A~hdS5}=<{S@#B4A5}iXm*N zSR00@%+kKE0*!A7nZ#=>GsWUJpD03Bn$H5mxUJjLGy|bHt?Ca%gQqrAP}T&dsplwZ zTT^v9_zwdqLEOyPa%n&Q;!$jQL2(IoGtX2zC_;TFWx9~&{_hXpVm2%2m)qC>rl4Wp zTP5;{B7OvFnU>+BU53_y3s97fJAD-F;!Ogjm_9mOo`HCHTp?l8iEnV2{4YF~GcOJ~ zbqX-XG`GiE|saZnMlMb%*FV zeb;ozD&|$yI4Sw%T2y0#LOjr;tJ)iAFl^{*O!c0+^Jvxt!bt!l1NZTN{f2jTkuP7# zLcz3CQq9l-~l)AaA12&4jf zyJA+&Z{I8JE6}Uat20X7`ueuETD~3m_5P3Omue1^6SJ4Sj{VFltXR z17B?JC7OnO%lcgV^4`0l_-6yCIEF?zIb%>b@||KP(9>QMULaf;dhpQVZ3VxyUOpMc zcO^}@?%Iz-9X*CE5V^*08lwIvRu)phdbYtp$}Sp>l$))e)KbS>V59bPdy)t+H)A1@ zRML2~{PL;nnCv2lkhp6#x$SjN-S+ouKtVQX9^5zR5?0Y)Sz`JYZXYq(Hde9A-(F|0 z(eE<^ZqwtT;h2E27j?@k)lbZ#N~+jbHoAdl&vKr6|MGNFZq}&ozTD6|mx!|I_rz;C zJNs_c+2XM=fMfsRUN`86H#j~#DQrYPog;Q{WB%G~em@aAJo#>C^au8m&~ARTijMO< zItKRP;vpOwsgEX|I4m_P@-zMzT17wZ9=!M1@#Awytirpu4ke5h{AAy-)vN-)XO^|^ z*m9Ub5_6cGr#q?Jl?YScB0na}Bsc-rPzCd*9+~ZKyyccr={Gk^wD>~^c7LsZr=AI|O8HhUZ#q z5emXyxmZR4FIetXx4PHOTnL7z!cnNC<>ACx<6xyRy#qJ4@t?(_9~N%k8`q*)x@t@{ z>RJ@3=H#KZq4@Jhcadchu2wWYW?ZOARPEoF+5)Cd9I?Z;Wir@*7^6(3kWOhFnHq(m zQF!wP&ao+9jaK<`n^K_i)Y1}(FA-tzyxItfQqNrf)^HmX6ng$ebZ$K*f1xb$;MA8Q z8QLKnHwvD)G&NU#h#x7-~q+cmh=O_1%qLqGUcYr|F$LsLqc zS_Y-Qn67L(YOBLHdcFJb8ogUl#=78tIGMYSIk;TqpmTB1xpmy57wQA}jV&U$*8Tl`F#kCJ*w2&nQ)gIfjS zong8`$AClq0;xrQ3`NOB8?jME^a_`}^_bx>UYff%h{UH9ENsg0joWH}Y@6E&uevP$ zlV&R>;+v|uCviXSQvE~-#Zh1IH|IS%mI20E_%@%ev`?e{e9?HU!SFn2^_N{nMbZZv zZ?FzOEtb!;yFk4}TO7wMlnuQ0HYc2Y!D@ZFQX8~joXQ}#+mf^;ym|AAF#I;6d6LXy z2D*Muy%U<4LknY1a4Y>tQ&JF`Ln7aIAQ9py6kgP)TEXDp))^J$T9A1HZLwfRmubz>ucJ znxmjyTC?}z(a(lV!5w0_>n#y=&KKS!~rd-QUvhoiYUx5{gop=!g{SaSdeT@q z=fo^Wbih{Aqt0{UVFTu&GP5h+VlMobb>VkRfeCC_g)yA>bWgqXmRgxndC{j@I_eC!dn>Ma;hor6EXP}@rin|#FwUPF8ythB5d!976J9utXpgdY|nZ`$-Dg{ za=dGxkKyR(Ea-xN9LE}D$ zRmRoH^PPyPIl`eqYYy>r72tC>Uf)9a2yq|6pjQWors~rYuKibUL#O`F9kEkIHM3J&pX>C_ z->1`vvhR#w+g!(B_hD=!Q0`Q->Gkv;_xc%I)=e$u$#eJ(O@``v!M|Gf=Q7bjc~pL3 z{k32O*|D>rqa0zK&qSYq;sFsK+q_o0fQ4-^3XgEH(dj@;p`C~w>Vd}J*$KhD+{VJ{i+;69tdxv(gs(SQr4#0T*wTgBc^!e#r z4f8M$HsjTW?QQ#N4v(hXg@bbNI$d+YO4sVY8U|wxid`c%N*LrBp6-;dW!GB6tj*+& z62YF<0|6*6ZupdrGJ7sdjJW2LrwRts$ExKe(+F4f4tx0FM=X<=-+cQU;!Yf-c^%mTZhJN6f# z37drK5Y_~fT`aIURkaBNE6WJJI`;vO?$@g^y8Xl$eBl-*GSik|B8dfl28@{0wH&-v z5B12eb{jzCF+>0VcKJx`mh5Td^Oo|Vvq(UWukl_+BCdZ-=)={A5seDX z(#Haf`Ae01K3Rc#sLSKR2EDMAft}?lG|JgS=15YZPeg2@LAXc5%`_|;k8b81`fh&7 zsMKc{4t^tW;r_zAopsQAmZZqujkva`8N~9NRU6xM(|k7dh0I4M2aANww%zgy5$17c zM0=0Yd%bE22}$pKl&&dNW_w3inKT5>s$3QaX_JFGHQs*%y__ zkTwp!ha%TR$pfy#C>WY}5B*Ce58fTZ1x*HSiUx%(wct~7fe7WpbC1;Mg3vTDg(jGf zi>+HoJNYGkZyT}xcwx<;5B~POXX`r{h{)~)PB1&xB2e>4`4SvgXyJ2V@t`e10j5Ey zJc{*OID*vOp^!N}>Vbudv-PlW#w+2l657{B>A@x%DY><`rf>`h+mr5}n{sHj*&C1Z z`E;t;*V+s^T;(2uJhr883_~(fe4a^Hc32-(K1qwW(y}VZM=FjLMT06j>5wba8qg&f z>{C`%najf>F@>^)T_Ss-paDIZr1-#l;V5JT&0-IxW31TCej1^XDTdnc&X z0ls6@uVyXSV)JCnq0S|h!01AAHG*jyIK$q`BYpcJVVry7o}pRaOqE$E0Ub}_>B#pq zbX*JShEM+o+XxI5Vd9L`Vq!i*Ze3yJ+5%E|BP}2)pSTkAlW1OlBN+IwE^+1M+Q&!z zCs&bFEe~ep$*9?|+L$OEm2KDXf5Zi0m zUBm7YUogM^UBTSfDSlH6y1M4p3>r?`G=Pl(yD&#|V{bz|oPQJQ0d|2~ki=R7R4*DE za)^0-1ndK3P*SoFoNhS5_{GK0_DFc5uG-J+d$1#(eEIAqCUA{GXj;Ha^rG<&Nf$nE zOi)iIQ-!&6f)fd#=`%gUWBQM+uwj_px>)+CYdpJy-~QXPBAoBHKAW=^Is{u*BGw@C z%rlpngkW2X>wgdVf6>-*`fMLfe7QzM2bu?@TP>8C+aDtf#fb=bL)CbmAiIO{a|7Wv zi%>t+L@k(52k!FBJvF?=eb;z$3WljK(eLa?G&{b`88Y(F5jXuuU~ldfNt60Q+4Prt zxNBP1inzCWG3?f=mN*^V!uF$Y7>MLPtQ4RdTJs_!%OKv74A%~p=!UX8!*9*I|FlU3 z8Krr0;ZSNMYedrrHbdW}I2=ipAZoQLUV7nO$Ft2w`slh&E=2L{f0EdxwvmpE33Lx_ zH6T3^=JxelDoaiBP^LNg<$cUNof{RU#C&4lFm$UFNMVe8+kP=xC4$F)D?QN_CmqE@$-Ny-kutM6O?cUwRuiq z&hVVq75QX`PZDZvFnN7%3R6_J8*uI~%4{hlsw~((%aw~PSbpWs&->4vrp)5ILK)iC zGMUZ($dFMd_!+evi)O!jM*~Ck`b7UpY7`(&xk#)^R4W5{-vhn^F7!ztrGJL4!8uaN zyVx7#?P|1(n7}KLkm(F0Z7<{bATd_4swM&8qOFn5S|lEkJk&qcQq912u?VC#vJJO5 zTZz|~?mHbN2726JdUY4wffKuTk5^fHJaJ?D>qC%}p%Ms5pZ2R_v!xk%SP9X0BaQru zq1$FaokuYewvrmAfiVPB^&w;1D5Q}ERiJ;OtCYqN)2+|{G-(zJe&lk>A(haJi(m}A z;rmfGapkwFS;!+Dle}Zs6Mx5hQUi>4ukXe(pR%)apOh}c1r!z6zaZQn1ST)kOp`4- zA2G}TAHP<@1=3e#*++;d%s~DNlmBSAN~yX~M-}%xfwGi^7$T42tE*YmH*z=LN(E}{ zoYa9!7-Ld2T(Z6ybS3)xE2tIENLA8(eiqmP$z&OpJlVJW;1KRowHF}{RS--zS}_@Q zgjv(Xdz;NAZzKOIH4}||Gj&mCQX*C!KezGAm|RFxJt0w$xqhKgfibpuBD;7oIl4eO zU9&lSPw3?x8Tc+Op$)Z-66EZ&1&8;KENl)eFCSRp`2mNbUcGDir z%>|U>nRxRhv_e@G5fVou1q2*|z>-WkNZ4=)=BIppI3cNaBZEqWWo0X35J{d)u0|Vq z1fm>$9J)~hDBhw`rP1qEyLZ8Hvn5}4DU7)MdYp~W*l$Z0Hlt*R=Q4Wh+$<+BO?a&O zZSMK&qMSZ!leUvIoKjF=bPBSCADln;Z&82YMA!S=iz}rpr$yJLV7Abdvd>q4u6+J{ z8~m^-c!bz8?)W6F@EY+j#jdZ=y+<+Q?R~S-=={|7kGk9HU>wlPI)$2`-DGTj4XRv= zJ0B0S&YqWcg{TzCdB|+SE-Y#d^_05)w39iDH5hc=8ZK#_QtagS(ut?TyAa9Fk>y)S zo!vm1sheyYZkoCmL*nnm}+^UW}e8taD5w- zc+UPxE93V|SLQvO=Kzi4L7x4O!nq$;E#LgUW1_+0;I5&u4%I|MN9dmhmnq!;xLLSL zsa`M4E6fPU5PkZg$O-Y?U*<)+-|tq|lJH_yAI57`b&_5Io3MBoP*OlD3c%>Y0O`dZ zrU%I@1ga7EzaKeAD`W2;8{cgrP%b%5=|mh6Aj%8M}z9k|7RYGuXw?>Jcry)8u6 zC-~XyehKp=^**r}Z~*VGL}ApJ-rc;6%4#91C|M07p_){k!t|6{t|%aZ1tR1gcuwwT zzcNl2&H}BKP1UIRI5+GJoY;;F{Se4$6`#Y0mmq0PM9226#{M(k5Vje%ql4`ZpoYi* z!meY1321L0;)YfxzU53IH%nfs!%>q&+Au`R5QS`oK^*@P%O+xzu->dOu0)#sNZ z>?*bw1q{#Tvpu4U82JvxUm(vYC)wLD`k;qMAz3wuB0C3{@IGY+GM(PAwl@ok8rKg- zj0puohmt3AXP&lR+ac4==)mQwk}{nl0NdVJDqel>V}H|YIil$utU)z^RouOiB-?jKOhD3!wq$ta*Zo6EPga z{QxXV0&vfGQltnJrtC{;w6Sjcl3hrZj^X@#3-T{Ln1kIx5h3o2*1A+4Y<1(ym}5!s9zTm>6d{2{!gkqFYn27HH29dlooH{4T^vr7(5HG zZ03SC1?984t5Gmp6>D48g)q7j`w^TYa_*=)vs0WviXEGuLEr(Kv~0gbIz+{{4@Lff zXf@;(;P|A|IbWFF?IJSi*8y*u#?}Ge_h{>>%-e&hU6WmHCZlUPdP5;qtf65Ea=Rqa7`^PfFqE>BmQ^{10Z zn@#`b`0K+{y<@5De*~4#{|AD}|1HI)dJ^Df;~EDSRld{l85M5~ZU{i{!2U5P?lfCk zLM;NyGP-Fs-8cm5cLIKy0)=pMGlA)&PIUmnh>y>qmaGmepB_lQ5R5mjn^c#$`SCvx zQp0}`(sItLa>evsj2dNP;(P?f$*%<&794He1d z?(v-+Rr(V9`Q!Um{-tNWvobIcHT5mLZ(hnnQISi=akfHT^3e>pC$;$`JT6PZ091;3 z>OiA!@)7lXV=!`d0YM!LD5$B@FI|vy3?&|uNM7oa*Mw)QQ21Ev&gq(n;N2r`KEdy8 zz|fKL11i~9L}r>BCif2q4-ClHpbrVeX?4F2Z!wZ$D60qqtMSmX*CN+Cf7_UkJ|1&J ztzwEtPeG#l|0j^BmN~4&qdM1t5##t63(z`1mB=}om2R*vCk$pVa8{gZA#JZ5q4qIW zw2L64FBh!IgXyCG1rxnKc5y$j+oP^majZ#2I0g;u7juMQsZ>Qk4@y#aYCheLDn7q( zf@G}*OBy^^?u6%=I17(LLh=+PH!@mqRD2vzJq0OC7xHvFNDpv)hszD3KdhBlSv|2j zC@X1s&Z-%f=R0WD1uB>=^$&hG!ocj%(5FLx+uxdf+bj>@Nj(?)_}#faYmTVO($nQU z%tc+BTIxFo><)&N4IR!+uJBKxR9?do5zD{(ByoKRa)WXeT?W^XM<2{amd?v5EyaoN zN6E%2*~~^^Yuejz$=~A7d{o5ih5iG1zG7j_fGzey(s!hT9>RdpH9+RZKGNPucjxvI zav$^jN-Jf5<6ke>;(l2lez9D;>*{vHkMK!n3kF@4Sjd5Q`0ao`ni~4Wxdw^HTavAUY+4co)eaEn|0M&pPuStUgRYkc%sPeyd!D{ zbUvu=fO$Sq0=i7#u-N|cE=*zq~{uoxCiI3;$ z%SR$4;jBzT4~gy+S~SO*`gA2C!jMZz+-71tL3Mm`;R+L7v3UH!th)PZ7owd%jx?JC zNEC4+VLE;02PkYP7YCVQ_*sCl3j-~@hYoy@?D-|+CQx^RG;DQi3aIkG_mXHHyjW(v zfHq?|g39vz-(7&m&@gNQVN{zaY33(Z7tk2Ki_z+GR1-6igXkRWjm*Xu-f@6<+4m3Z zeeb6pqe@h#PqI~e6C13x>C1?Aa&*Ss?oSP-5|3Go?j^Lo&b+=Vl^2G5nK47JOP;j2 zt>JhM>P2#sn09|u_6_Sr&MSg2yQ6x+vkNo9-6EVzyZ$mi&%vJF>xU=hbP&@hL;_ou zU_5Zq%TzGe_k-ERDY&f$(RrD2k#V7KA~!F)huN|VH7~v|&}`ruy9PAa9E>mtXX`sn ze<9}c9OuTKeYS)MYbT;O-eBe9zrO^(rtFe6UW}PcuBwziW-r-?X`ye0mk&6NE&#sP zo7p$RFcHsd-*nnTOPC@U)nzmZiAU)cf8Hp$-UlMTk;c@bmFz^`$!VG z;i1eOo%QurT z7G17?=zYtYpyS8^w8W8muL0j(=n}4oJbeL|(X-D39(7ok=nY}>Sm@b$TB3jcSvyGm zJLqrJFEjWX+TrEu-#+=O&JERpB22RYX*eCnmU7|Lv}q4s_>EF2HB2VG?sa$l;mh3unhJ?-kqKSsQIarT3*7*k$9e(v^!Qp8@ z#iL3>o7j(FFPg>2H~pqs`{HlR5CYO9-_YMm_$TT*E#=I=SO@mnEf~%&GG_Qt_RmMa zl+MM1y5WJ0R|^4Z<7%}_D>Ku`TME8j(t2lA>~dggL`{#yZy2rGg+ z+&k?~O@G~y2c)+3*=j&^CB=cbhp_tAG(L-ueT+{7qHA#ia5s-7m~iqXWI@Xs*G9$e zI+ePrWkGc#DC=-8R6dtgl;TEEw1TxjC&F^HRn4w&t6-jOpz?L=>|dn+JbfrIZNZpN z?5bWRI#YlMYRk@c@Ax0jt%ZKFLCWxi4^GRmj4Aj=Wn17J*O5H=580qJKf> zSwBbFp=Tb-1r3T4P1iYa)xqwpIitXx7md_LAzOiAdtuKEkzm)y)b7FVn0Bv>7 zoaxIADxb*bYpek2*x@1a7v?8`*oJP8aEFTV*h97fNzNw`Kym`mpQfKWEa&iA*S6Lm z-81s|Ou)Q~T^Zr$O$`;uq60mnqW;=JmEfy5e9@M(gNl|&=twzZeCab%n+aTi+tVt@ zha-$?xYKcBJh=-nOK^Z@&@*6nCW~Mdu;>yf)|+e}7p+tTH>fUwH|J0a@6gLUcU1+U z>rg!N6krE2H{5IjkMict$$5OBPNwx^vNh|Ol=F1_zDHmT|8p!OQy-M_WEZ&@yuf(Z z(!TfpN=3VU(Pd+{#m!gsW!9{7kiTqPtm?1XQ_wyA$-QOWl9WT6zX}d#c_mW-E=r@1ko3v0u zMU(QJDX0+l6(?(_GbuU|U$m#jF;VtucTjgOd9kk(IL0iX7f_+WI^5DY<}1z{9mr+q zb4Rrfp&NaZAH6USk7fcU^+o;n>~c557~ztpbWVZ z$FVf=h8!Oz(uaxuC9{eEG*?@rX>)Ome5_DIW;{oeK4CD=R!NO3`RUoxIdSaZEXHgd ztOUbh*wgO{F@~0a(L%skeSOsd*U{N53iOXxY^uhw7E$t8>u~@E}p~E5rj1l8V zfCb#SPlnUIUi8K2d5(89@_r~;A3?=;|B~BbHBYu^DPtgbhvZL#EU+C3Vx7O`A5)zV zuV=e<79!OwRVAnkl*@zcDI(FaSARLni``TB1lh%;DGba$$r7hrQC&a<+r7cD7cxQh zjGmZu%-Fd;f|Q>oC{?PuU#R*JYa7V&NSB}i?*uC)oetkZ&PN=sVjBrO?u=2&hC1K{ zL+hVqJlQn*5Atyi(tI)znAY&B2#jB>Fc@dCi(yV-1p)&Xgdb4f5(KX@@IKmaVyT(Mk#H_hipElaHR|h5CviOZ4_=0c=Twk*AP1 zvrG;_3v26W)AyC8!;QC_mFufOLSf*C$6*NySc8Atct{70h5WHVTlVivqVr@btIjdi zfSASGAMb%*u9eUbNUNBKZ3~YR*hE%AGm~HfvZ#b0`(5M*djAB574AA`D>BxaNI+BU z!zaWDXaPe2a3Rh`BeK;6koPLp!NH08vDPuBUU4vb7otBpnQo4Y3ACYlLbu?tocHDp zmj&l`yPJf9-ryxMm1?uSkZ2hv37}2(i>qq5#k>$9C zs8^^H@x|hSM(f!yQx=+nWneGsqUSNHoILiyr6Fl_&y|uM=n`coqQzYCQ#`NW!e?#m zBAnQ^erEH1>{}#P_~UPn+xX=!C9Q1qO6*4v<}6|%GD_aQ#7lC_2-eE6Gn z%pyr5E>e`SL#4QYm}Gu!QH0YMyd)H1!4K&XNO@@tSb=t>MBuhQw{Fx&U8QBjgDirA z7rBL@kSITEG+Sq>E~o>Zj!OjE{ukpWn~%G3LC({6nE!ipAUAK0KK!IGIkgh3`kVN` zNGLjxfWm5Fk-e8xtpS(M$NT~eM?u+j+TTPf1c5vy(BZB*!>UQkE8CAJh z8x+07A;8S1JWy1=D_mL@vQw)%qpjPji)Y+KQX&QK$QEW0on;cp6gXFR4d-JF7y3$yP}`3xLkQIs#1-FqoU z)rv{&GwzY560v9O-%>btZG+l#hAry5(NES7s#1``Mtj}6)R6TjoD_k6EEb}_%L@Fw z1qy*a^N-=%h!yFVdXoq!H+q^^Lj9@XWUWMZ@|c~}a^|DZ7h0b7Hb58Ghbf5v`7ZJp zb-nuMxDS=g&`8o@12mbeFPXc85+?9E z(g44=ssDOIa}>}C>X$aC73%isxy160^@2uI4JZFUs7;^k^DWCK4Ua}Z+)#d_=UBK_TPuyTH#0vN<(aT}I zNiqmzH)7MNgm_Thb%2`lgP%7)c^id>ALxPJf|ulE`Qy1-gDTe>&qNMiJe-VdSeo@U zJm>wv!jkwJqwt%0>RQ}iYKE-SF7d{JS3=lYldRtnO7Qd)Je{@exA#Mn7h)8C)0OMp z4&Px;D|Ib2m(#aLH+5j5IVpIQX3OZBeqROeunus`@+VLuexFUgIf(Rw>>@~x$w(5R zOBAy7`e%wd9$~3SG#LKvF^aB6+=YBIQT0%C0ztok&NR;rQk#FDx2XFtX+gws*#)b!=&ZybPiKQkkPZ zW};WUKw$ZMGW@kKfv=J<*vhp8kOnVkF-E%wFF3oj4}6;D-j`HxuswEW_*R%va~kqo z+GKrs^=?9j`Aq`Lr{M6aFd*dK9)9rbdmoC!U1ZDeo=JP|2|2ZN7Y(o`EWD}*qV!z? z$8Y1SpR5FBy;6>_jw1s3Ix~6%b8Nmx6PSf zbm4o}(Oa;9Y7@K5!2RwsK?`KRwUolqp^7S?AU>Cd%Y47Z=2c6>5+2hJYHI~?NgSi$ zF;oawG)Ht?vp(&~4;SeqWr=t8>>;5(GCO&_5WEOM;=%|q+}0dMf*c0QEw1+Z6;zv+ zjoL9`WSKg8gvmy%04Aa2Kx?*w36kaiHoOdthDjAUHR0YKl~jQjXglE0P|xD;%j$Z_ z%3Au-Frv9;!q$D|8GX_4&+G=VUS1jtYid|S&)V=0>jX|wTgMbWLs`uDF~jS z?L@i#iw%x#J5-b(Eu(`Tnx_+3X1EpT5bV53`ptAp5l)h!aMSyQg%S(>7GBHjYd;oH z<_fQ)?%twIx^DJR;$l?6Lko8QFH8^}%)=bMseykDn2XJ3oC7yPFo&2{53DoU?%-k@ z^)z#2A{1Qa>g8cdE;D)rQMq+Wa+3A0gp2dS2*_{|>4vRa#Py_6i+hTr^Nl4KG03m=zn%}pIiMzhoW zv>TWG+e>&wn|4J4P-!n?1zS77BSeohma&D2{uuEql+>DW0nu6J-WmyhTPADg30j{t&Z7*H02Eu`otD3VOdg!t`VNPi{D386jX$$prDjk0UL-YAT25)DhgJL2oVthQ9=zp ziy%@36_suTR8+e38hR9ji1biG4-i5~0!bjr?st8E=6lP`{R8fu>oAkyhn=%$cg~*k zdcB_4^E|1_^r}7ZVH)v*1^wEp5S2@Xu;DD|?!$HIyt3s9yx$4@u*X2vekY+;xNX6_J7ym6~f+p zS1#N);*)-Yo_>Debfd$c*I(x8BAZd+XF-|HO^jrTqy0Ol1bl*&+5P4#S|mOJ+c{29 zTd-u)i$N^y4vq&#?pLu#fJ8wro-xMqDLB$Y03UNH@f>pyL%tuY(ZJMzRxR>#zya-{FiLki@V|k zaO4xaf8;QD1kThXfoq^b=UF}i@fLRjl&~Ed>O-~?>c5Uz|IK1rQA=V|l!&4UdFFlH zDI{bR!Fs=SYriR%E1$7? znJMIG!N>{`_cdzm7KS&oRe>dwSG4HsGh22v?6e^1F-u(^m21wiJnH3e) zraJ&R!KE7uQXUz21kJ9^)Hugi3(pu$7iMwrN%u;iZ@MoEH+{`jb%CyrTsU#QMed>~ zIgg9^2|6lt%zs(Llwfne|f8x~Sz?sLda$@ch0x3J{TK1{%*X(M+W3 z*q{nK;#FOBAtP@I47clg8dBhnhEmM(eZU^UsV-3)x4c}{{H=iSEY52amuBjuUpaqH zK~>B+p4PCPs_<4)(@uWklT=`Uzq9k}qC&B}+BVhs5rLG>BXGS1FWv;MD8Tv1Zj=2N zpN0Q6lLp=bIx=??Jz89}D^!oVn}f#&uhUWciY?3yJC5h|CGCSufLD>(Eo;%XqX2nsJv7lS-i{L`cEAJ$wG6TO8us@!cOT z36wWVh6&L5gvzX$Ws4T^8t}et4^8b_M!-duh|!+&sT?jX{}a#Z*()}EJHMp-c+JE7 zvB}DUCMzZoG!6)g4Se7H`r6jqJbdVDc}8gz@4P+2sGSqFbw4beEC^ zB<3~2rV9sBl){{I3}pAi(f*~7^{2Le+X2YG9zUomdS+|Y!wyqEevjE`e08$4h@s{9 zOX%R4wO4~r5VQ9T)u}F?>N>53iVgj%my!2JOAOf9Sqy-eGXv zyKTDrYAw6uOieW2jlb=A7^7f;k`$H>jTtC7Q)71Z@vl?vbUA6XQPM7 zq>NNC^u_5hj|&AkzHxDqSqNuQrhfvkg@vW2-bse7tav2;*AMHTqDQ8m{N8cL1wX)L zR^d82P?1td;;zu9?C6>AqHOBh(MK)b=a1EH`n)3b;;#G{!&iIvI3jFftJITo6WSZ~e+9&RK$BPehg&V--8n15Eiw*)$IpcuNWlcNaykWt$K zaAsCA;UEOV#b4S-`dt(@LE$>KAeOCr=p#V_`b{{qUzg^OjoJ8yS9TbZFItyj_|W^Dg}S$K>GO0cXtS5p7y13`4#ENG#gJyt zQX!4tpc7tyBlXAMyEf-HxWT2z&0ZU_n&QqRb>1O>yAUk(T(f%)_~UBP%RfKC>DIl- z=#(vq!|Hj6g`F?en>RLcz5F9jH+{3hFB7<$vgvPfg{Zg|8d266`36|Gd%W}M<+CeU z`?ue{0VZ7wlbk*XFWG13;nOVq?w7Mo@KCL&wGzykGoL;Py0&dnJp^6(a!!oBU3m(5 zE<;dsogJ6E^->NG$$a6vSUNy7Q(3;=tC0q(ZEzf?gjQ*59SF=l*nQyrEWW+&}o0TJ8 znVr;z8T`p@RiGtH*8R%&+3t0TX_*&Ks zBE*s0T-bm}ZP4etJ-T|C3t4*#C(asJsP*kooA`Xy{D>N-a@Tqc;$VjcF6xtytj+23 zjFwB5=f;R5?RXo$?4hM2or3cR`ks{+%|9ALg*81oK)UscertEI&HOpc?qdyTarE8%z4)ssIV~MG!=`=i zZj4>q->kayx%2fy_(d(6!5`Qmcu3Zj9RXv2dN6`mXc+~JpAw(fhbT_fRI8D&N3 zYtE-_;eP@T`Ik!kbpgFu)yt0axu+dEt-QSfIG&go0l3g07NLJLCrMJ-a(IQFy zg%C=HfJ0zMKq3biqN%_k(XW8;zm*9VqGsJlaOgG-a!q^kg(X>*fRD^PLc9j<-QJ1{ zJ==8tzq+8-_|XoU0^q~r_ciMBPg!*>j_f@Gilm2F$Q)9Ewr!dsjKP<}r9HD1g!;+Y zXKW+wp=dA zz4PYQ!$!A{C24r}_in-YbuL{N=DcXYG^T_D)tA1U1Vpy2;K$9rpM_`C_w@rWTZoL< z`br(QkZXJi!`<@e>Z?m#9TknGIXRYaz-Rde&@H;MCw>yRcQIqk=tq4@;C^!QB(NcC zjk5aB)~K*?Qm_#d#y&Ugb%8Q@;)qL{zS}%_FUIqYuZ5n6d)f~CDCDd-ij@!KSm*w4 zEdXWkV0A29A%61`db(pw?&${m?PLpGr%g2+l>Ya`GY8&6^)JphM!)Qg!Zstr?eJ5O zgGj%D>9`Y3m^GRjI+oR5PvAKVEgQx{^`Ad+`e=ZwlI+N{?~S%xhOy&;n)J6m)){ z_BLN9$oLko(#to7Lx_A~_((p!zE-`Ut2W_{x;d z$_sOoE6;e$P6^^3a`ZbLHM&&aQ|bNh-#*?|Z^lL(m3JL8>pk!MX0J>2Q~9FH#WKgd z;aW)7tG&&WK>gR+iE~c-w9{`0QpVRR)AZJzUs8{dwCcQ}40(F#oOpH%q{i%$TL-TW zMB4r@={((LLuP`XlA)MqjIm|$vWl)pJS%SE&#S$k-T-H0bvH)9%bqm4E)fPYn(f;r zk?R{3Vj?CKo1v#+JT-`t$BeTruwfu;;|#fGO6wJ^m7p`$r&5H* z;vTPyz;2)>wOie0Oo!iPYv9*c3daUd9KJxu-_Dp zkuf6{8F_*Px%DkmiswXaopzgl>CAuL{y_I-dlaha5*({J{=EXmJnM>d zL1Xm^98>rlc>$Ah zIB=%(K{_sO+SsR^p`slD=WLz{hYciTUO{VjG$TXSoOF}6g){GjfgfnB_Fk5{e;v2-uexCY^Ku*F5-@;Kajrj5_lKj5&j|S{}AqW%w2|A(mmL)8Bv>i-b+|3^gqABz4DMgNDQ|3lIL zq3Hj3>Hm1?|G(y?yBCoF`l-I+e6+h!)!1{np-wkcfX8DFpIEb?37N0Cz%?s-2URfk z48eWSX4eH9WI<8NGZ~S6&G=tBFloDP0A`*C-)0gEMP#Mk8yN{whz?s#zy#aBRO(cl ziv)+ZSl~kq%vVqUj{A}F6oT8_%coaT7Dv<8+g|>(b+{FSYDSo|J;tbkIzfzCur|lJ8pZSn8xxfd%S3=XH(w z$6JweWrQ>#kev%0otcg#G*Gtox2#yu@|R_x5@@bUijLAhFy6c#ZMGC-?ey942Jtf4 zaQAi;m~ll*O0O>Yomo|Aq;Iy(bjVt8KiRVUWo7F^nH1=FEE zUK7kRKk~0uwtWmf0ZEkI5bUx%;1AD1#2Y5Y#~*;{^U5%NK9eT})93fY^m%)=m{$P|8GZw49}iw zyBCo*QTA@AwgVV0=_^qSS}~zl((*CkNUU!aRUi7KyIajUflXbXfOdE0ul5mUCnyuH zKA3RIw5G6h=)v0+<#>9#kS)rdI982$lVIcT955kA>F&_yreebfDTOtx+xd@;hi6T( zHpM6QOR1GFhH$wmMst?YPS7!td<5-0F(3k<%oBloPCIFCP=F3}Fu`VIMDUq{!K0`{SmP>!MZa)(<%{k> zGvgWH&V9H&al6&KTbL$3FosC-&FMUQm~H>t$MUtJ9CJAfZC&#_w;C$I_Y$}WLQBK0bxA^Y{;b@(Ll-8? zv}&)Gi72IeTu>`jH_`^=XjD!?Z#zXk|l_MA(LEZe_o1(?PaJfCGA3#5z+ z(Ji;53lxE%tL1VI{b`3A*E`Y)Yu0tnIfz}PAwPcqsCVsz*J1o5WSX$~lGsI1X3TlB zsZV~p48ZzB6GlKOQ4r^aAaHMk(Rr)b*|s-dnrVs}sfPp9VlR2GZ_6vi!416oB5HY` z)Lgj)NH}O{s#iEnv*kX=s;#OeWpr&&{UiG4BN0txtOZQt=N_0qfao? zRBhK?RLtP!QNtA&kwNazU$^tYeAsuJ9F7uQ-;F#?Q5EflmYKyAN| z<>cuLv|I!j%=7yZcCS!Y6~*e)s7bVZNiG8^UY*tlO^-mVW3ClnR|sr=TtS_zt;;Z! z)q1>8!+v*9Bh>X^eUfX+({*XeV#J7+FV49CbRA~PH$iKjFGl>Q5iuDhAPtE<+x5E~ z^~!MUF3h%FyZbM%ZfpnJu|;~d1B@WR%S2CwrzEn&->Y?HylI72>h3qMIqxSYFFV!U zir#JKn{vcfDeR~I0xs^-(Uo`~!6p7z0AgHqZmBv2-Ov>-aR;VT2~5howU8TAPmgKL zYCd}87eiEw2D;;?-UpsZeMerQ;!%$tVh27YSk?638Qj4+CV)MmZH5<0jfZ*{s4<)f zJdQ&`m5LDurak>V1hu=P0p$OS8Y1dc$bDXRtC_b7;}O4Q+@sU+foPf z2xF_*U!6XM;XOpv93Tx6u%XU`jarcbTdJ=VTn@B!Itb>*65mkZAVN`f*7gGG02Jm&G&A@VWmujmHfuvYqrH3eB{= z&#!9pTYfb4u+dwNWA8=_%aguLnC5CW2(knOQRa6=tl6n#8;`KMI^pmBNqZ*)y@GO^b(z&J zG`GF!7tnuW%aO+Ju=^&hPLSQTgH(A~r{?F4@i>86V3eXvZSx)G*K7Sr9nC6FyjSq` zHwyNhA5;eyN7Bqgjn5)ga<77YjjRhXD2}e3r7Ehq(mRW|&LfPTKD)I`@(#z)Sri-2 zlRklfHl4_CYP5{VJH?hIJu)~S!~UqO|5d?lt7e#b3tHHN=4-`Vr{u2WQj;80ch#^@ zya$)EtZ57FnCI9B#%kf!RqQxWGaHzbv+p2ZLg;P^QNXHUtB4eCt!f`11({IGUw`|BzlO<#)$40gVwJDZMR|5;niT<}E3gopO&rJ=T( z=H51M0LaJ*x13=ju$0sRjVIK8)StIQxWolEWs-;KB630lVS}&t#42)*OI&%{7xX!S zYC*)QIJbG|ppMzNycGsPx@o|3#QP?1c4aN6BzlmBh6de9pRLn?@?FVe+d5jRfGYnH z+q+2B0ktku9KCw&kXSx)>ZJt%SUL;YMemgFfwV)dt&G?rdkq_(?$sH1-oTVk+zg00 z{Qk9VpR0B+w!$V#pM;{oC*^u6SJMfbumJ=z!?cFqWEIL8x=D5li&o zj-xMbx1!Gcid_bTZEsmUg;6YIZ}V@rdl9Ck>DL4%TVRGeYV;m)BSz`9mde+A6>?0> zPh7|ufyR7<{RUa>?QLl_P>4Z5B&vn-kw@U#CzN1t)MV@`QElKIH|{oJ04$5_Lv#Oz zQAt{*Xjt9_qW~dLiUA5aPv(Ozq;M!tFH^SboRcF$h?tu5eu6&x8!`gP@{fMH@m1s0 zJBxZA5Aja*d}ZJF6y%(ocq==IdJ#!l&q1+%>>r#!a(UQZDpKzB(fCIxEZUdxNq z44_ObZSe!A-x&3IqV|uVDwJT>`*&nL6mZuP*6uj11`n-e3xrZc7UnNl_mzquEw$?~ zF+!T4oY06sRR__mDUX!(c=hzorXxxn>;)2GgbQy$!Yp23Gre>RHV<+^^U2t`uUVyo z10;HWEW!Chlg9u`?YL<2=~>rRd}!3#!0)QQMOqcN%sN|_Cn}5N*+eV`@ zc%chxCoxT_;bpE!Je3TPc}Cd#du7gPQ95^0&5z``T&l&jOlJ?7WU}MQr7n z*59+24?!ZbQK))ki_NAO$6L>@`a;iq#hikVq+U(+YF|1ml zDX9KXGk(WJoNkmLj%mmwZk@3!6PGl3L7+&awpp-W1(L1lYJ;eF2>80X44Kfs?f@q{qlo zSOnr5x0X>urwY!3c~q!NgF+3xuV;zsYVmMJOcPFXN$vYg2(ou0PmQK8kqoG(K^N2P zVQ2@M&`>o>c884R-5=z0&DsxuvW#1Dm{0bfR_v%7$<8G!!!30*&#BQ7HGgZz?5oN% z0y~Yi8H0Is$+*LP%gSIl5=Y~tR-^i3&uFwvSk9CDte)fg`P$e!BtB8CEJp{WI<(mA zJsz81Ifox?vjx9%_ynk>P)VF;e@08o)xm7t_=KhPy77nDJKne=WiSyR&Yd_Q&PJGw zcbr%q6!J(tO%C@HW#cmAK&Js2c0O0=0*=1U*X^Zz3$?jvMMLO)R@`3lcu*XYf|^iq zxmdC@2vcK0W`7RvX3^_$%+E`$DA5bUHCsLxBptk;$G{fV`#n&m1ELSle98DKL@r(v zPRCiz%O}hS1;{qslv3Ca}p`=N6o(H5YJZN??kf@*>=s5XXRP&$rz_~%6e{CTi| z6_nu5wSF+h4AK8rGo})~ya^_I z#f_^eG!nX^qq64S=U3@$cpyCm1dmnpnA$4oblqpL^MU=q7IOn}hoZGO!OS`R3++m$ zZiK&(BR3>3M~Ppqa#avsIF)7j2;LAZ`GrWRkTb%q?i<7=w$Ena?EoYwQ(a3I$e{h* z`M#z{=U>!nTO4cHaggq)3DcA?|x*1h4B)n59PW{Uy_2*-}dbhu#P9agAA zfwCBJwC1biPYu(W59LfyE~w?3s3|~o%>@~>N8tvcF$J78O9BD^bY6 z7_8Qbm4@ptrPI+U5UtjP*ZFg6y4=3Oraewhg62+?_3Rb}M>M6ozSewR5iuDv6O9C( zH+?W%-6A|7f2{i^ardbeiU4#8tJ4Ult+^|Zg6U@49G_EEz@?3LPtBP!1O5->7aw&0 zRU7%tyx;W7FA_Uk)lhcflUqIdD)5%o7EHumT-__DZOSSo&}QG3SaDcS>93Z|RpG%jr?3!g z+!8{nm*7g{zwb89KAXOWJ^DM;^ZRVezs zBU%+(A23Xd%uoLNqv=OBPiSt>D z;H;C|Q_HPkw7*}@aU3^%A!WUP@gw-lQLpidpM$zdD*b8cfa;3zPD!1ax&fbBHhsVl zxmqwE=k9qlW#S0B;sd%-%wUmHDKU+r%u3@_4;;gIyD{Wy6}fSz z#s%DRqW;6b8}WC~d){PcQXDQ;OZ_1}nRNvh}Wkf1bXPcVVOq*guNuRA&0Lb>t z3qiS0oEp>`c2oBt;u|W_B{pm5VLFOI)+O2m&`XNYyf6q;S{zvSjf*hop~cE2 z#wt9~yGiuW)SWMA^b~wb5}tmV6lD1TYhoQYCSvlOPCR|4Vo9M0_5}24cfA+TX@u6O zQs>N^Wb^$%w%Oo82;r4#s{keHwV*lBfGk{NnD-6b0rx|_sT|g0(Th##3fKxaS zi8x(2IK-{x{xmr}@%G5`IIyYc{q2PsR@3C<9}PV73cHihJ9vcUHJ_oParMdP60$va zf-+OG>SX`qvt}1rb`vN4$WEBkPrt`{L_eMx>)@5voADy~za@~4J?>Y6O}tJ#QEBSZ zdEx3zl#xu;HQmJOJG#HV*zdB*DsjqG@@meAw)vuP#Wq*rVRvCnUe}%C>r$2XJDXLz zC6)4fB{k+OFZ4MdBY)XsQ#f;9=8;&ug-iG@@1vQt;+;>rpXFJ46^fUH{>(0EjL~`5 z>3t_!^U*nrqm}((%w zcV9;5&fc6}$z$F}eR}LCMj;Y~fa5$v3!3I9Z=y7Ho69|AGl}#6TMK~YX>dY!ssib) zs6QrSaJs5>sj4!iJrMHqoY=W~2m4}yvMtdgAWG8vsG)9dkD^>SB>cA8j7`tRdTWL1 zb=7v=Q848q`=y`3YiV=h{~H!qhaicrPamMShL4-=Ikh;qRnktf^1g1C)g~Y@XT+Yn zSeD;9o|L4IV;7}m+*yxP`ef~8uXkIH$g}Qu{Y`utcKdrQhEA+MA)Qmdz%=s>3Cb2i zm#ZF9ilJM4a-Zsgx2ZK1HSw#fN|6!*w8p>xLrWC2p;tWuMne@JP__bbkVo(l;fnId z>hhyZI#Ub}C!T4HK%!R{A7K)x=-umD?Q9$X*Q-VhmcJ6%V{@D0DiH=e^MMC7s}#*b zs2?F#9=<~QN`=1!jqlYkLt6sg0!gRBOz@?|l9@`g}MwH77S z4yZ?5LYc3`mm66``}~CDFqL0xSbeqwir`|nuBp0>X;kDD0TP-3if<}1e_#h3cttYR zI2CrmXE>Nlp))zOosPhCQLmE(ubUVHw1-X@p}T7Z4(#jv+c7@meC>O&?r)ejsUM$pJqw>eNhSx{d4d^7O+WuT^$Wa8BK-`>&hbKU@(l|)m=)r9SCyY5+L-u|u%?RWg&*T8aA#Nlrm=b2jR z*T6%m1$FA@@=D@bMPmu&7n@P#KIosduF}QO=Ev{Gm6!5MxefCYB|HR!x~9g8|2JXd z294jiHQ#?!KrD%ZD(L!n_T^t!RS$1Q?z@dQXDtoHSlfJpI^M@%hQuIpk8=zYO_2Re+8^5Wy zBa9>HS{QCDA@txTv~2D7$MW;JVW{oPopHGCdHZQ&gYj*e3qo?1fp8Z4UKl9Mb=`qr zO@MTR$9;0|%1tlk>eH-Hop}UB0R5=fVC5&_JAgm0@r^En8AOjFldQ$vZI&O{oj$eEr~-qPH}Owms`^DnvdI*1nNn=gFGBJ4MY&MG-m>{I zD4UFd>xQ)>S?~QH>HY$5t!r70F6!{O>)Wn|`)M-xRBSQb)rF`S5TpMC3C!-b-Se?z zM^l@cs+lFg7p)S;fRq6797JXFSu8+uMd;6d#1eH!I)VF;8jAIbqs|W(8!K4(xPs}}J*`rn`=$Fh^%ji`UzH30AF2%t4i;Kkn~H^0y<6$li`UT8zkxRv*?Cfn`Tovui^r`!Jq>zWhudC zH67rCrZzP5GD9W`-uRd;oNB6mzD>Y#&p`15fHTUWJ9BQXB7*uG$wug^FG36cnvH?8 zO8f$GNy52C!0*tU$dh%aT(Z)zcd8b%Wm*y+2BSh3|0L9wr@4(Al3!>I$r`M-sX19F zsi_qs;;MVgIdTN1oI#`P&uRPSlU3n>?RAb^pO{`arn$K}iK*NGn0qNbI58d000RDu(EG=lDQ1u26fM`(Er$d!T;J z>tj|5{!^6PN@ydZb5%@o9O|$IY=TOJfgn*Wl6!m2TZK0B7Vh^DLq{@c0VV7-E;B@& zzQ`c_&fTJ11EmWx1=+%I$Hc(SJIB>jM;$KE2?xqe^%VWl7dEUw*#J2A}=jxciF^3_pFDzBa>F2xA#N*P&dDTs_4{I%di;J5$9cJs3C5wOPtx-%8&r2t3; z6x%g>k<}~hMN(#ZcARib(BQlhbb7V=-3gK-il6jTM}KlJVywVC*^V5GuZ?plMM!$- zjsP|gq8bv$yF+@A^ixEFJvcQ~sp!}As)zv62n!RvD`;MWjOn7z(r-gYG_=|`{Ekh4 zL>NAD;|Nf}t&TNs2AhbNC+MLA+P#aE^_ZLDnYN1|GD`spGRGHy;$xS?75x)mOzpyP zG*D~}yL&cJ8wSGSMfyB`sS|(Y6{7Z^xnjqwBJg#GMFY!cGNLmGczYawPh->ckNsKN zOzT955?@BnWiV?)1 zO>|35mm#h$F^@wCeP;MWi>$cVB*!95AUmbVKCZ$A54|*$d~f5|yhHu2ESw+OrRRRR znZu=&d_qS|-Ssl4m;K;_sNbSo>N%GayU^@=HbxXBlQyk&Tr}8ojiFXw@i}iVwHERF zCwUb!&j?S|*Q6TCR91Z6@hvkz-F9ncJ;_K}AQ7{6RKD_e`>EV(3d7ON5LpSFs?HeZ z-j+4g-(jwK&S7&iwlOkIPK%zMn*b5}HQM2-UY=Kv+%o3guikSZ6d!HMlam#xJMER8 z;&LZlr&4BOgZgg4U9Sd8o$Q*9bn)T4U7LG4Yh=|TI!CD)QlMzd8aGLsndxNnFaPJ1 z7yp~`%(?&jlr=R#WfV0twgp=Q+&IfcUhL*lzEOe~F#ORCpwudYXf`)!M#ImezQ*Bu zpokti-hCDhb#(%0;qyTsaF(RVe6Qy7Tfw!&jz#5{lGOD#@1;t_V`DZAubPm*XmpEqkF!4%U zF`jNP*I`_xm_-T#<>5j}Sz-zlgVNWaI@1db5zeSpd(>{>qDnsqbc#cs-B)Wr1xc;6 zWBl!PCtn(7w0ERJSU6MVyiTZYs;!_G6*JI5SNWr2DCx47z3GLtGRUU1D;mwZ4?RD@ zD52PN;I)SH)mvt|xGRw;DzWRbe9wr(5RgQX_~7P{_u@8M_^t0l)m5a$k}`uAkIm8@ z*(5RYt4p0Y^l5-{4};cbRCCtzPl+hPr`;D~*>}9K3ratRyoU^|(82uVgwc{bkCOnsf9YM`ikLy4H*aJz2rgwtEF>9zGEs)h%l?&YM z*KNXY@@2*2=M6J|8vN)4VgVd(oDf*|l&-+=b>#oV%MzXi*OpcHoaMBMtEfsvpRTYQEFTk~YHT@>>X8d~_Y3Nn={7l40%el|6?D5@90p9(< z=nR!S_Z@bvQR^SGYm!x87oo+O$$&jRXnzHESBU8IalU{F>annw*%9!7q-yy3AJhV$aRa2kKZr7Q68l1K`G6}dv00m z&eCt6KDkTE5)eFL`a=eoS)V_V$|VVed54jZZAuGu(NWtx<#)38TY3CY>Ok28gyOG3 z`OP13kT=M%S=Q(CrhY_b07R6>wi)3?!F!b9#JDS z5Qhf@24x5h1#!A$+^INz97<|2g}_X?MzmW+1MM#4GKgDNsMq@w97~wXmBM(BK(+ST zK_D9+UNSg5+&gECTOwu{oF@L#Mp3)*FAAj~HmC?x);T`bN<^gvEL7A`)s2x|iqYZA z*bkT*$Pl6XQ8<7XTInw2`FQJZF&{^^=utiKGy$CY;=32%bJ5_v&yq>>%5@-l#R+vK z9F_Fc9tTQB*;FCc${`=dOeqMZcNRC&u>)bwc!QX~n#%rAGRkeOkIE+C<&)NfS?GP| zQ=4Ewgc>4#vjev7kvpyVRJtRIkj((LI!Xgi8@dyIH#kXM`Ti}*3mycyg1#rcXTuHc zHaHy{zRAmIN{y`j$g?-i;E8OVj@@Q>H=Q>ar1Wgz_A_Ot$pVRajtXe{T;AR-#Hlb@ zTx#Op@#MQ&s|%e1gKcK^TQG%%nPHwzxP4)NI{VFdcvn;sBy^ciI~ZWrK`jZJW%K!F zrE|T{mn-LzQ{XQkj;pW?itWk~7$0~8RdLU$7{l7hqxWpE_4;{(K9Hx4oklMH(7BoAs+%C=kaL38~JGJ7->DOe0?@8nufSF@i|q*b;z)} zztf8U{ly8ka&)g>M~8h>+P+_v-x0e{$#p%`fr`Lk4KfX&d6!G0OaTU;Y+}nBkL*n1 zJQ0lEixC_JC77vDHd^?osuMI1ygTy?$-+|+Zk`5wl}5>5E>JV^ie2*eNKmV)QF^Da ztyyXHSnW)++9R=ktuktU5EJaIQq=s4-Y;Gi!eft(%ukFpxV>4Sm4Ee?3XHDqaH~&H zJARXJpTTN0H_m@{d{rZ0rT#=qIr-B z-z^x4owb`7UNstFn)*CSu)$TN-1VE%QsNnhyOXM5<-WJu{FVxwbHF!Q2A}kQfeQkP zaG0`UBtTYi&;dQ!WQHI%NCRn%9c|@+xOx-trQChl4gsTIZ9=MBoAo)PKvQbXfIX61 zj!m%^Gp&QpvC~gJNwD>|D#EXu9fsNoLakn+8>u#L#$K%?6k$-V;6C|^o-Gs9e#Z|u zeP^dkd0b24jDJMXR;nJ~J7SSo-g|P{&yD1J-I=gmgw5&6w=JF%xZcaM8AF0dE^4)( zx26ficOd&Fv8vC{rdwlWvB{{NdoyljoIbQueQ221xM4f^An$E#F3r`AMS1O}M@{CQs`=+9_W8U4wqnLy+*RPV)+j&%(vqLU|&flvaN^8%_etoYkNK{`qetmkZ_r1J)fb`P$0{uSOTaOiDv6_KWcT zCl-p8w)S;63$^&0rd9(!GrO{mvN}ch>F{`nr_0&+%(dz5#@6V*JYKY~hT9Fnu%BQG zSA6E-7+HS3#M%7UkXYV0ek^&-?RgkbTnPn1 z4mapm{f<^CA8*YsgR`Lwo8_A97J+?f`D#ri)OjdB&^_m5ePu5n#mOU1?)X?Q+e)m< zRVx#jC!HEPjhrmQcJ|G$Ex9`BGS~2aEDqINH}PC9pE4!TodWXS-V6y<0`mieG;$-< z(T!#9Uv}vY$a3WGbV9_k5kA2!U z-F|4#cz+7d5VpBVzUA@w2o~Oh(L6Z}b`H|tx*<#O_~YRi$uOS9n$4Z+9(On2@znka z!5}p^6w%$Y*R#%fv27QX-^S#gQ9t9QSL5Ab3dbuxeXw+UH%aaR zq@K?DxZPr|D)vbS#N08xx^LZ&$+Lmxb)=&WFLi)|sEERbvD{l$Wxi|LMQH`C0@5$M zX4X`|2!F(2^;3c^*WxIlnMJ-?31nQN+guv?kv{O|rSM zr)I)u$db-mIE`(8hF!W5-xTJxZXp45t;d;%jk8yaS@s8RgP#({oY`NVt5Zlnw=PZQff*7CRRj8GKh~8sn8h zEFdvF)^~w*Tb3->OsQ0VpsNyBH=?)v)9kH{&w$aTqsgb)wx>sf5&OeN#%EV|LZ82} zxXaL3+!|B+)~o{v6|~(3;--&Q+>X})LeEVWwm(AyFRKevzH_8QFK5SUfVeDwR!N*{ zZ8Smi+jxdg@2$6HV>f$K$73db^o_J^a5t+pciqwPOg714(y(@WTeGZ-RV(Jh5f$}x z*$vn3?e2k8)_ZnE+<1k#oPvE(`0gMm;U!@jNtnB+viZzqVdm%cd#-5CUWj}X6?l_8 z^H#n0doe+@QZ%a=)${6_bnqLF@zqUd{(yg=la;@a4g-}a}g ztT`l9J9Ti4pVKPkEZ^eo5Of>p)ZW&!byDu?*Fey)Q6~EPejvgE*Mmf?CLfQgrv^geype` z>{Dj7OF;$Vvw{fT{q`IR>uTjK|ftiq<5)YGD}F*(`JF^VA>+R3FS z9IhGA-yx(~lMQuA0<4{L0C(xi;=QVIfE(?W_9!RrAW7 z4ov@w{h@M|IAz1)t>QDl(O&9-A@ zcPER7br<%SUWv^G?8i{{j^dtWlU2ho|)faw?e*pmjr5h0eB_$;#2BZWDX~{t; z=~AQ*A%b+H)DY4sr64dM4I3#^~c8|64G1Hr59(A+uDH; z0p}r4)Cvh2`IO@~*52nEIZqR$2~n)}oy8kb(bq~<0A*>eWy~Q7*N{1M*KY?_;Th?b zHUT}D0{8C0WSJ$u?$4wch%*}qkysuR)^T@A*F9BAadQ65<>16ZGRig9vwpu1<=(kf zdzSny?GXSJ(ObS*Ryp2;K_-oJj{H|3$yoz>XE}9ixE;@mJNK&(_Ip%FdJymm=E4-`~8+QSm^ zyfZ@nlwBI1c<8rXppA#v2OrB`Hp3aG9qB4Xo6P$hu(xw!8?v|7;ZWDkq)d#K(T*AC zv^n-2Eu$ri);uL_PT@l0dUEqUlC+WQng0m~a!cb8L?b-?SNV~_p57yf#ic%R_hjl@ z-zMx@nof@8*EL%Wa<}E=Hy@&FjCEc3r&#YtTcy8 zRA5*0oL|W?Zc`NdI@L~x^>NrG9jC6HCLotjV+Ss08GBE`zzdfi8k#QE!{MFeZ!CU< ziIX#;TD2W)l{NoCF@xP=m+47HLBjrLKWVa~JYxmM@9PIyF^RN`q)-Fl_s@m}cxi9U za+#z&*c;Bfx#KazQU297!v>8QDU8I6%S!>(*wvwAP>t#)fnt$5y)|Cbkzkw&yWh*s zWmWoXG>4=uh+y!4W`MJ+Uz;`W4I}iQLObMQ zdoV2A>1Ycw6?@3)^uX)ui!>Xh*Mq6fwpyYWY$3K?|Mfl}z^d)Ta>=%UzbceJ6`K2B zJ-jEm7*6-_y@vi~eM zudiG{?oW%YOIhc0p+|Vf$Ei7w)w@P0GwDu}73m;079=2Cxz2$ATY%iv!z_>WRL{Kz z@55gX^FGP3mD-#Slc*Rzy#8hu{0f#O(PKBGo0r7b?XXBsabz8!y1k9W4<8`D>z-SI z+hs&7$1v=c&iaKtCa=9@kAXeylYC(F`7ci;k}NSc>yI-YbmY}9hBjP?ZxxQ{2^zc69&#jAeP1~i<*Mv4H`qIy{Hr??RZ{;{ z<(GswtBZ>B&-(gjPXzhik!9Tt9~EXP!~J5eQBWCalKXP^YC2!OT4Ju=Jn4b1pB}Gt zP&Y?S%*-hFV+(3eJj-wGTe^`%8nyRHOZBn@=Axw5W$-@&#t#z6*0<=tfV_fu+H>&H z{Y7_mxAh8q)jcJDK(zCa=_2n?hF43SdHy@92Y>JB)4{*32TY#tKVYr zo7;f8XmnCoT)*cfQvKw6jFVqB&F!Dl5iIS@tM$N!FV&1SgVweg)z;&SAwo?tQL}Y5 z3n#;`L=Lgy(+SuM6w)@`E$}+CPZIOM;iF$C!5luXJD+pO9|SZEfMlX9%Ey$@(6^;x z9_QcF7Vj)Zhb3N2udV58r_E!}&>DQV%~s4%xR?KJWqHN?ND&{uafLBZ3cwF`Q@mWf zh)bWhDxi9W$@BW6)_efekDH3TH+<96iG+cT?^NhB$UcYn1eJVBAbg^(%CSyN{SlbV zZGUf?idKuRwYq8e=BXj`Q@Yj+?Q6~_w=EwqeLRJQD5DXMiXrO%b6Af=nD(7TNEsZo zn^z%;)X#F7pqmdpAr&%*-NO5viPzN%92fAH;*~vXAm;_!DLcKU($81S%ufyd9Uo(b zL>v$K`k#J*Zr>k+jL1keU#o8czlIMpzz80LJ)Td&1`WkU=|s?ANOvSoaQk)Dt0*70 z{f0h}RJ>L!LOC_|Rp#!#K?FXdP<5b2rdt}d7>&S?mOQez4g2VukjwD*71CKZ)CEH# zPDw91#xX5zYx7<2NodxugnZg2!`m}mo7HjNw)oOR-_&Z(|LS794r}C-?JDakwg z1QtySTEDJ+E02=?`1Po_c>7t6yx5JD3OQRBUknW^4ecMl9T(5bWwvO+m>68;gN$mC z=4TGrB}h^tmtvb{qDsMg=E^E77&I6wnSP}2{`7hs2Mg1Yrw@{3mDMGonuh7E7-#{B znY0F685jjNf&(Mr`-Un6c z^q?qgF?Q5*p-PO;H2?GzAd2^;?){j+UccDQgLN8A1dCtOQ-|2hvAyjX9qAh@Qf~r_ zA(6AE(D=+pCN(s?_a%mB8WE*cg|xL*{5kdq30phRaeStzKHdZ}-fsd8{@ko)7J1e~ zE}(`+yfWDW#-Or}HFPo(D81RnGFsw8Qdmw**e_dpQo4U*=#jPR7Qp{7dJ}EEsUu&_ zY%+(=2A~(zr)X;U zM{M3RYG*??f5H8>3Xcoeu%Fc3?lS%`cj=t3>+2czjEis!zL)KD_uYaA?jD7b$$no9 z-^^WVoOexi3vS!<))CV2)d#N^FD#dDbBMqA(ej%N3~XgQWZ|UZC_P#@gUvKFds`bo zCaP`f$!Y1GBKvKi2Zp|#?zW?B*X=u9yaOt2M?P(4Tm! zIc&Ec_#!G!ggg6R5Fc*@o^SO*6UDIGOl;luWV>%MlI8$Gn+Fs$duID%;MeCztuWn8 zEzVd_%pxfsz|ho;+NP>vC}0?O-?MVK0M37_zvWY}Al-Y%tHOCxqU=IGMbYfURGt49 zD?qcQQ!pLEvSW(eHCUsp&| zQqHIEC7&}eCxg9j*?0S}F}U?rJ|ra=-2XaF2V(J~`EJlP20fM_Rj-1@+O44r%J7tz zRLktyN4=N}p~}blx}q8D<4|+FqRsy|5{*)QME@0fYi97Bg*Q($6tc zOJ0#zXei1=rT=xDmF-igm`@BpQs~we#B4JLJ~1AHzCdS%#b@PBAZ5w}H>?2hmBqXG zyn1Acv$I9F8+nn7LnWCjc1hCk%bn;RA@Z&cGmG7YHRM{MP~tkbR+)>{Ra~dK+?ty| zL*Tg2Jr=k6#1^cJ+avO*4>NAi1u$spRLQGftev_kf6-KnC7&NGxO>K9LLBkKZD!~8 z@%*?BgZvdBnPW8->^HX)8r{#WwdIw-COgpkjLBO)*yd(JzD`HFccPVoT&0_|&Eb>l z4LdnR~LB$tmC*ukoo`60^!jre2kkwur#oU+7+&|VxIGl|JBr>NNk0?6%vKOl?Mj?@)#rm*WfQ_BznLB~Dcsp_lAIf_>}rW} zlu@r&Z=ZcUmGeef>^Glim&dCX>14-+VJ`9(e=fn#;#nEVfXuI+NYb} zX2&IGA`{ZK9sCZ$@T6L)rFer{tY)~Wi-#nVCyn7+5}|<=&2we7GVaX;QsJOWB#X)7 zMG$a8xa|;VP~7VLb$x^N>iPhB*}ytC-n=9CL>|XP{ZREcrgR9j?5TlnOU^xCBxPBT zd5<|5Q7x{pl|Z*_4%EGy$(#9WSwD)(5j1XBJ!rbFdd(`5EMF}T{(bj(Uu11~2&y?L z%r+wG7D1&I1*q5rox+Z)+nzR*u0Gdl)q^xEPnjj`MpB%LAMU|Dc>QSj(m;$noK7hex0CuPzb%X z@x+q+8IhVsykc6`^MrPasDs_P&Bq^RrJva^`K95~F)vLz*=NOAsu*vm&>Mf;~(9S z%6SDYMac6EYF6lMoK&n0@K>k6F)Iw7bEU=oDhw9y2}~#(1l!c(!a(&wflU{pragxd zN9SJ!tD7gsa~0o)zFzdgM@a8VjNkbyWuo%NU!*M{h$b+CHK(%H>?$DeO73F8tf%L^ zal!N1?AvsUly_stgM;6YCn!-W1lLH=)q2u1!NlZ{={=v&%#i9OU{eNFx-|xD$C2{W znj9p?r~J8CXdNH*KfskGUiQK&vPlM)tY$7v3=#GA^h~ktqg<((VykjEW7E3Dfgc0E zRCub2YumbA6Xtl(_S~7n?cGcb$ZrXfN^k0H_G$1}$DBJBRJaI)_7jdSGSCC{MT(iX z@a|r$6Bi#bkH0wnIF#OY`Jg2!DF2%)m6LlLZE&1IqIK}N4C?H+cZ%s&nwe|jpx_V9 zFTQyMy~CM=zqz#et6s;hmTA?#?Sq}A=&yg&33$aje&bE*`J3U^iSj(nb{R8r`Ae&- zU3*yZ+Xi`|)fmXSW|X2smAg5X5A<08`R4xiBNOy>u&HxAT!noWUPI!xzJDDHYi{DF zEQKm;+4Lr|#WGim#D*i)OI#iyjtS(HiphLd5NP7oo5||e`qoVIP(Skab@xtQW#Ea-{f^N%g z&FQ1z+flsgXw>#wr?PUPH!?1tnKO8)qQ&0SIskF~E7kx_lvhjx#~C8I{>uy?1MD{P zeVYHgvFe^`n_A`Eu{g0DQrhrRQtqD1tg8hqa{T;p>~9lr(QWQonpxxBij)4uZ@Xa! z?>HyydeVlAuP9`8-KBSaJzD=f98gqIyUk}RBzQzC-23CoE_09=cBv@6xm)MK$Rs3C zb5z|tP^L?0&C*|0jF6=`^Y%f3#)n%x8bRbFS^y+PfB%sGcdT)xI_X9*%7w{#=bA>-2JtK4>u_+3X-%UrqC-4kiPs4 zbGmU=dAvX1?EGrce=?u!sCpURUQt~cx*sl<=da6l$>9pB*VBE<+mjI(S`#Warbs`J z#UBHF{QbE82V&CaGu4AQ%bkTustf09f31QnngZ*Tb^}|EM&UB3ys9=Mec9|(MW|;A z`C5e^2Q(WdjfHgbs@u@aLsINscdlfWA!bVqkyr7ZxuIIFq>qiY3s!kA{=%aqitH*x;Loq~%*|*InO80<2(jf2E^z zyWfFF6jymtE@rE-@SRUQT8(d?{#}I+n!Khs>fO2U=hcabM5WjClhr5k@Rb8S*ba<++*ZyI;wd9hL1 zItrP3L>70sCC(pwx1jiAeBNnO+i(d7uy1|xukPLns$x$z{^UFp8wP1fq8`3m49cvC zQL>S{blu3OL#+&RPYz=Tn>>fap%LTxlZJB^=b8@lO$fc?n($2Uhk@O%Pk7+jI!Qc` z|1hF6z<(+*_+S_%_zgcDhhFrJKE(um>*!@W0VFpLF=T=%{PyPT$sUJo{XLm|^$z1L zs=2ZWnQlau-~N2?wPcEDE}s)_pUieSmZY_@Un1E@V)xuc$x!yFSA)Gb;Z)USk`(B7 zCqtjievR^VrAVs1KH&;8{*d6!7SwNkQ4NqFN9An?<}Q1Fz;3%apOw*hFsXrZmjRu( zr{O|_D;u%AKB+B)=|7uSeFA1M*)8Few#zW%u4bX)W(+5N}%^5YR$6c z^YYht-xy7lJ7;~tuLf3{{42|kimTJ*Rm~cp&(!Ddc(zHJy-vrH{-<^+&UkcM)d#5! zY(!a?AH%uo(FfTorLn7QBjaR(kKrJ8*vYm*+$iFARrvIdTbWmZh1bf#VE2x-sK8WIJ@mJ&iC2R0BEX zIs-7E9l@VJRePb8%MvcMtwj*wt^e51s&%6>G_n5f=dk(xz^JUC333N$3tGxBt=67D zZE1dUva{Xe^iehtp}+PGzSx$>$maAe)Tpbl?{7*GSO^LvfS(&19#0psuyRUIqw0~T zuvpy*n|4&?Bb;HEfVkzUyJ)ck7&=u?`3^T)WUbfW|Y80jZ;>Izr>iiw5_9SN2 zDE**+XB)xMyr}A}FLS?rO0`q9?dp4;ijaJSDV9`ys-Yw9|COagJdXQz@u>%HaQZp1 zlKM4&Rj?~3cDUr^PQ=(zY4)$z;fX8we%o-|tKbX4SiWJYlsN6Tq6KuSQDlEh0YcKB zX=uqZ+12NdA5Zqj7%}n+<`#o=JU7j@Q6{H8{ITTP>-9pYr%4JRbrziZ!^$H+Qcz==o3mO4Y}_z>hy= z!^zihcW{>YD%?_tLLDRNzxa}a=7T4wL+U z!a0pH2fVNilM(<_Zv0&MR^kHiAC-k>lP4wXMdqKU>8VTi;*C6VOi{B6!)M&>IE8B( zp;8Z)SbU}{!gf^s?23Xkpn{wKdnHeK@F=5EDr-PUVN=!uw1%#$*@+K)A`1ZR`2%RJ z9Z|9!mF9}0+V3-LZSr?5K3aOvQaLLREUz`Dn5P}+7qoR_z$XD_7TvSNO!em+sW5Fx zy?;w+vq#Zd``U|GrSiplPzW2pIv4woTGqm`yvD6hs$fmv#y`p~N?{-A34OkH%LvWS zO+Qz=FJ#ZsN0Recv%f8@McS>wwgM{-98st1rujNI9oA0CJRk`=+E}c4PlKE=z?6$vG>^3o8;o zGX7K8TnLJ%C7r7YpSo_xSsFxc%tDiV4^lzyUjkSiG#hgU#800DMm+xlZSAP&CitR& zseeC9Vyk^WnFF3!54Q2y&2_Yr{pX$d9iooE;R{gt=BH~E;P+i5bLZey3z$a8juUp+ zVjAiMjjXkybWo=OPgsiDPR72D+3^)5*15xKb>fa2!v} z5M=DR)MP+pK~ohDZVZXVj-8{igo*I&B1{+RP42e&HoTi-Psc*#A0hkI&R|toqYAd6 z@Uf?*3i@Dfa$RapH+71ni^l2qd|SXc^kKN9=Vi(%vb_VVjE0Ys{C!&Tk&|X_Jl5nd z6p9dcnseejHgJ(L)xdgDQWb)qjlO_%EPUQH=VLt&Pl3seF_4cw?kRBBDR##$Wx$*WYW#Iji%{PLS%RDWS=67Xtsc8 z7_axZhiuNvJg8hE9V~ve{Fj}ygtR8a9?fRVq5Lcu;;o*VzB;*@toN3;Y$>EQFT*d< ztH;=oXf5v%KJ8@@yLK-8PnMNLU^TQa-WB&KU}x2zOq`4yIbMEp0Jeai#rh(qD(VoK z!sBv%!kbbZZ<>8X*x3Ia3;z^lwd^iNSB`1;NlFv+w@SDP>>-Uh3+qYVaVw|pPX5PL z!|PM3JjEKVNvQ{a-87^90`2*IqA&TkZhy?_-v0fIf3LbBeLJwmnMGmQPIV^#9#+S= z(=_)Vv9jfeV;E-+Kl!Ue^Jv_2rwcI$jt{D5%(^73yEOByc;|qnj{z|sLJDkaB^`wx z5QnbcA{O>0izS?X{V3`hi2HhRlhOaMC?G=eNXV~C`ELT1J`|}mhsYl>CfBE0Sv93; zP6mq)3>@WaG$69>yq^ZI4G>Qv#8&-M&(*~KGhnzA_ViiBy{K^ztzmfL08~AA|Gu9z zc3b~%g54NM`{5M!9xrOl5I2{=*H9j9Xk4-^LkC(lee(2E#Yl;KIJpBCHvVS<7$K~n z+dS}+FaLoNW%m|>+9})rc?Hq15hPSLAtZBoW2W6lEtC?5RYWnBoQseGK^+w6_ZKz zs84-li;=5O*n@Q$*Bd;PstfmNfF~30DClyZKOolg<#aJG1ss5h$WvIi5l138Z9yof za>na2ma75L{AILCbaEa@tblFdkNyi$rFUYhzCa>Q>bU*NiFNaN7F+Ir2HRXi(K^XG9if#0Y5&bV7ts$e7G%`I!_a{5rLnpOT)pO}2N z0Pi%)y(&)?7fR93R$n_mFu0HFFvWbzJy~3@9>xbiHy#tcSYyuiypP+K9^tBm(B38d zCk5_;}@gGgo`EeiFzps>exrOP4AozPbId2qHu>&C`;&l zuyS(VWK6-m6=H8}w!&4mzfZnf_K@u!SFc{!+sE4!_R+t3HrUT8)}OPPK97i)>r0WP z)`iboFkV?ouFv0k09M(S0+Wj+%nkru{J(u$F5xb6{G2-Doep*QT^8}F>KE@gw4t4n zLdZq@Ay#N2e};zoCCt&Fdsl;FkK!h6%fbn+jcL0OM!RbZxYK|mudsLzRS*d`vAWrO zcE;2if|yd^HvR{VB?fN>k2!>s)*QS+(viZ7_x!!`-*`SFksnFVJ(s7=&10C)lnyww zv0f8;P+6Pq+yA0DlvETsPSqO9N-WfQW)mmgv@(isM=Yvjp*Kqe3egO}J*fW$fX0-+<%d|T4<+W0P?fyuMQcj+PrRGS!> z`$;ui+L2p*(}24{&qc#J{|C)4>D#|a!e*bIlww53p9Z5&Oli`C$ONu`_E9uZnF-+# z0bEQmRp_j`Vfa;!hUPWrt%-!~R86?K)2mdy`d#X*<#!y7dK1r};^#b}-=&fWJKE#l zPry|0(xal5*GmdPs$A`xUw-a$NN_EsofkhZAzEI^Fc7|ldHVEPpVqy!gwZn2bdC*g z`@I6+uHY+cd{+m6G~AI=uu7e<6E{`;|34M>|2owR$GrnMl*A5t97eBS9<=DMWt7|c zFudA(nN7D-eerk?`Pbz>ere5hWaXXjvDQ$brPzC)oHNA9`zz{SW)8hHCw#Cj)xeES zlOO}tJ7-F+T|J`UWkGhi@|*v+7r@7o;eY|-e-|9%1pKDX3i0o9R(F|Ys%JH?kmFw$ z-+q*R$xq5^(9?~mxETUpy8AUX-+QI{IZxbo#+(o2)gf%-r9jjZV!F;j-NCc;|K2|N zt21NK)pKJ37kEi0@0pt?lYR(cBXIzt@CE7qje2&?sgOvu67qyA|M<0N4HK zUoL$7%H{U$m`40D=8giToC4`g9bGbPlcr??w_OU4k}d!xML@!2qyqcGlsvuZ~j01_$EC~Yjn zQf1b|2=W}C!>yUko6qq9pE9p~-r@j&j%;d1#0zPV>;RJc6l)4T|t4>k! z;$k&LQ0d>dm|?k#s0Du=vZCS+V4IhN0{DLG?vZ;UbiKFI1O+XgC{LNeiG%{!(XsNI zm}|@*01ZQ5G^7J-Le=}rAa87pp=)0J(ktlOkPt! zvpxj;N-J#lcnp`*6_}j?ZH;zb$H5@x
*&W->DR+x)-Y*ESiHA7&k25a3WDidC{|z99qrCexto4 zR*mWFSE6^HgG)N3E=j*lmYdj<%U6n`j$0E&@?U==oiG@}apCvepV;AT0JoO>zZIf0t8mN$3y~(z<4$2sw0`T>6Iq)FG!G&4!E`nu5%R30Af%N7pxy@%&bC{vRA2 zBa8#0h*KDAME>U$_TvjR5HyS};U_mv%@{nhcb{_*mEF%2y&RY3*hFxPdZqQ5>MQ45 z-AZh|p!YqmBl+A{F?)6(P}d$-GgAp073zB2OZEzqot?Y7;$VP(v92e3jg!uni!uk6 zzQ$Aeq8mAItzj->;W8}O_ifOtK>ssk@Eq6ZT0Dg!4X*e>zj(lEGh4;-nmLiq*syyT;*R# ztDfzGQ{BwcgT}ysSR)}bt+EUg9D0N-0NsW!2-3D~cX7iyOZLlzp|j!X3P+C{r5kA+ z*Q>cDWQ-QNAG4qbj^LnTkfxR4#B z)vZ5>oga#~HU4fH{ASuxgb_?ie1^d}QXnf%2cX;beeY99|DFdt3w&oejf^sXtzY=b zvJjUBIW;HZaz0!dG&Lz9YggEnBmNIOZtPvyTev7?^5K-!SBcm0g|DEHFkk6Y%EV6j z&@hX?&GM#N*=pZGzeE~s$}2oD&>c7opdEL;+tuS55WNW&V4cSBFCqU!)ldXTd(SGS zY`f|pl4uit&vWG?X=2NzisCu|Y?gmNqxC^i6 zBHJ6+wnOB#B~#X$q(<{z+d!7A3&}(_?2ENW2t^EMqL##{e1vj2ZBrHI_qd#uO=;|` zqlsVEQFQCCx+|I|kxw*Poy?-{I2aYJA{pEgFZ*>;B){;m(n=o$Fl7bQ_HoM#cG3Vm zVcc$&0S{}rDk`gbUONaYt6J}9=>&~m{r!4mV)#sLIFvKvTjSm5SC6V_r7^-(i@_d! z>>N8Q0xWV_m77}d*d_BR``&n790a#vu@!m2#9W5Cw&4X7f z$-yRV@S$$>f`mrJ@=PO9D8TXq&LFbp;zncg)^G0k@-NS4}v20}R)q;k3RxW-2*Hw-Wo4M`b^0quCONBxsr_ zBd!H&46@-@D0E{V5NNUX1668K95Y5v91^q1@C(f7$!Tcqm)YhQ4M1HhPjzfk&s;u8 z?{ok5$NYE8fc=lDb4)x6>#{Y6e~CK2c=BuDe3<$Q{gp#a3D@@eWumwdr#lCPmdqLG zohRDQcJQ@qt>Zj_OKrTxRMh0I+7E?hH0tNRSUVXGdNQTrGu~w!La~R@`*lGefcRtO zQ?lY#FnqL$Dd+lvm1_ru_J%2+UO&4gz>oe7)8 zdpu3oR!w3#{iJ!q=`q<9zrFN%>7DoMXW_1-y_q^OuAb%@{S!plwUoz7U-~~<76oJ@ zak*do)CR6YD+5@$Djxc}c_qlf5(_Ub?8}XgQ8*%d19R68425`OfnJpBRVsVr68AyI6O@l?kyj7{0 z4h6)(p@l|PpXX~{=5h0397sH?Co;fo)iU*G{NT_fn=?jVo~z*REdC{1_TX!UhQ|6$ zY4;;<1L{?V#MmH&Z0H~YC`!8c<=*2-@@Yi6)l_%jmMX~#6cc%9Kn zS=BWq>dHd#RN#6Uyy$k62A{7tOwRsPArhaGTfDvZ3b7^d5u~t&h zZZ$LVO*m!8ce8)#8pUwc$)xU=tXb~b7 zpblDmqo0Vf{^i3AxOdZW6zA~a-}&%k*A##d*T%A6fuvi%<&4*gGL?-?b!|=r4(BkE zADx9K(S30_?y_wrz+KC7fL`=oO)CL~UIOvpOJ}wj9UG3tAopGi2@npM65SzltD3PQ zkPV4b+BR+_#hkyZlSTu-!!>&7@(1up`zhdRHr0|*yGWSm&CS)+Q>ngb_f1v~h{9SL zE6D-$wtROE*~PQmtH!&t|2?+3{eq#o zi5Vdfc@^pk_IkZ_>~k=j<1aP39gmk+xPj|_Df{Gx*HBmho!{&OzXS@0x0qd2JaFHW z+|}wk);p8?&N+qUF|~^^f8)6h=4$riqv4c}+mgY6S72DT`$nMX5)1j1xdqWRTkcy! zh+Y{Ik!4-}_1Qal$i#kdDW7ceH<6jkgX6LEIgCFA6K}w~KLVWv$k0p?@ZoX1lZ^sP z5PE!XgD6OJ1%|xAf^I$1!$;vOVjoD&A6U6%-IJHFi#(iFs{e&s*p^Q*N4 zDlBKew*_ppf%rwz`!M{$90=zIdi=rX;{~i0%fRqX{7Vg&*}|Elx{#Xc$f#M5g| z35s8!JyTge^t-cVlJ}n6lV_7+LF#|gWVnPGDd!Q^ui{dJ?E*f`=5l4qK8@CsCrm*Q zs4%hk^0Ea>`rHRg0as-7Ph2+nZyUXRHgLu*fp2bmNWuQUX=?~i$=%m(AsZNFgHfEAC^V7;%?xCUd4JetF_$POjvvhVz6E}n z!uI|$;W!8sl2;4%HO~G@N^teHFOd<-ix>#5ZY>Vzt8H6iW?>wdd+E<)8SpW+^2c5B zRRkGHROCRdO>%KS+Fy**z@^VR3pZs-^G1evYEYA}SFCi-^MW$SN2p8jyh5DWJliy-D6{PVw!(7s%l-oKPu z-KBLGsHpvf=0d+YWamI#HupuZ+7Y~=BEC4DDDGP-L(0HpV&xJ4$*kYL=fC&BvoFuI zs537H3T;jLIrw&ZR#ASwBlcD&a@CG>{d|xF)tkwz-KoV24HxG}ARon70}Ud{Bv%Q+ zyh{H)c<-=l`;d7MG&to*(fRZ2N_pPnDaH%hOv^eqcFvc=*@A*Oux_Ko&0266{cEh5 z=@jDYIlLL?w)bUjvBknod#VTRo$m)MJsY{?o)8@W?t&;LiRtH2+$GoZYT?nZl*wd~ zb@`8BeoI7mZe?!F@b(6iBTEJ$Dzr~1T8c&eiqsZIJTf$ z%1oSE;5Le5&|NKXJahUNC{Kfy%mZs>5LiOZpa%KR+7=?ivft60+|>yhj)r>K=0jZ~ z-I{jT!6KhjX{21+hUpX+!T=gbXC+LK3OZxctnp^3OR`Q(Gc$g?Gwi?U5^?E69rV=1 zsls6o!XsXSgHipl<%nBOF!diN-c0*3+|3?56))rEb`xejr&zo>h&KAQb$U;LYE}5< zgzOGHlDkbjmrGthcgQ7#yTOCt=V3A%gD*)=u3bCL_;K8~)ul`MA@WN(zs3J*=9s4< z8jAWTeL*N-cvZz&&JmZ%+=GXPQZsUxn@o4z|I5h-q~?MTQ5a@aID|7+6OVPH zo{=VE&vMl*XO^D@Q@!Gf&@X0^Kw=76SCkr;p8NWAQTaGR|1p(03xjCM+Eo>p5(mB` zz;ptI3*{!x&Oys8=d`{j@*F!=PMR<&_D@))oJF|k;!NoTf7gc^U|OHVUP@6qGWS(XKdTC82Mhqpgz0D+h^0^2Q3M0q)9yQqVS>?JA!LS`w8E1deHz$qrF~!|2 z^*zcIKpbp9JdZBy)rn40+GI29H3g=F)dih#OR~e>mE$>DBTDB)mzg zG6x4i{xn>&V(wH0>rRjR29l$%g?Mx;X47~~=?0SptZ_sy8C0&b<|R0RW$$2l$nkeA zX(_o(Wne)xah{Q@q~q`J#rj+a%|KGY4LhxTzNZfJ)LL}AovljY#Q7Wdkd^bB&#tnH zaA2CEXF^!LgP z_!O4D#Ik1Q%u662i_Jdhw?a}RD228CM8%F-O#TIjrBW@Y0gWSOJ~9M>)%gmUZOyv% zVYBz0_LULyb0xNcg_T1p**RDm^3>L61^dt?8>P(_wC6MvMO^1;@|i2v)%yPIRfYKa z&Mxe7;JTD07v?j@7Fe5;{~KgW>yR*umOb#bBz|x}fQ@w%`FEA{}?yujM^B^AE z3r{tiZzCjbC1!ONsIH6agRqEOMK7~+$npCVrdRbPD!;v6 zultj7aPJ?WaR9cdPxHYde9;Ybf$TDoTHs6{dhmn)UJ=C3Q!sPpp@AV8U-^OS6BqztbClhE9<~j-PjQbUC?jhE~c*)qNl` z6#MoOjy3=l7(B58u8J;7d)8nOu#V2exni@E#+0j&!7tSz?7ORV`O+66caI`PJIc-) zBWX>YMN)!A+ivL-AS^f7o@3B%$g%tju`xdrY??<`Zowq42oj@Y|39n|x+QK>%r z_T(8N*6s6N`s1+QpHT8R)2WL5rJ_tt;M}-PU)ak5`L%li(X0`Nw{#YXYA?Pix!Ocy zw#M20AKCiOuueNx3?beY^$mH?2psqjyBhrm@7ts~3dl$3?5M%>xi*}>U+*Yu_pm>@ zuIb(9_`VgYlm0EQFWxUk8oGpM%Cbn}-SPO|`V&%v{Tx5{j)bl8AH$xz2`0i;mgFrJ zlr(>OQq+8*Jus^*&890_I)B;o5jL^j`6+GBO74{N3dKX5m!IT~J{0_($wMIt zoemx889M7to2Abo&A_7@)Br(?feSYN>NWj%U*x)x zeoPz#t3Ad_=<&~DpTq+U)^^*#8i#ZWWcgmpWyPnX7YPbpbe&B7h?sr|CFBx)d+H`Lq@OkcBEY6F?e0cucUzi-cShB+lssou9c#f)p zt>W7s(vM`0OYZD71X^8K2v1gQ@q%D)>JOS^7JoJ@LCZJYJnE{q z>3sKk`-LF#*Er9}!JL`Vx_%57pMN4WKkjjNH~jKe$Dvd2z`c#ym&S3g3HCmN;t^OD zZHA6a_8X}5^b>GnmH0_O88Q{vDtt|MBMy4Ms`49KHz+wrxmsiL-(cz^cg6NhS_82* zC84YX4mN|JoRzQfghlqa^0h_TWYg+yg!1i&c%1paF$EAG#I-=%A^ptcS*O#Bxo2wvZuGP6oaJgFg?N>*WF%9gdiATnMKSZfqTfkHCFV74 z4kVY^=4SimNmLDXW?-4lgfHs8#Bsu-@3FSuh%e44)A$%2W!ZJC))T*26o9DvZ*jGY zP2mgpS>hqbwa)b;HyTG3!e@~FP{mtCoWafu8sDE~elX>$)b{7@gs&F1xKW7w98Pg8 zE%S+Czkl${R<$Vg_QYGg0cTom-UH1&%v!B4u6gUNWC?f6f)`{J?s;o<-Skr&3WSCK z{2D?J@9zoWv=G$d7?A?K9d-~WqeoZ&&9h=YNdhG)HE>2G$zPq7;XR3xCbD?x-Hz$xJ|I-VPN zyR2B6){FALykIkmYLTF3a7&>AnYhKnODjZjZ zUM-Wp4=-)OXCWvyJZZEun{rSICiT1$^mm_}EFm|^bT*&=aY-H69FR-Au4rd%^sSyj zB?ZOHOaOIKbm}i5XqUf@_LB#(!uU)tTJah`ei_q%`eQWv!|W|&OfDx-oyW;GqWa_Ek5f;}S$-zj1;jRD6$Q1Rw8XxHroZ^P zs1NZ>Mw@6Gc5LwaMcx3SsJE3}{ z5wHwnZHl#spcPeuR!G1I*stb%aX)Z)Q||}!_g~fEa0YJ}ay9mL#jx8LaP(|y^Lwb} zCd~D*;xc>05)DQA(%ygE%Vy`_J8&XHPbP^xXYEeaR1a;ykFOjdojO*ltM1`w&Cj8x$~hLhv8Z{(|6=aFqMG`mcTuHFM?iW9se(uqkkC;P zsnR7Ni1gl30wP_SpeRU}E(Da05NXn+3!#TzrGy?xvhT*<|J-vP?!z79zTB~10-3vH z?^WiU>-)Z0fG_>BrshC1QR~>EeiWJ-(W@zjYEC{FAg7$`M7Af@dCdn~>)&TeMDXZRBY++lZoXvi zuZVm)UyowA5@~z#mtsn*`9~FPOVy%hG6x6<@j5m2;V49tkNUfOJHJ9t1?us zX=2we-FpQ4kIMfLC{F%@E5r#C*mCzs|HtI-k`(Q<0$7V@E&(3mnv|Z8l12d${!Q3i z`kU9>GyL)27)m&gB;BAAB$Cs**R8j-O7(Es2L7Udx-^^K~@;%mC2*fZ?=$En$H z1Wd(-1Yk0$D?O)Kat#@1tb2fI`$E*u)=SaTLHm-mp8c2pYAVv{<-py?ipR~1h%RPG z46+RNEr=sn6f!N5|M|8Yrg&kWNv}xs^j9WxQK0%21iX^8+R>xoc;ckbFBFDrKh$i+ zaj@yJQ%4lPrTJM~V3$I2qS(6L+9$#*(J4197x;w?T8N(-0+$|3d85L5I%D$w0a^7!;0Zn} zz#?5ObTju*OHjCHUB@GIVTR$X571j$yBf7H2I@OpkH-Hhddtp7UW*lwNLdWM=v}xy z8xt1ccJGbyQNgbtN>z5;%Xre5M_GNJ$H&s3AG+nyQCFz?pSivq$3InG7a{T^2*7Ro z@o~McaST0%@viJk5n=zd*>Un6P%EXtZ<)(Y>L@B}b9!7AI!Nyp#p|^XP=LwRh@s`5 z962|b=L2#m6Cu`ba>mYZd;6uYpSvahH6!98{hncUb@Foijc+*~3wn^O^4sdIi@6>A2MVF!zeKcl#B=S*)tSMvzR~-onyTp-oeqkd&~hf8 zf0_is&}y<-zQKann5il=cO&%fA89z&8oi+r1QKKpq@_}sKi-U*Dn%~_74s{>0o*-Nx_BM7LOSt`4D0;HL!e!n&SojPpn{>}oa z6Cqj_kd2L1Gcj|&o8rMK7}0&tdL@c&UjtLDk$rBw=yFnfIHFRF-ms!&K_+UH{=#1T ziB`6!kxKnp+%mN`1Pk9wU3PMgO5)&&z-8=x19g-Ali^viAG`$;+7O%8p3jy(+SNG5 z>=n`sD(FN7mMJwsu&cdy&c>ZAtYbcO)E2{d>>gokxLoi_n6f&)Y2n-wMY}nK)bZ>Hm(m`OnW`+B zsk<+9bs4Nqps5^D+#4hmD($JXz`a32{XIyEf)GKwN9pC!Wdk~KIQJ3;#RqhXC1MAZMWV42kPD-z8J$jZYo;4jBLWgf&dB*bh zARUJ9FIEpm7t~H~5L*$rv0*vJnmRrB{fM6Y+l>1i4MoBlDePg6{k}#`M6OJJ zV@z%5Lmy!L^=n_1*>@Dl2FyP{{Ob>*Xj1V;!1S0*Zj(jp9-v1`*Oz*ZT==3tFb6~= zR1v>%RT%^4O0nVOvtq#GJ;A1a_J>DL4M5$9R)*UKWwKWB@9~?NrIU@<4S{rN9P(5+ z?R&tZF9R=3E#I#-vHNfwx`*`lc?5W+iaHvTQ>?oGDN4J}REXvM%Ks11tW7kEZ+CgJ zmj?D%4-f;(74&rS7$MdykaFqL7C{^FAe1lrrYrA%@NBB`p38l|KapD;pNt-Z%0o@bUNgXhVJ?(D#PW|SWd%XHcE za~P~l>}&sU4}wunVDLERYfS^w}+$J0*8Oe2X8B@2e%<7UGH$E9))g%4O)VV`&)PzKC2c6D| zvwf=AUy-9mvluJe_k%@d&)#pHdHkqel@Hx|Oz->0)aqu(-eRviO}Gd^Hk&1Jo7OIL z+gjbT>1DC(J$L<8W)^)g^?c$SbaIYDF4Z0iT1?#xj39V@!11ak$4#!{y*)oSYMWBL zYkS*Eomf4sPq?9UF>_dkMpOBaA|J@K09IVt+Yh;dS-s}GMDSEnUE-&ls`gn@!AX_ zO=mzgV7iiKq@f&b>f_N!+>Gax@HiqG}a45Mec14S2eZQwK|MMH4!T-H&MA zr41U`w*MKinSrUj>6g;k0tyX8po}oQ$Iq|hny>;C|>K9`D9t}JS@^b~{ zPhp*kzjH)L`e$CPhPAqOsN5(}A75@Xg35_~xfy?waX3(yyy(@G< zP-0;o-(2Kb^ezAtOP?$_d1z+ieeXFUKXAXSyx?*t-v>S(=&GIeA^Q$#Gmg)wFjAo} z=J1D5`;u>7xd}FMBn{Y|I>sqiDPvWRAJb3gUrQc_Y&V2I+earYK0#adF6{Hw+}O6r z5v5jup0ixF>C7ayKDjdsW}fc$?&f`hvAw&&MoLDOg&VT=Fm4a)1h8?nN0Z$Pw;0Zd z2t;vvK%Wh~Wh;{*kg%U?=*E->G{v;#;f&E$I4boP8~!qx>r`yp>Yluqq8+CMVD1Qc zq;K>>X>qWjkf-ALOf4!a-BnCoSA+VnC-rFmSf+SzYpdviRApM*w9)>jFq&~TG56L7 zE88Hhg+95hhRPMsRqgH+vR~n`MSl^2`Yi6f+n66L7rFD(p)!0|YSvq78YAHny^ONS z=(T6z?gGdTb7A&`Qatcl>1Ao1q*t{TZEkl`bTF|95zQ@>6%OL{5L`|RPM&c}|8VZK zkX|bU6{B&w0OSCCWysHw1}}HKGrQi){+_H;9D7y5u2Uz;UHFRNl)`Wu0UMFQ(F4rMY(c; zbL@y1q!l2{HZWf{Yb zj{(1Z6lrv0s;AKJGRUuhW=suwP;uGjjFRPXn`$+w%T#6SrYuI@^a-Xd#%%Q>S?OT( z!^RVW}fxdTQq~0p7QO{CdHjM!3~&2iiWmuy?mo-DNje7;174~ zNFtAuqODbVWnWs4Tx%?PQQxXHyp_D~d=xI3 zJ;(*>b0#O4E~uY`d2s5}KD76{`G^4nrrkc6lLRdQs}k|9=Rzq|Xya4Tur`kR{$#hn z)kRkiqBLG$A;POCv7Y_5{-E+_c;gx3ZUV6Ru~=7D=4vtSVO09ls1t4`_$*iDijUuK z$#y-ao;cCT0eWz4pwFopdhFQE4G>o)ROED8_uBlT-01qQQS?t!o)pZ$Q94Nimvxzo;K&& z`5Io&suDetI1M~j%-@ZY3?VP*Lrw&uKWiOs!73hq0v!9vDw%yg0VUs64l2C+1gWnE zry3hBp}4=t9D}S7XX^-Wek0%yGa>b}CrUT3f4KR+&4+cM9H`Bh2;~!a$6AadWoLY@ ze;pW>hl1J#e=@rIu`K}|52dC)m_KxZ#r%M|fIDeU-X$0Ytd z-n(LczZ7Vz?^=~ZOc1I}m`rQ1A00%6e;=w*cNxVT1#LiU+DOvnf?wcFF+xVI8`4eE zu|LSBKlCq|wH~6{_kKu$!4KzKYrzwDfn9AO~y#tZ)BAj<4M z2$vi=2*vdGA?gSF?)A?SoVm+B1P79{X}Qd{_#htwlnMIVQ;nO;KOyJUeSHl2nyqZ7)RG^5E~v+R5| zz5>!rAKa>!ZTo!idZ%2roaL3Hzl0ng9P&ZVzs@b`H}|NJIpyNx0ot? zlK%%$1KCjNR#p=X4pv)rdT@!p`@MZWGXgh>4OY0ziXxHML} z!|sBi1KzUuYGwSBE$AhSY|Xs;fX7yz@A*A7iSqF@SuIr^aKQtc&AIdvlAHhlUr#BK!xM4KOMkF9o@KXf zagzh|!}aLu{Z=L{hvjcuj?*xnxACy<;MflR9O}8#?3Jy#1JEUQvf+;G-JD z@}gUPZeUjHCIRfP0BQ=*~NY!calQq3}?$sfb(kzEcaWn>T~jNC)4IB?h}Qt|d5G zzPyKzWLjwco2r43q{-Uq`FM9s`CiqXHRk2bw1HNd9d**PH0;4y^sunU>b@KC%VQ-} z=t6{hkRN`EDUHh1T*mANvBsYS7AgC)Ay}V{4EwvoBAQv>5hk-~djzKL${QYtz)v>5 z`EN-*rip6zHe~D>tq(f)axfz>H+H7P=vRod>orm{)_Fh=l}deh*=JgRu3v3>EVD7l zJ$R(xY4PK6^)p^BUE=aY%t61cwp`mhvRjaSqVzdP9e)>(_Qfagy1*lmn<0uN9)>CV z!zo-Eoto5Rl6cOnzcV{NaoDwf#oJS*QaAG4jQR@(nTc;XJw7Pl_~ph?cuLmd`kbX8 zSD+9JGGp<@XjA9A1?dwGWnNEOQ@k03-QK?bkam&|wg6xn<)Fw>6cTY5=+5R}T6z}_ z4CWiSe7B?1>TCN_BS~&yI1T)mg1G-!ktip-_2x@0Ms^2#Z*)zOQ~Wy`MGU$3)8%zI zWJAk8IX4My_eInLziJ8pYP6bhKo9(Z$W2zwToSHc6>MD0gw88DYp49;j-=58bm}P> zE*Xc#9?rAgtN7yOhhk#^LT;_PA>)#uA_--hw=SqU3MN>P$p0{ns7hVON(j;1Guw>+;d@QhUX!AM3p%|{ZCOT55H^NJ7vlHBm#GIoP@!`%@PiLr+5KjF81I^SP2 z5xJARd$(Fs5Osd-S}l_>1g(byhbf?5@5qDm;Lh`^gjAQMcimy=nxKCAbEo-f+k>c) zf7#sit8cu4)C&kJIUw^P70(p*Cm94>^WorZ@uH;r`i_CU$I(J>ifD7KV2s4pN*}Xt ztAkKAFko*4!dOa{9hJ2ILOvfQLtz+W#IrU?w1}B6CmAU9Cf;fP_85r|{(66^>kQ=5 zZwt|qJrTB6dV>o(;qid2)*D%N2oUfW^0)FW$s)O@GKUwIuR=!u?y z;Z)vpv_@C+7N=@Q9>T`OUcJ8ctXlF@;4x|aFpY&byFx7r^NXf$2@oB|I`*=W_1^=W!0z3{zIcy|xB<&9yNl znjmKZJO|DAH6xOGjz|Gj2FBC2&`75}1l7I>teX|zpIPf4Dd@x0Xg zh|jc(+vnV%%iQJF3qlSkr>GyO@&8-l$@5L(CvC+E=0W#kwrUw&1`qz{-Am=@cY2#VTz8u4P-^8z@s7dLdh39!_XtCW3JM;l;$k;|#2E8L@XO{#3S*ig&p zGJD zSv*yA8*uE!2}~S3-eFohFDLt_h}j5uudBRhZz*x`0dX`EPoMrZwP_wMS(nao>aEnY z*a8KLjin03u3!F?BEYX7WESChkz8l9Rxm%@t3`_EF|bwcJ2$9-O!G>J8D#d=%DH8l zI(<~P{DM3<>@QK#H1(kTz)#UD6p^4V0Tx%@98Zclm2gjACWZuFSHIZ%9e zEA_xGiXOK0x-4W1YA5UjE4wjnAU$d3osC1?A2@=6)N}9&*l_#GZNk{}Qt;_EV()M- zBAmhfkT;E&EE^s5Lzpf1?pu(m+`tgFkZJ7WE45x~A)ju1nPI7GF9vP4l^wjMfDhMt zai*29%1pbjFBxQ)~pNBRnG{x9#-;VvXB;v#ky_VKZbwwf;cr{Qe(-)aC8 z_N)1rHQPT{zu{I+M2yh!Pz~=fd{VzUvnpNI;U7Q##I)M;h&D(Br$^i^Eey_(bom$k zfXTH|SSidQz)Sa{e8LMh1w99)(Awum0ix?SF2xw==Xr4ODWaDe4Y3E!c`h>Uij@jB z zpBXQGl4b?75y={72_o(JfP5z->QV_J4Df=du;-aGO^mhgtYE**SI%dSt19wW&Ivid zRq{#fw}rKHiUF04*LKR0$VOTqX z4ONQ}$)snX{VHDclHJ^gOF!?RlM90Ug`hP;BV<$Wm0EAPPkT=!nGLj*#O4o_%%G?z zLx{er?U&8|yL*`rUoOIL=77=>qks_kt5u=k>&h}XxazxYJyhw$%MB13I{g0fehM<8 z6q5Gy79TLLpBD(NoR3`+>9?YRqPQiie;oPnCzTNW&zC|2rA2vo{+BU#nfgI!-~UO> z2TtCXt$}KeA84=N6qPR1%Tv$riUgUUtXG`h>Xll==G~YCg>5cv@c+ZNFEbqKV)K~h zO1Y|UYx7%{$5-Tk=>6gEx1Mf6r=d@%YD=nkq$K(a0UlOxyIv587d*J9@jAfB$!7LT z=9lV|^yofX-5~WCk*uV}`X)%4PdjItwk4-;{8jW-r8@45S0Szt z$kUQ;x-PDOgJ+W8Dx5aXd<5vki~>>qmu&cIVNw0ByiQh^6O$ny>r-%Iy7_K9shK=D zZp&!ez7p~VRAU{cHu~5jqufQ#P=24)^pZ@c51RHKcR%7sas@CRF@Qw)UMm=xK_%q; z#1#3##I*f=8m|o_50Z=bsuDxY5kgbylD4lafb-Zk0-vX|Z4vlB{ywJs1fF)B#|UP* zs4jx@c!A4PD+FYy%+)~gKgqnx^_{o=qy9_})kp1RILC>8Bs%88`PF#ZtNtEEbyEK0a>!E zu{R0jei_h$9t|iai;GEfCcV=Lc-a}+5O5Sr8itTtwkJ(oGTUP=neB!E$ZQAe88T3*nONY|KE?7&Te-Z^t`FQCW~aDpgztc9yqJ5eg{oEXxS4+6$zpPZs~xooeKxkA6}fE z-{KSBcZG)P6M~EkEYyCjf4e-6Y#Q6z=V{f+q_jBB*goFIR=QGgj zO@t>Orf*4(CWdrc3!vpba^A;{!P#q5zxb3P)^Vp93rms#fg1C7fLudf01(bmYc_ z656+p^^U)^5Efnroj3Yd~bVX=MLis^_A$fzbF(qiVlLWHJxf%VLswJ=1 z*FYlGWXPD1UtZfTXsdyeAYnS1UHjgDM)73&f<%V4;Hl_6buRwbshGi2DJ7pUdGm$l zu9996h9n2ttV6zt-^N=@Zbi>&yY3X@iUHA#+(8d7G>oDGm{-|CT&45p^7`aQdVifG z{WCDE)Lqf5oV*Q-p~5{^UKPE(tw+$OEm;z5Sd2`a_$1uxjgjyc!n|K$)w%hxB zwyHaNGzU8Iw6+OL7GGqf8I8N59;Z`J!f+QCW=v4X>pYuJ%=#7hd3DVhMNPsrdBCFMtf~D3ItT?5nT|n>8E-n%;p% z4?%gwd)_!nMgt?^lO<;(-lg~35{vo+ZH3PIS9|FbDj1d6SQ|j0iC`yKk*pI;no=$= zCFUH|0IDikT`;^=LSwS*LY_^Pa9_L~1wG+WE^+x^S^(M|buCelA}N_7ycoCC2}HB|A_I$o#LPY7qH%D{#lu2Rkm;`pm2TmOUM~YO`*@ zI);6&e>1m7op-t-GNbGtCw;UG=Je>fle=-KE$ah~<|j6Nr1u%AwZ%Nu>K;AU{QO8J z{S4F^Qw0$dR-y}~u&~nh#gD{~dqCAPz^I~xGkPjR2Mjp7tzxz`o=}cw>75ABgO;42@Bs(zBUB2-x;?$&k>jd7Kk%ik`JMe2;#EOQMG68;ybVh z^#B%!^nsbpjp8{{k6@d?6UMDX2^?V91l%(EjQ4nO46JfPNc7hjOkA8^9$Hf=$S05R z>r{UlyVLBL(aZ)PVYmyuW%S-jM0=acsdd#fXvCt>tXUYcb`u?)DBX91bu1aoZmkOKf1T- z+8dQWUp5Wn)4zpeWz?mi%{dHjh~hd3CncNwl|DQMn)!c2KvHPta{@c9%+lI#?FQ^B zr_4DJ%KSTle5Ju}->x5*zADrVF`;c-5M|6gFfoz;l@8xIAec{s4(s2(4(kG8Sbv3< zY9Ob~4ia4+8oan9r-vI9HU}#)fE-Z*&w>%_oou}e3U)qO8(t=P#-H9Geh=za;&^{L zM;+8^AGs!df?g6gIx`}pc>YpG4x zwWwu~X0Y+7ljP9~K{3H3t_iPy5+R(ED6>i*N%{g0W?8_&^j}EjL`=3uouPrp&G;5L z-k1DPxPD5bb^S&CUBHc+dKv8*AK7&kJJrox2B(MD z$jBe2h&*91(4Z~l)h#Q@W#&;4x~k?es?MD@oT+l1ySEvr?N@j)G`knVFTo!aWPTXp zF?cs*&u8#=M(wHcX=bBOG~Z{bGe|{MSBj-%k29CfGK1EwAy0-w@f~NxF@F+aKc_<- zafTbS7X_Ie-wbIJnmvZW{}_k3frj-Z2E#&3&<=rMgw9z! zSn;rns|&iywF<3()Zj8D8GV>$?cLSF7Tmd-($ny{%L-w=3KS%T={zds-wVYVCV$T>^9QiOyJ>dNiLSQyD0tR7nMr5c(^a%}0Q-5aD%q5)pgGYxhtk5#({f1A63l1Qco z?_(`5clqGPCUcMLm0@OY|E$wc#J_SIKa5g_Z5)!ugzM%&!c>px;M<4MUg|JFiZ)qY zEWu1|e4R!`|0Q9}zo*;?-8f9|$Q^=`L@2||n}*kEbUuIJdseZ@@~8Y9+cv}%)tfjA z2z+T3t5$uqU>4HxL2PnMdBqDDsO)LDnUN#(jneg{-bbrQk}G1U9vc0N`L}nYD9V3w zYukLyA5BpmgHs4D+swb{NPP9Cc5{LVtVDSE?unwz-+og`spcswl|XBuNk1|uQ5GlE z$)t+tW1oO+ZZd%ZR)dd+usY)6JM)hrDngB8<%KNF_a z2H25v(`_gbNd-*Gwn`s=Pi_qgXoc$kys?Eo<%9oJRa0A>wfpg0QI7SMVvKBzHY@z7 zdP#T`W9HB{b^iJGrXT$GMPR>wC56t63m(h&O;rR-MeMqa5k*Vvdzv{kFRF`w0EF?$ zT#u=`pzXGW2YRm;W|^ziF{YS$ZSo4p9eYO{9UZ~;A*VcjhV@(KCEm$uJ!6IPl0@#m zS_SZ>q$Rx{SF)#FWTC|DpYp9ixkI&4rUqO<;Jn8>R!-3vMMly6!>4aEU`om77rfOq!@KieP&H-OMb zRwM~7LU>XFNIc+W>jtT!2MC*26&~b3sM52(!3^Mf)mcU}F!i8kwv0Crj*J z-#Te)TUMp{WY!dV$YbSV#ed}7fd4&uSwz*_kZpC8B;V_ISFr86UVHcQzFo3>Gqd3# z+)fnRg;)x<25Z^&uE4Iec^P265+~i;=)0gc`PkH>``$u;0wO)3JII6u??apV z8cB;!H%YE#&U5NwxT=x?0iJgn2as+)D{Xx7nbgvAT6MH3=0$I;CY$amG%WX+^b;*n zF9TbGRj?y21>BckRB^;H-kuC4#pC<-I7RRTkS-`+ilne?7nFCgP#GX3Sw*e9Ie-)9 zVc9Ot4&VWGl_mywp6gLsOLR|W_O{RgKP9_qES4ad0hS<4q=7#D8-p-ngW(kw*O14M7htw|={@zb zSQ?N9I5;8XKJ)R%9poH9X!Wth>W;XAW9>z?2fX5%&5j|DYW3*qsIm?`!YdcjnCXO= zcbsz&eGfU;cLdEo+UmtlYF){fIl)~MHH?E(KcLUwD%>GZ9-AKfY+PKsM#`Od4SfTz zO0_f+`;HL}=eyp@4`dodUj3yrBaElLez|OU=aJB*kHn_I2w0a!!u#S5^tAaKzF?&a z$T@z4RsijFaB2I0W0h0t*6-2zM9}-?ecmoqSX7{I?zjB28MMg`UcxxZr=913(igYmJuqA)s!;X6f835esC(oPtG>draXzz}k!Nda z4E{q>j7$%HLv_4)(timS-9F6yK`hVrp6>yK&@qaHXfo#gcC779KZCT%XT^3Ni(}|#t%u^LJSKAEU==?tp*OGBxVG7?I2o_h1 zC;ZJQTs+!*PFB!S!dU)8->He+=#T>VqQw9XDUVdMMb$!$57s;HsppdIA3dA3ow{3m zCKs)NM?R1y)xr#Al?9r>Paa0QgWQBf;>J!xv>=^6Hp__MmckHND1EtsmVLO`G^8*- zN(0?p&7E#0f)~Qe6T~5W(pBQy>?-xKepY0OOCIaJB}W#j!uBiIb-JLJjfVHqVo=Id z2rq(HfZp+>%=T@kq!YHUvldJ@`=GkUTQeC{dH?Qv_J*MaKEwWRE_ZL})Kl+Ry}6+{ z#F&Bd;*tMZ`R~H~17QDH%pr5cL5hhi_=l``03Yz0Soop6ad1U0!yWI~RhAe?LrN0z zRvJsb0@u`YTE_3zN%bKN7uL_86GfiiVR4p0>@xvPq{UFGz1s;`8tyCCyK)5d!eWgI z_X|zU4X^^&`yd?=F|h}5uY6Ze&OboaELVvx*+XQSm`g@EhVR#A(pof_c6ZHhrYb94pEAa}%2PmG@PakGpV zZPPr*Y|1@Z{ryhwOj4^Cm=D+X70+OK4gWPbRuKQ|(~4uVO?Gf>h|VG5two5ul-c!fj`6UL zQiAkwvjBOfUV!{g9ON+@H-Q{$_Vaevpu96Fc`1Ez^BP9bL5NY=e|B37EtleTKMh7M zl?MCFszPMzt6@U#&`+_N)AlLcuqVF)n<`17xp0Tp2&?PLe!UCTPT2)5CZSbjW=ZLK zGj*Vb4)AoyW%%YOig$RcO4mTy$pE8gi?%mSL~K%oHZ|er_{h4%VYubD%f3OQ$<<|t z0k_Gb^TzLyyf4Z8QwQD>R`E8QP^8#&AEJ$QhE$wT5(9m&O0VObd910aSn0uX)t;gpX?G5KtWMKYQwM( z@qm#aAKT1_33v!G!b&RmNf?3GNjS^LE{BfXi&!)P%V&>3Xhng)_wf&CYw>>EP3gkG zXNtq28^7Kitk=AXGG_h*lPN?9yJ0Sb^-ij%V-5Y)`qbbWDU% zsb2M&_UE<0KHw)kCPy)TX(&daK=zc0ux#s*B8_}}(UP8L`LcAk5t?*g2v4Magh6^$ zJx!22$G%sU9x0s!CmTQr$grd2iv651=xy>9(t2_{(TYy^43^FTUgEahj{Vr7B#Eqg zgymNrY@_JRwcT2+zsFnoy?BVN-9D~uD-LcMe1~CLFTrTL(&1lR1-cOCZ&QF?Ky24j zaqN)hW|>b6Y>BJ!#p;J|6T8MNw2LO{c-3x4>6N=0wiCmVw>g{C?zu(H7hTKiUnA!V zjFCp{3$k@WcvI$)ullFe1pz4~1{izfU`iVep*9*h?Dc5v=f}V`oj3#$I2+uTf8d(# z!d6b5wFFmR4!aWH>9}Cm5IOwZC(f6g?k`hkQdA&r=)*0b9^(kEk^RX8lTy=GBR$)_E?bg}~@$Y&r>A!%2 z`OR+mBmcRFpEE^o)V(mohV)xnbf9%F7S84wyO~rzEPJU7H*%!@QF1LbXXQ=A_-{_V)tlf$%Ua6pxy_Cjz=1 z$!!TwJ#mXK8eajZ0oT8!LuWsoJ(D?1x?wJBqR+}m3=!rx5Nm48bCbr6Rj+4_=jjxc z8Ger5%A3HpqwbFMzN1u|5oY}1&FbeK3p?p{#y~y!)Ot(UQkBkpAF6gYblg!82G=GL zuxb4mJ{K&=Cw`1QO=k5PKdIAneS&|2mk}qDd(Be7$=)_E@|gYa#{hFJ!ByzLhWahW zDfTUKVUedRfKJQME)R66L8n^qWe1igtIMTW7$_>w;=|+p+aI_c|3SFzY!u8K<o!Q4|V`NJ9y~UfHM185K|I0jkV+D7(cNv1o9wwPF)~pUdGMA$DdSTi@d5 z!gV1}&9>}OeRpuoWE}!^v8=D#T$6Lz84PeLr+7++#5~GCu@L=mJu`azYtTWB@K^(V zw4MXt_a->clP4Z4cXc27%uq>mwQvtEHzD120z6RDc?g0mH(8xQI0B(~!(Z5S#uf{k z$99K*7Ql`jpO#l{?;SA#gi&K$uvOpzx7M&@Hs1t{#c3xd=;$0PcaVhYNkU4SeXOZ@ zW0LSO(LzQ0%JKpIaI#kV%SH7aR0>xMa=b{Ybd3oQx*`&MZzG9m*yt3WhPjq6;|@4l~NZ;j44e7AHXoq+6h)?*+(ZyX`siR_~<%!}1}`vq%4ik*vF4fw9a2{yK%M>}t`sfa z_r6>s>+mehp`iFDC_GCVfSr7=czPOK0oI2*bn;sjt>2uecOPht^*~tSr;)Xrb9>J8 zTRRI5ga*fZr#tTeYDXpV3acVcAP4nIu1>=2ETO4In!|AI@fpU z4y~-+ds$rDJl9f@s8FUm)Is&^du{CDmB+~SoS}uAM-&&{+3TEKj>tAfMJz|}i#y>S z;VVu9u#s-~sZ{pJG_k~_2}B`JH=1jHnhixJ@GEGM!lHRmqW**wwy^V;Zfv6h=n@S_fCGWsx=M&S@ElCLus9z;v?OsjwD0Przl_pd+H4wpx|R%}x+t7J2(@9M;>n)ZZhXXR{9{<62nW+%CXD7`rveUyita=GpN4F zI>bRIRW4sp;^M}%lP3P&sSs-?FCgCsX=9DPoCZ&4*UlgreBNOr*W5+o*{aixFD~)s8zkP^nX8-ZIQr$iR`*t-w|7LZGsn@uL z1VY%AB%syDCB30j9&kImdLamDtnF5=tJpusZ(;xjITR?l?Rq48qTMOd;0U(y70P2K{=LdF)Ae70P1>C!oZqmA@g=S|yZ z9Sh(m^ys7v+gV8_ua9~p%0jQ&s^8BNy<&Nu#0;6_I_wKN>(GQ-H@SUsc6jZ_J765{Tz-mx>!$x;XkCQOP3#5@O$-JoAN638+Uk)x3CKf_ zyLv83fq^be&CJ^Ipf0v{jXzkLYP?WlyhvLqf19;6ZytXNvus|iuuvpIeF+^ew^Tg1 z6=?ZW&O-5Qp{{A5V&*9RBMnSx2OCgzw{r;#J(s9>wNF^_bb+Aa-mX{0PV0sL1axB? zIs8jR&~4yWpxHu7l52T9^oQl(bY#CUOlDWe3}FnXz9zZH@OhhL+Tp zKdq>}m@Te-by}M;eMI%S|9j5dbaT?l=F1|c){?p9<>DpP@PLJYIU2Vet&^}TbxF2Z z=xK9uigaju|DRDR#@z5i_%xQeFXhu*Bt=~r_`2##rWb9U8(QAO?akCEzTB0-=hx~jHQ_E`H7D5xmdy+=`(Cch zpAw<+psuN1;m98(-N)uiY2MzM)JnHG-&DyliOF4TMs}cWhN8hmiv~7nV%mKS2z>5# zbFW1ogKRL%3na{q{+By*GiEa2rzM^Hq;@tM4(+I%cQ4F=IG~bYy)l0HBwHDX{%j14 zFkjcD9oET?_J}x>0DMpe*sGK&amZ)AcX{#dmTuqNqy~zncO1XX_&2XN2+3xtf=P~n zscMddv-O1@ppMB(<=sN;lAcO+ohg>;_1CNPuXTg$oX_Wrv`ST~8)Kbh$KACHFp;pu zD`nf32jpEvl&Jt=o$km~;>~7rq;7LDLD$;NqABBMj)b)U&pHa|Qrt76H(fBahmXIj zHXo1*FjD6>wF=|qZ^j`qyHmb%1GD>8ayb=;Gb=L#2%1+huK-05^RX!)T8!%{vOIGUsL*zlj3Kc5h{zDKy;bJ#N3Wm2B{n7v7J zU>!Eu5rvaO+!YJlFv)&-;Ow`c+ObOk4I2)Dn*_AR&*euA0><|SQTFSk4gjW!`Eb8O z8&GQ9c<~C2s;Z`1K3;||PoIVS4M*dp{>nqB`I7io%|H8&GOvf8W^R8ye$BMvWw-m{ zp6F44`Jh|(hxOWS;=Wmj?C-4?bLDOqdyv0xnSw&g`=7ekD&R#WPuO||Hx7d_WWOFv zgGE6btmWNj9`yp^Jsj}#0Xrmxta4(hUT3ZH&KgAyEez8+;@oLxS5bgG6 zT}iM~cnI-<>2=bNea%;_dsdP!r%@Fq&3n5($CwI0qD>G1;6BE$gThR3F8U#WkBg=I z`H3DTbH6@RY!&Mf)GsMJ1b8K}f%_g?$H0E{j#&3u{PbGJUk8#**24eC-q8#B**H?? z1d);z06hMD`rhs^<|Pe#-M|x;x$Lmk8S3L{r@x4}o45qME?n|$*5lzVDJ8bEdg4YG zrEvW+^v22~ie_n*MH&g?3rX@cTN^#@={g6kNXVQ?O<@^^N@)=)Jo1Ss@2oz>?o+*O zDsWeo*

HK-pLb|C{d-abM1~DYi6O2+z+d!~naMXjV06V+3x5MMm5n_&V~29Ly%? zuxf}Hq`~OSYVjp$8`ntHR-wao)Pwgs|MQ~PvB#vX=$WTj|9UoR;QJEl!^U$|xEnlB z(-ODRUGa4DA=)|UqNt?;F@M@Rk$w0#C~7m+e5;zQ`8VXov$tlgIo&zwi|+w#M04$7 z4x-j_F29xy#q4lf3A0icyg+EFxJb0kp4no7>aOP%tVNbW7WC8*r}Jk6M|`&@IAztQ zzD2ety63U`%iwqFEG_4DDofQtUsz!f#SagD&Q8)=JNtjJ_ufHGy=}iQ2!hnmi!>1h z6s3sLAyNcU5wL+GU7D1D^cs3EDj*<81QZ)pq$nlyP^5|U5=aE31PGxdu*f=#-}}7p ze&)t48y;cpBJfv8N`ACKQ^fJw zKJ;c_)#mB~85~7STyr7FVAKs3Nfu6jS&_LL5eq>R%EN@o`UJsdR?uC@-X#3j;wEx% zvX(%YB5P`{2zed>^Z$e=CeQOoPTg*a$Wz#HdMrBN*GVpyg3{ z>p@N-2VkLiEdn!@E9O>30xvX0KsSfM)GQaV1nEY|A%m-EPKkM;y--E)Kryd2C8<{K z^0tNM4&qV$>`pj5^#BfX!_mZX`C68!$|>%A4tZt(^3TdG4{uR3U9H2H3K z?lY%$H*sDV|Ei5S^Nm1a&rCB9dT5a45;iYKZ880W6otJ~@8s!Z?|`U(BJOtY$+^o8 zSMlFY$3@!rz+ z1+Vhui$@xS!F~MW3MzR`;e{bl?a(i$H_s2Wb|=11aB_-TE=MKjhSZt5=Qb(<{GL@~ z9*>%s%g-apZ4!!#>yIAR?69UyMRNF2A4cUizB=fV5Nhfp*xyDY_eQOf$7=ArBtjpe zlZ{$eIW0MDQnMjwZ!uBXv}_cRsqW@(S;6mtvp4`Pr*<5Cz7>ruVMsLMNtF$Bem_60 zsf$z~KifDhA&0TaVMyxVnjhY}l<9yLkI1tF5_}JJ(Y&QIc7=pxDZhoJ^^imo>qa*deZ|t2B7YH904-s9 za@J%X&8v-2L_^`snWd@5YS%;b;KcM;y@xA02gCR&Eh+a``kP*Ubo_TB5`Fl#xfvOi z+kg>EJ0~ACLld@TZNR9Wx;GztDPX89z$W>{LH3C;un*BHv$Da5UyI2dM5rAblS!k~ zcO^60mCXwf42_?D1iTj@1HKT!r7EE^D{b#5B5dGqgcA&>a- zbtIUL?ZiaS}xe}rqWcZ zw_m*r2$}o*q`5JGzW>J8?#5bBNKq-7@k~3si0=EZ!*$^m$}LC%u>2y{~Q0QoF%O zHdb{_j&sy^W7!{AQJ?R3h($rw%(WA(74OMY_BUwok8Q7*+@V6E6n!3RahD9_ulqmm z>ficwe*p@l#2p3R4>`}1%KOFkJygALO_qLa3b65;QoBuTU)Z&0Ji1R+i0_XC?3Gtp z$)U0Y(nQH9FhOqXl6`Yni_KXw3gMnmCQ;b7@aUKJ+O|=zL$NC1OeP=t5h1TH3}E*s zz}~gt$*I{`p~@@~`$;2H4{J)|SV;CWI?o{ZEURbz6s{!_RpZAw-Hud5Fe8+Zsz}jv zx>o%Jm%M{x{7)`6z40(`UaYHJRVEs(+M6?V^m#bG9B)b%+Smbx65B8o?;EvxZ5}-M z0vdK+IvZqF?%qldrkA4NQS0y?gIe2ys)dd)I}KroY@i)41U${A$ftGP zX}RrwjbVw(7sq#2l!;88c3~UJsa>Tdl)X9^su^;mm>*qHZSTd6t+BuXv%x;DS>L=b z-2}kevFzgNUVP`JCQC%202>*cEU)$3Y!N5VF}AWZ3?G-}n1R!*<@u|E$K~SVZppH! ze{&D0S_n4|(P>bKAaD5EHoTu7nC`qVl{>q3E%?ARpbnp8Wi8@|dd7`qbR&cz$BSL9 z6g!bqQx*rrG}~NK(c*)}bXZEIC0;9mk?U3P<+;E<8dJbuBPz6THiEUGE zqTt%}dnZDv3>-qGVl#K=6l#_?#P$!2JNtE#$ajA*Dd7hCK}wO4#nzy*8ueMB6F(>y zotq#vuRjMK&TS*9nE9y`Xn&i!+Vs3qWLd?PNi1c3pK{fLXNqa+&Rq=~n%;F#vX-z`ybus2x;4^=7%JvAZdRW0syd7#@79atG?v=PX_(Jhq}pSDV`^G`bJnCz z?mL>n^!9OHCOYx8Qth!}yCj-A+s4*_;rql%a@+dM17y6=8slTHJoryZ64E0!3ZYv5 za5C!H_fn{+xUIRHoee8puBk2Hb6EE8wW!&xOzWfw@s~YY=oed>PVvxKLRo^P2OZ|S zyUbYmSvhP)wdUS7iq3rAHfcEp(L5li@wR$uwyc_z&#$<6UzGv#BvopvnKP~vChUrQsFsFJUaXaRQ|0>>cGxiuzvpM zZC$#?vlCJ8ZaTS~JyVnRwuRc{SwkEgdG-|G6m;y;X>xWj7|v8XsQ+j@B_q4OyJq%o zZ6NUBQ>va3q_`W2FxfD=_ZPkeKQ-8PyFBAMe&j0)Pl6bp{bmR1NzRef45s2K_B)uH zcRx-2fq%V~mshRaEXH&k6^6M(&HC64{`o%-B2A@<=;t?xs(=0xo?L%{7s-9)sQ9^Z z_2Zu8`E9Yi&!T=o9UU|IWzdlrqwUXuH-G;@3LT)h9i$eqyf9~i1-89!VWYO&YG(%V z>=zSm-=hmzXS|oQu!AM(47M~|O|cdIMNnfQ{Okm%rSn^V_(Ell&GeHZY-QeVdx{Qr zbI*9%n;`Uq0_1VM$7URk*7kf8`iW(-OV6@`t*L+Z`A`_nhP?L+g33>3{%s4a-`N9h|5qZ&t$~A1nqfG!RZ0Mw-pUtjtohLh|BatFfchoTLK`Wj1VZLM0AIM=p7g3)pAFuJgynN>hCAo#phIp}ZP zhM2&Lq7^{apTXVDa0OzR@ObAEWX(Z6UJbK{ zJsYR&;n`zWOO(&2Vq;|g+J84*{7Co}z>Sfy8k)4+n3SCRJ?rh`YJdkFhz}yrp-)}? z8+6TawzNV6OK@C#O)JuMPt`Dwav#Ok;i=TR5z0hw={AQvl*Ht1#e&vl7O65u;`knVq>1MLk1ftkyhXEs&oDXIyD znU}u4<48RG-fu4``8VsB7;SyFJ!>;natLU(25ws4bfRt{A|Zs6yGP{5o~p_9r>f6x zXq>ki3Jb!aP}vk@8hx1V(BUb)rj4aEm3ge4#=?`XdY^XdRcF$G5+JC__+;8vb=^u| zOu0?A&z|SN5psBHbynvliUBscWuAe5m9nuZAkKYyw+c)o8JdC=UrKq+9^g z{Egr_kRNL-110xY?D$=WKWHW7=9MSotIi!duQiuC3Pq20t2+`!8ocJl!j9t?26)K< zV+b~UwQ=Yq;`P3Y($Bw7qr3#l%ZVFGwH{xbs1j~p%`}l45mY}}vdH%G&A{QQ#U`!A zruGh^BaJ%$G;8PR5**fKgy8S)Y-L-~NU6{+8d%_3%~FnZrwML69QIP2Zel}@)qmS< zfz1b5wp&k@8lK0d$fQ(jVxVlV@!*BM@#@guQf>Hq%`ritFM(k58g40M#w31YgzJBy}Ew?B5 zg^%nK%{zNTUh^mg-k=aik2Ne89L|a;D5WkxsP#y1l*GcAAU?>D`{WPaBD*Teo2m!e zJHQfRxD=tNUyHI-VX%iUO$ZzWuCq`VBh>BINVaPhShamytZF4EIL)l2U?ivO#xK28 z>os@=_BB^ba@7@xJH440`WVcUR3R)uf zAkeEj@@{W~ZKiqWJH=Bsw%T)suRkvn5GZgQRmk3JOFeXQoR#)=q0)c0vAfO;_KOi= zJQ0fBV7P7?fA2@TA$=kyG67?`*6_IX4}r^7E&rP1_48C8;HZ21AN|`*XaUrWvuTTo zVXCHH@)QBzVF|O)Spe5;K>0#1$ zTJ}Xb)SB{^@z_qSxAX4)rb-ScQu#z{@Z6#ljc+aj8E4s0P8zUcLX@5bog)r@xaJNZ zB!bR%?tbb8YJWJm*5juZjRSAE)`@E#u5Z-%@rXWBk@LvjZ^>}AlNg~WI3=e_Ogo3E zzXe;3rv34`o#T=ReS5uY+-;Mt^Jp&Rrm6G@LYpT^0JFTbzgxSPixuHsjrmSzxMr;R z-o01Va+g79Lq=;rK<+hxfFKD`{;o5+b#z*2jy1a}!PWZu#Uz!Ak#tiYxT$7jhj+K& zEHyPuHZ-i>kVSaX%uAQ7h&mT{XK^q6QfBv-d5P~Ai|TFzCw?y?pMw=X>0DI1&YLuiR9Gvb%O`V=KijHfYE;lrI_ zC-R4ADp8|qUsS&L$85Ak4Z^K1EL zpDTVsuPMIL<}BbpS7d8K0&0^{kEZR@FUlov=gX`Yi36UVvo=c$T#^X>GGC3PpU-(f zwXhM;Ez_dr>l50^*afa#FHRcI$vQXbo`8~P(*i^9tc3b5zY}?_TC<3vQ-1_K3I8**_McHKQ<| zU{m3e#YIbQvhE0vLqSJ!Ed?rh6>=Bc3YdDUK(HWf|CZLgt2ELJFck$VG`Q zc;s{2C#5!z1!*y^MLC+z{P>t4Jz>R@rHZ1+Q3N%=ngssb83^4NevO_Laynf1uikB! z!o><|%Pb+#64V2K684VqshhUD8s$S{akj%Hge$Y-kGISG)dmYXr^`EJdaU=kvfPv5 z7x~^S-iAo(OTUPGk5w>N*_e1-dj_=BN8v2?UASU8*`x=M`Nz-UlT#qNbd73YI}WfEW;MSQS((HA>dcT)6N zi<0GXkJamf&~(ivZA2o+l#t-0s}*<37pv+kNC&L}xclJ5K%alcwTWv==p+^Sle?0~ zgC$y980(+y)6Uw^5PdjA`Q#N{HL@_tLD(K&qntDm?&$Velki&m5;r#?8H}hDpCjJ+ zHX~I)zW5|}S>Hz6Fgx2Vo>6;!MZt@G*h3_Hay6fj7cZ4wAE4ukVrOCrj4XySNX$cp z(`8o-T$d8jp%la0X><$PPQOY^{OzmSyUE?M{=)8EZL<5>zb%V>9MewsR9->a=tk4z zo6+?qj$N%EFP2{`OiPG{%b%a)tUS#H=BaO?=NJT!nKYbix<|ozo^>n|t*%JYl6(Qu zwjUE6x(M~2?;23~r%&n9B$~BlowpV)Yj^uy#OF7JIkN>1H*c2saqSG~z0w6n3OzY- zmH3o=XCbLAGdm^dcPn%pdQ?UZmH@rl_&>(l0%Y-cE>6?&W&UQS9J`sF(uyfd{a^c+Z(6{$DWmBCKV6+Y6`^u4 zX4j1{ASU8D6`{4LP;j>yq{F_@8ADmnZ65GI8`$x+`jLaWh{Jo$dv*=GW5ynYwDqhdgrt|~)B-a9sjy`Z+$moC zRxT1b`$H_wogf%MIP^ykPo{;%V_#z|*w@p%BbHf_+j(o^ocZ8I#K;3MMVWt2Xg}T7 zS!HHH)_qF|$VLrTeKVUyZ;jMhkxK*Blbtk$5UBxz=ck>!l$3g_+woI|g1hw8*VXKS=-0?O+pDm=263PRUKDe5i{qUbhFE#r zQtDc03dus_KG7LSuEZCJ|0or?lIet_=Z7D+oqU9JIdu3oxPNn1ZUB{|S54Ik`EycH z$04N2iROK3z;{(YzWbk&MxAquv`6cYRIcHy`Yat-E@Yo!tLf)2Yxk)BSh(A;nOD@T zhy0Y$cjY<#{KE}{_50J&Si17@S-9n_?B$DJOIGE+Of7D-^^>r!at%L(VaMl2W(d*T zo9DOG{|E@`%Rb{oi$B6AVsf>Nr#%y2@RM?lX4(9xrG06tyAv4n@pY^;*6*q7o@Pn4 zt95<#m&Xoy>0;kR{O>uEzA3pQjCvsd^i`D)DeLzg$6ihh}4pK&E_hxsE$yln0-|lA;qt@Hf@cC^dyg5Lw zC5jh}tmSzN3(I|jamANCu;5#=PxoZ+UoPCCi0_u>qnXy`QyXQcA7lpy5tHx)CZ(i# zXQbOMskwg%^LP~8t!wft#fDz?m=nVmFRYIDby~G-U>51#=I1W&e)q9x1bE%Q)(VcR z;(|)cya`H{l=CKwd5%j$`|9A@?%72Yn^TGg7#x8#Xgd2t1N`oF7kDmIy}a&ES1jj2 zaII57z?{aJ9?uXHf$(0FLiSBR2zgArV1el1+jOJC5;#%>x~ZZ zYLla|_EhF6CoQ&F?9kINW-DCo%5XZ!Jhr7VyFD|UKY6l0I0i+Vz1G5ckXD*!=;~j0 z*R{63wAa!e@CV$Tb5jiR8yj4FJw8sg(bj1V_ch-?P^AMpLl0=l4pV^xQxEur3P(<+ zx;U60(k$&8%f*Ac-h%K$m&amN_3dXFAXj4%_P7RO(cU(Qzeiee5L( zzh)q0!22Qd(dDNAF>3KeoVZ1i7~>!iVPtTfEX5Or(7!C3xb>eb07W8VjyC+S)_TNI zS=F=EIYMQPg2s?6C8!?Drb9Wf(Y;giqR(j~cLFEIXk{H+nZ$4^#g`bXkn9irz6QCe z7A&G#6gCrJC-lC&YDLECf5@{pc+{c?r2ivlI}e^>lL3Q%2~aWlL*Xwnq}G^TLWKj(V~EVm{i!o z!ClnTFL4??%38!yWYzo=F%(nd}HN95({!VB3smAMDbEzu~m{fL*E`OT!l< z>!%kQSO#h>`P`G08`pgUHs^Y5(O-$4q;acN517^3cPVOoYL%8)XY(i~uZZtMWNxQS zZm-W$?P@J?^bp+2LJ)kx2}wBroWqixGBHp#d#Z-lgStzqb@l#ZffqUmD|28Ep$pv^ z0FxQ%J?R9i$^%mR=rL{XU$H1K34N6t6WE&IzAqOdB%CXTV8l#1&!JNASUs%)K4BJ2 zfq2l&t{+}XG__^m8ab9?yb z&;ovbRVdgA@_4!H7p+}Cpd1%R*Q+=I?AzD$qq8T@JnU`xsv0{A&?T|IHv0h@xTc=5 zje+>yE3NQB;cTvt8S%*e;)pZ=VKYCt{RSx4jp&&W0*UE&YaWJIg*%Ct2W*u*%^Du| z1Zb6g^PgER&}AD=yzP=2O7txKOo>60rNATK#(iqeS<8XUv!wkvJ?pDG8jy*~k@IRM zQeBqe&5?XCfQ?;94ArUNa?ftz> zQ>cf$>E%t4hu;|sad|1)iFT7mTTY1qXmEwa@?Zw zb({ICWV{s92YS+uc(Rf|Y?3dMexTnAIkDFM$x)en1q^qh`7Fatj_7(NQT&u*E zdn&fl%vyzgi5@pjQ`rq+#z5KV?IO7(drFb~xk2?INbE)1uNrvS; z@{f5HQ$5w|-D>bz%tZp5)ELkzCY*izZI2+)m>LK3&*aifJ)>MQv(s2e#+&eHsCgvy zK!*M6dG_`Td;LczGLUg?9krff1E%jAGdQOm15uCMsC9$^aRXy9&LpkECdK6j`yec{=) zx^3pNhu5Y5T=Q{%e24q{(BCP31;<6yx5i(XuZ+JL#7p|!ljphmunI|un^Jd6)-H2_ z;hmdMdNr%(`I26?lW&v<`D4%gy<3#7>^*>xX9`@=mj-#9AbZbQ4Y%a+0-y*OmWUYz z1uq@RQDi>8-I3xp_@OVS&+hf&s1Ny4=ypo}{p)GwFMoJ=8my)bS5mXJ)#T4Ki);7F zDB71I+0I0@R|<|NAU{MQV{Nevv9iZ%j{b(Tj$`IEz$b5&!|Wtjzz7|h3sk|=Ra&FwugzCzM8=OdMc37Ct=a21EgX&Rs0!QULk+DpY2_PxY0hzY@up zR0j#$D_F4!J0oT<=!CT>u5s+H^qp4lIvQgo|0PVHtv5#`Nrx(VH7omjVF;(|s zwERtA{ni~1>7aL_g$jO^R^ycP6GW9`YQ>idExuhw#=Vy~q6FNIEx7>nyn_<+xjIPW zcXQM(0avcd{KkNS2`oHv>RYX*RcG6x)y+?Tw(odvaoMgMpcBqO(&*T-=(=-VqhRL5 zrF-DFA$w$a%We<62hIxXXI{%dy6Q9CdnJ&wn%%k31E=L8{w=+(33&tJAp(FG14#}Y zEDaW+-?ZTdGGR{%R(l?hEV|mSnYIVrJ6#C*N!9N-|9&MLagVqmd=l)pp-NowIrKdo zSRW9>kL+K66lzgsFep3e@k+epuM)Wp@hBL{9!bQhUS+mlz_N9AnUnfr=h|& zGk+rYcOF9F#W3Dj-4EouZ)H>_b*}d3lXJ0nrhG`-v^j=r4~rR;HG4W9(NV)T^Ie8o`l#X8{B#hN zucLox42dm;iur-1ntR_%EvZ{!vQ6BeI^P!ZY602IF64tgO;jUJOhMIu0qq?E$o(8y z!p?W(jK2B3ltG4j{F$W zKepQZGba-?=P39`hwZd%5%)Qe{&IT}T_*1WD)qXjA;n3bS1mgWG8c-UseeRG5TfAM z_BDg~cRu*I%su|wKb#`Ybga)=&-PWZBKe0G)#JpZu;`7{w-mMyf+u6An|oI7?LFnZ z*O=rk1*r4HG6Z6{Zj9+lQ3bCmUOv^VUs7`0_dgx<*8j~xk3-R3)}=e<*8XoFr}|2> zve|Y2ulaFxK@i6>g3t4e*PydR)<0Ifyq3uRZ~vV7#Ew+fLmWpS!w;(8cSq-O-jN+C zGgg+?;eWvu$k_izM*s$BUKh~PD%m;seh#ia@p-nRh8t9azP!pOOz&3h|1&!_D&t=7Nt zPdm=b-Fq#I`3jqxOlv0DZaoB21~? zAI|5Gd};huAjCyM*=x&I-UAHjd5}vP|FZt8$kH=zFOOI)7m7omlM7vEUqMzfcxpIq zWBTl%-Db874^o6bvWPqtc8?z68Po_{}a(Jx=-&h4{|>x$&~-(S}rrFQ%()m7&&=;-&3S@eH=g1ut(pNS0> z?Yi%frH(xU2hK(Cbt{^6fx`Te65y(aV5XIfbWHMNyFdmU%;OK7(PZ{eP`jY%R_vobIp~SJ-m+}grhZETq&pF=M|A4uw!z&y7qWc+0S_`EdTU_w zl-<$u@e5F&`~?at0$4a`y5mKQ$^d)Tf5I}3!Z!XZ%vIrX&F1rY5y|_|KIY+1^wLbt zs8?cnnzkWK2U<^u;vc}FU<$p}?o+D;PUdcv&&wA^Rez(NxURqbdq2wD`Orc14gk>$ zhW3-L=jE)*O?P2dMMH{N=dGR>`@PuGxyr+s;aA++8I?S$fAG}w_VG5)Gku7>;t0=WJ_JQ?;}{zN#PPzP4@p=rVo2#-=8c>QZj(#JUL5j2TG^BYw)?ZD_e`u2M<0!+(jyogN5FbQNIr@eEe+DlUPBl=nIoLtmo%uhf|{>{nxh8k;%BU zVTSnTlvt-6d5a??LO*MU4xc*&4HG-f(IAj)3K%^R=2}3tvMz}Ig9Q+IF>-_jpg8>t z3jjX(H!Of2u~uG}IQEHFk3(PoiGpoSg9#v#CwOH4Pecpao6tk{{|;<>r}#uB9)gCnYO@2y3gYvu_&2T9B;12ptm0{*&Q`cv8(4qM~y*qW9`8 z2;Qi2rv4ct0)dmjTEb>Rml*$};jN3-`diH8POd zPM3V7!807~@0|n}te-(HvZQz4wRAJZudSwSiySN5f$$JfEc)!F@WlY_e`-JLkBlwEotSWGaQ!;<&94C-Wb*Bq-lKUWnxw%xYo7s|y8Be7 zux(g-4_y5vl1r}?DH6Z#&;ws;1*&2y-vQq*>Jgg+LrD2U{(wk-z+FPAW{FO2lzE%- zSi0-O{hJnF?M39=+OCsePYjI6W=#Su=aphp$xa^!=lMLlKvrRsXpNi$2n%E?b-j?R ze!BN#ABF4jS54)u^CE1xxqB;jFHku@sOE3MH}$AX{e@)3!v2qahaKUc)rR}&`U^ji zxu;iD`iS=cT8%__3uh?RSJ=(WK3NOtz+vQbUg^l@3ehtJ+6+#AD6Tp}CqZ>3eDtXG z71Yy1mja2%Xe}@$v*T#UNW+o9X*RCFyyc)+`wnZ&HR!38m0CpZoohz(Ub7(j>y4UD zQE(ean1JQCR=>`1f3}$m2?rq(MQgHn1){VU6OwVqpW2Huknmx z!KSiGJDg%&*YVRIVg>MEp0NLX9&7{|(vNKYQe2#-6$8~N)p4@X5L6^_OHTC?*|vh- z;IWdKfU}}3gc~{qUGNYj;0?|=&d#QYy!zV{x*C=xqQn-JA(Rp)G+uH-CQ|tL)G{e; zRLqB?6dq4MV-F1(_|tR1%l^nXGS`)?-h*aOw7JILHCgA)nK30;}`t8`Tt^J=P zV7sARzpAi#16GD5cEZf7YcqC_r?N~hqoXu*-<=loofmtzEUaiQta8te8|*H){YsC) z-E87E6(P-Wj_x{fz_h{=LS?x|$Jt`f-G81Aa>~o`BD+#i?(9M$-^gPG#l;8Zg{|#PsNUdCs zNQD2eRrAH|1H`a9_Tp0bHz;9n%d*&PKdH&=dxQ565As-fv>q{T`tK7%t#ed2 zWooB&8n=whALQ9B8rbHCyNSYzl}s2(P(n$hg#N#d$%N{CG|b$S6dQqEkzbK{WP|30 z4Df}}n+plB$uefEiTqcNr@(&&vFZNWI|>2@hURZm4Lp3lp1J&C2K1{EB|b_xRiL@o zh7#~pQj-YH`f~<|DdFfS_``C3#jAr}izT@1OW?6xTT93r_at#h2aq_i1!s7!1Jf7p z5Tm){XYb(;>g}uyP>|#3ZJr2sasj$y-a&%EBV%jYKR*Mh zhlhblU|$RWcI9W~^Av062A4;*=6pGk=ak4p7mWl7~ek<^;e6k|?Sf08iNj0itN=wo%&vc`5Ma^M$T44+_qpWJpLFZO zZ-{Mj6f9>O`IwS5_aB@;{Tfs1&c`_$`qMq=tL^Ct3+%d5_{D~(*CO`kS82^U++TTvpUsj&Ct1|O!zw2)@ZbQ7ye(%;w5vJ+(AA9lw zljZFaib!@7i-r=dRB7~4`R=CJkz(LJwfpDWe@=q+B5P{oDC%lqSw(5S=+|y8MaYxP zX72U~`4sxfo%Z98lKH2Hf(hD4(AH1WVpME*$+{F0-0WO=gmuV&2CM!tirlf+zJ@uK z=yxOQ2F7F+;(t|B$M)e-QKR4^(De|NW4bzk8rv5_Kg+nu$s(V7K~Iv}q~aE0nw~ZP zjmLg`YkSE}v=5=OKb)-inBhk&{Gk+-am8BX6~D&8+J4~54LFl3t5m)K_NniHsy{IW zT+p&)t6Wig6;=ut;JI!8X@_z6AK!Ab%QtGF?i2c<{o(MzeZ^3Tho(u2?uQd}FCol8 zKLb^@n-^qdAQ}U?pk>dSHh-r_K-!QuJI{RLY?*rBM#3*dm_uo6)y$6x#uCVUlO?2> ze~u~?!2{M82?$pi5tp1WsDSFG7A0@lZ-@pUGz~O^i@GKS zvZI`-^qeu9um|zTnYbU=LF=b3Jper9HdVRe`nJgvBcf9{_pLjCk(5pWSt#!aB*`Y3 zT$e+5_~JVzS>N20Dro*>RQp9W^S!H|$SlHKA|2SdXdEILsF=^&mT56nFh2K*Y?PDhI++z01kU4Z1`*^U5zB@KOj;Y2R=ckm52u zPo;|keBP18pqa5QAT>Nrq$Z+Nup_xPDO`!)AxYI%@ zFFPAqkB?m7UvcAe|MbgGwudj~g3ENTiAmqHf_}%LSN}Wge}5tWAGt!12VaeA(?ZsR z2Peh$PF0)L<1F*6ffXiqZ0{PU`)clIS4*#r-RJ7o6KAL&)`(Iq!Y}Bme0qPdK=(&| z$1RhS?{7?fm$u;f`p!#ESNlqDj9TtBi_hQQBz?Ume)HtsCFs_A{ITlcxBuI)%ntFr zz?S{t=NICZ8u0ESG9{ z1l9Yi-`;~&XkBCmo_aIwc`UQpSC2DWIv|i}W zngCt`w-JEqwpx~(H87b#MjrE+Edut_0LdmcziA?z{Y^L8%kF@yN4MPjQ@Bv@JhtGk zbDkC8KS7R*(^i@k818ywxeEkFvT*oNX<6rocC00S7fMdYYF9d+qY6M< zqp;t;`Ug7Hi`0vTS0-`RU)xVru7r0|^wMKg1rrm>EfRgiFJ}Z_fWhwCklC0upj*FUyzSpGR<9oUymLcK+;h zkKQ2j1gPm^AB{8KfQRa@)dRt|6(SNP>h7it8|wG|sL z4lSfE=tx6+o|7;Ek#z0{0a}8pP>o(&&}EeKw8sPXl3#ZyW~-=ok)z0MNGN@kDZF4O z?tAP}Sw#T)#7pi65Y1PBo)6-~u#eaM+&Uj9%;-$W=JwM5z@xtGxXF z!3Zz@JG)XF=84FJ*JNd~<8RsvZ;U@ahxdQ}UvU5eJy5@^&F?<;qcBuPY>YQ<0XtX8 z60@%)jOvEPQXer+))L)^cVNPHD6_ALCa~r2nIQ0G{re6uKp3-LLP8h3CG>MMJ`wDW zdVI;{P^kAzDHz4?)NzOaGmG;2hxV>C%HiTJlfT=zJeI)@{}SD*PJskuatm6={8aCJ z>}wi(ONxES>N4&m7i74sd$Ygy(yc)`wh;QYXYaI*9pqjmsWXqW-8)tedA)?LR;65j z;eVm+L|p=sJx1|F{gXSG=j5EL)Fo*fWfP_bt}7I%_C6KQbC~KNK&wna>CUsZ%I5r5 zI1?s$lyLuRAS%TFRXTkQZ7-{XvJixbwVkSoVBnlaSf}PIFF%#3%i|0E z@Y~NnjDhQ=Bx9R~!5t~>b|lBKQ!4B+r<27b)eJh&8jPaw!+eL25dQ_Os10O+sB=e* zZzm0%7AIGZ*FoPdUaFEdO+=lW{x+31ROo0cn<2VcL3-x0NVsGsR^H`yrVsmqn@Sr) zc1j&`sr=XlLXce8#{vkxRTa-s^zfN`n$$H0-34rAkSUdI@gHQR-0QK7ql8p0yWptH z2IF{t|3AWR9w7?3prY$lomChM>ywVJ!mDYn)-D@o()Q`C!+imtUemFD-`{gtlkt5 zKfxysy%5dxn~9Uoj#85aOeVUTr&9$}Uj0*dNTbB#VA>TYDu+7jE*(o9(^pgcC5o>h|p-KtfL*W7NpCgGaqQWFlv_2}#Zy1uo{G}hgoxfidt@z#J zyG+d|W=Q3H+W6sn#b6%O)KUta&}sV3hLh$G8n~XGOqP6V%O8469H9LVoiurg^(A;* zxr-wlH?sM@hd)k6L1^b=jODgM0)#2%+l=?%><07H{-0NN=UGzyH!~shYskUu;!L00tGIBULw562wicx2}`{#zq505}BJrd8+fBLbaDykwWX8t~0`vA0ufP|I z7w80P7(D1rh*A~HC(cpN@IXyr?nJbN55#;2+O}J+lx;yVb-stO<)88as zi*R>H6JyK1dbQ*%HQUAS`AE*6XERfxbil6yY)qi zQ?I;^7)8vo-`19{>+gmfG&L)KDcI@~yZT)ET=(K2VjbB7ihh0uZCqf_C3M7<+y@$a zm&B0OO@~&m-nbYE4ohP5yzV**!uXuqXc=FPqu#@+6*M!d@h;H z-*2luUfbNc7i~yod_oh8Gk$;FDH&JprAvw9eCKG@ldRGM#zr13pX4i#n3enzXL{#i zHw?MW3ZRA6ag%}S2OJ9gG@{UUEx}ciAxCXSp9@E3#gM6WFVbN()n5zZACoNJajcw5HJOD(C`_-W#~N@zC|cRwL_SIPTt|9MJ%AtyB#kn7{2UD|9sHzKfdX7jnDiW*Z$gsmDNS>4Sz4}gDY44Y-|8!c9 zx~+T3?bwqZV=Gstq1_=OnB)Id#D%p+QmKC@3Z}c#uByVaR`((r%-BB9+HPMtZ1zj; zJlQMoVI3o|hIJ2)*}XP5;yg90v|HYS(ln2Cm`PI@M^EiC6Ox+04AnHcua#36Z^~osB>_7^XWHtMD8X`xEaMz65MF!Y)NmGQbTa(29Gfk5u4Fh zUg{@gSc95ec_10c$Ia)1oR=Jtls z`hu@NZ|qnV_;LnZnXIWb0$BdvJe7(Jg};1M3aI14pMKjfx2oHf#FJ^aFd|>^bSxll zwg*e9S&Db-9dbN=**Q%Vx=p0CBTx1V69;Uht#BhWU7!CC_TDopif(HceHA6=oUz^a_tD{_Lb@LvFOxvDqEVvMF=E{ZtpvBv*d;rFaz9L5xGhcKJH zb|BxSpUX~0vuKP^wgS6+g%@B2{lD@rX2pFF(@q=jUFQ|MQQouubBEvdBi&V=Fj7!g z66>9&vWI1U-DDmVJ(wYUhZt*lUHK1l@JhDoUbs!G;kTCEW=oTa81Q|iNJu1V0OFn? zxM0Ow>9qG8Er%vm4~>|dLV*Lsw$^HlId0SPvii}?gGJBh=7kq=Gaw_B9YQ!zu9P>Z zuWnI&mSX>-qa?S$-)k7P;Q-}fDn-_M?y%^e!$OypMIX_*TK0p?b#LBJx_37h(#7Ed zZXx>c09IO8&m1v25;_uO=SY3lo*$>Zr6Bmi|@I42h2+6ak$vn6Xc&$ z^9U8=^VFJ_?MHt(AGi?6W%l>{PK5F7{Aa(TpYGH=Fo*DCcN+CK#GGF^V9|Ze872$neWy>Pg)Cl1;F;wLsLOb% z%i0}Xb!-+5!32_Lb`P$m<};M#w`VLo!~|<%16EfQ_f4ZiC>iYPW)x_jHpFI{dEOCM z$c^G_EH%Q%~*XZ^Iwd8$#-k*cgV8`6On7 z{>i6x$XAG%D2*60qBMIsY?sK0Xal)NO~p%pWCOPu5#WxGkHv3v1$=}%GKpU~CdQiF zH{hLkeJpoP-gT%?TM9Twh~7NG6hi)rXC~;Fd@6(rrixFz*>=&npL+M1y*y7I!fcAk zn=Oa2f%uhHEra13Yg~cUi(SS$N36j{Ah{gIZp3E*ALt5OwzQXcxMLYl+8cCI9?h}EP!>OoPHeh%h zu82ySdm65YVW6p5Ri3w9=+tj>hbpMTU2UCmaIza>6C3AbjWyU9(xFfV6$xrrse%Gq zqQeG7p2rzO(!-1x3L&oq9)V?c>m0SWizs913N1| z(cq;|d-~P5sI(AH=Do@U6#F-J>(2w39l%J(NY;*isvld#8iL-9Rd*w6U)iMs!q zxgVQMO;-VG{^64tfAM7BX~t|L&L;G6_JfURj~*o^nDprkaOEZyIKmbpytWK=&W6?b za@Py&0fFfd^r6f&i2qvhnGmXGN#X(Dheq(Uw-ad{(i+hZ3p%g;(-~T?BJT{465&u0 zi^Y{)EU{Z7hGCyBsCKQ=Sz>H)#0#=&$HUsHE!ycJ%YxSv)`(I;T|=EuduRpxa3>~;R?rX~gjBX8W_zyWU}FO`-GJ#@71b4rPP-v$Y2{;1b!Y}*4hz^13?cH?)`;XWW~Gwo zsew+V$-;iv%fDNGK^n#6DFJ=kAVeI4KUP3lgzAv0e`MO*Sp4a}@}J#T_GlHfKJTeR z(ms(>nOP>LF@!vL98SW`_O-5N{PF8L#EiC$VQAX?MjKRuE-$lbF+pJlI@QbH4JKcE$b+YnLh(9HNTZSu z@x~B0L}hJz2e7;DAh-%@U|adlJZ9B^BvF6P7}Bm@1tn!*r1>K{;@?GEsq7zh*$IBP zI!abLiwgVjy6X^UI^QJWV&TjkpF!_w-pZz)6iJ%tS-@& z@S+M3`psG+Rs%`)17Bo$AO5+<(EUGOldF{EzVKy`2PbP-(m5;W&gA_4OXrSociD0c zsjENNV*lHrtHQB%Hx# zoW^LmYr6`YHKChM1D7gDXE>ksLLSUcR|KQUSE~IQyJR2^!p@(S`-#K3p29m_+_RBm z$if$0FhKNa`r)^kjx1U}VWf;!4^vt^^J-!@Wfah42*mdAEG`23L{@bsXFx@ro}&oZ zQVRjRPHf;!v$Qe_ElbJ-zwck_*Nr)J~LI&gxfdp7&J2|6xDa$ou(jiJ(TTEmmYe7jB zpyb31F*UWXQAxK@V+b*G6Y>L&3W4+{Z2kFjM_1rmA(jiri%J+E>W!i8o_QKPYCnKB z1LHNpuxB~fs|8*^OutNGuB3ed7aFL+ij<8>OWo^6QdGh|#=fg_x-75Fc0+D}I4YEs zdUB4LRbB|kynXwvR8u^v&S+!k)yU)!R7REJ%OBfAR*t)^G-FmutfTX_N|mN^=;sL7 zFYxW9oc)Pl9bjM+_nnj@xwl}S_MOlgZLhdfQCHWxM>~JA0ecp_TbRPniFn`em`!s?@{hYg!P10acEcVKRbk`;jM98F|j-> zOG7t?J6$oI>)t!(Ynx)?mE&{G%(I+LqkH56k}iB!P$HG>(xw>Ry^uzBC1gN(St%;xHyH{%N{e3XOBTfnthUgSoXZi=(QVn+P7$^|8aEf zW5MZ%2tuib#O#yp=K^nOAcz$jtJjug+As!hM^35r(NgnYLP%JLaL+{7^FL$I@b1ac z?WnhbUj*4ke~u{qNB7a7;=tE~*ZWwb)X&ItVVb*X(e#wJv zPSOyytwOQ@b07gEIS>rzF4_`dHcp9 z0GPG&hQ!L?H0su)7gQS?f9p&0uMOBEJ=s1jC1BPESkkLsgXU_R=S08Nw13*K)_O&+ zhm7W^+lJw}{$u7hc?)QN8MF06wk9MgTV2{QdPIwF=w2GE3}5I!6prPFD3Q(OobMmm z)u%b?PGFg-UNHx>!|gIUPIrD4fgbr@f7nZQ!+1`e)0_6l97ks|heF!bf3xE!MJ#Rp zkoqvS+kGBcd)Svw@M*OqJ5U9VlubRCT52j2Vke)@1S`b^+%NDvkIa|0=Sl|u@+Z$k)M9{f0&5d>$-`~qYPnMlfUm7|PZ z&e(*qX3gHY9gXNN7*{q^^)VSg{&clQeLfKQ^mpE!Ld%~rLCh~rOIG{CSb!;TRPZr1 z0}TZ5rNCQESh0*jtBDujQ*OYpftrwn)dEeZend_MLlB9n%FHQ+Z7Wjbkk(HdfHm(;JL?S#v}F55VufV$xy98c;Zd*1y<7ep$5WzF@z~X+pgzC~y>WMJ3;AR^`^3*2J=* zPXce`5afp9qNvs9K>c-l1dH*_j)?$AwV+IhP@giX$W%>r%?(*|bUE2g*c(dR!W!-#*PLWi9va9JIFz zaQ%y{v^TkWeV*palV6`NKESra+6qzm7g^ob7?pB87`NF~pZEA=Wh4an)&4~mF9)3+ zg%IPmdD7GanE6FU@2Dpfu?=Kg{R?7na0{`t zOI4Aw5u}*BT}guJCt(|M3P41PSlz&G3MB4_Ip_uLt5kAn55We>H(^0LV;=%2T~!%6 zDHIgZ{_`LSiRFX+>LTST&2Ru0L7=4s zmvC~8ZY3NHx1KpsCc}2duX0lo-`3~IVzj_*b)oapeRFDd6OeQQu9da6M!LkJgZM$0 z4}0c<9WhgrM!IeY5}iFEHvE7mj*s+1hb}-@AWGaOlzb5|J;(+2^&zl|l@39!sE)sX zqkagA#@J-NG8@8R`1vX#ym6J1u%28Gm{odqfVQlP`9Vp>5OKb60v62QX)&KFpgjSu z3BU%yxdkcuq$E!kIs-(}vzUl9v&y^pj~BBaRl@H`2zI1#ZIId+QP8jh^PT7VH%+ZA zvrVjzH43U*OvS{tEL|TMzPPZRJh$RL(^M{f&GC=n!V8eM4uKy{kWFEO@l-F8JY5>= zS1FYl!y!gEl?8X4E{f4+wQW(?ShYZsThI1NEw zSQ!}JFa*kzV-44!R2gBpwvyk< zsgWrjh&;8aut5sS4&Nz?e{V;j&}xIkp&*rgSEuYs*ug{xq`B`u)q9b|Z^e5ShhtAD z5Yj4N&b-EZ2Czl3^Ai<6x?v-K#@}%AjFr~q`_ee1L)5g~K5%U?7~LmvN{7gWC=h4y ze9V+T?hPFRSp;YRH4g?22H{qL85n9TM`PUKo}#HI$?u85F`qq2YBzwDju5k z7y-$jDSC5Uudo-gT`#wCv{*u7@-%N9;TK?4ynQYb6KQvI*qgpuKq(NJ1mR=sJA z2md9T2BM(YV-`c1P_`F9s;=$v0dFshetjhxl+U_etrVrKcOer|^iZPx{)I-Yi*pZ? zcEX8%fns5qcdmkN62nTMH)pSy*m_(^-*x#otB#(OR{sBS$C)SK=hpD)|9ivEIX~fy zo7tYXk-H9eT3F}RzqljIOfn~7f4pE8_dz5o_OstlcG*fL*<7%|+)uU^QD*GyGrS;l zqHTyurI6!OnWBsDjN|~Ejq$Em313IjMX#kw(;azimYC0d`Z3==h!m{V6MYgwh8M^% zQsWgxgCeqwQ6i$mEDDj~8P5bwgZb7g2DrI2ZfhQ##VO-&1T2~lz!S&%;gdgQf{D3U z?fJSrNt&a-aF-W~9sEcc0uz0U?;HVZ|Ou+HY>3s+y7PfOkb%@-5!>nQd*XAu~AVzq@ z+6F<8X^p5X{V|txkvbU3pb&a;Vr>H1iScC*S}ZLUAzZ_}-(@GEEo&{)Au=Ms<5jp~ z@J8LB_!mWcrkQ~{G9;9ii3XC4m)E124x~bH3?Uh+xFKmkHqv?wB_8LB7`NAPz`fhe zQ}hBdvpk?!@r3+Y4BNgYnTi$~v)EjFhT4AD4=YzJNFij8G*qiZRzQ=ZDxe7g#t>T3 z#;!h=le-~2O{fP#JE);&GiH?>UyGQvhAizbY!Q{l0C6Vs0!z*L(=Cc4F_&1_5E6k8 z18r;n&+VMEN&$>_KyhZ-7#QE(5IOc)Ce|S2s5|htvgm6O>mP|A!5Jp6dZcny29u~3 zKg4qA$@cv_X$z0Jo_*b~l|3vrcwaG&@FgH+5gL3PdSGPT7R4 zHw@Y!{UbIH_Q+{N(;x?y&ZsVsDagLhItlLB`(Ry)LChM=2BqO*5ctyQlK&#^mr1*T z4_zheHoPAM4hyDGR3Km$^ac{-Hb|9@OOkM*UxMxHj5&8V@z#kgP%HvmADM~Ha?vw?R3ax43Fz9HOGXb> zM?jL*6+yy%mnlh5#Lx2c=DkTaBgGK_LP?dE?+jOaJ4^-P^dB&U$6 z#o1AaqXAw*0vxu;a9SD5^Bpw!qK_L2R+ur%V?CCo(VgUcVWc4-m9Mvfp6|pTvqwK< zwf`$(ykI}h0J0h5OQ4^%#2;8SYC~cv_R`Nf@tVBf);~Igjces%MRR_q`RAm1MDzqZqMf(S+|fP1?92@bf~?b4n%~J zfc8&KR6yxTvsoya7N%&ql>$F}0SrMxiZCFPUGETW1A5tjNSo_HrbG#V+TMOuP$?5p zc!$gCoO01HYB*!I2B5yikk86fzy1I`aPdRX`8dpC^BrlU###cGHGNBoD%Sm)hum?T z48{euQzevim9P8cy$=%3I8l_$w6ydx-y0`Bl-91Fu$h<)UXw|Crn`|!l_qZvBrS&a9W4}pq4Z^dp=r9YM2g!M|=AjYq2nkOEMgr;#x0}!yRYRt!p{_+YC zDVL6u*tf@R$~(tN7hzrlu!56uaz=vWnovPqUvyYr5MaI;oUCC21n7l3ca#4HGQ^p7 zcBUd0AP>2kklxPM`T6BY4WRA6Vr;`^T%0<}xO{hKu_z+ZRdA_G`2ue!@ z%ETtRqZEA}_%vkGNwT)P%9?re#6wC=ne?)$l4P@EY1itX6>3YK4)&mA= zidW31NA`w88Rz%~ul~UH7o=D;f#0y0ZPAv zq<>2Hq#Ks}mKpkZ^452mO2~$X;|j8FNQ&<6Z=p0VU&<9lcf-YJb6|D|YK5T;hSN5t z-Qin^vub)=8w?vLzV9M{9rNV$!QH>aR7P*I+BVGASro$Df@DhDqM`9m12_k@iIZ(6N@ykn>mb~^wDky-KlE8G;mgn%5wn5CKxEg0wmFYB;1ulPhG(V*oq1jg;-7{ zk8ms6Dxd+)?Yc8&5hV9;6M$)i?S3%Y#L-!@kdxGYWNgi=dRPOiLt^gT)dUcKCE|w) zIx}Xmi#b!lIXQ?7L@JolFhp?5?g?%?fc>unye1|0+f|`fvLK8+` z_uf%jc>3K+`azeLc&9_2rGUU+W%a(6WQfq9xmC)rr)Z>god_r42Hf3OPGV5{_8~Jj zX3#Vl4u{|w8tn!q4OhfxIT@SJy1ZTSw>b-_n@9ju1;A8Pyx-&s7r7QiP=JI?a*AdX zY5<{s2z5m%hOD?WqB7q1w$_=C%b{*DK)-w*cPzV)|JuAzXys zweew8@%8PG#rKPPjQ-cFLRP=(>jv!0Q(sT|g1N6&ik(}yV+R(Ro}}Io4e^&()Y%)l zZ|X6=zd~hhOII5^Bk!N8zNwu~*_?>tKbc>J4?rNh&o~MCvT_i z7`Ve!G5~Y9AKa&%@_YG%A%r(UZk!-L(dJz5vVyJ;UzVyX8sr?2(s#$ka^M=UJeb7H zh4;pTLWXFc05!M>%Ifb}TrNQ9Tvn1i1?`izhCq1i#0vI6tXEK5XS*mb*XK>x4{}{3#F=Xj<>q+V;1bOCF_aIFE~wAoA1wUiCfFUa@UtnMFYq5r(86Bim( zVbuv2c50sV_O-6kKKqkd>Ie7O=QsSApDB~t}K-T5ham` zLY#XLcO|#%&;^!p{sY&f2QRmn6yduC7vuBS+ZIM-uO(@iDu4WT{Z=@bJ!tu=2?&=LIN@WXl`=j23usq5ZA$2eewJ!#b#h`G5=s1*J|1FX(X(HD{UnRo+cCkC8{W*50k<&t1e7ZSM{Gl7JC&0%Ke zkAJKl(qrF#nD7!{5C9FodOsJRWr~d_=I4_GTv@dZ*h^vT%zv;4jO8!J@Jr`9Wv0G_ z&nL%b6~h|yJXf&oCZ&|>T&S}e_w}-66z{#L*_HlpC;+c z{mC4#r*Dth_w)*sA&S@TfstX=&1lBP)AaZ=N2?qej3q_vxY6t0=@XO9{n6?YMi{NW z$IpyvP=ZX7HQXWEQ_cC!KW=;rl-7=nx(VXlA5APkP83|cE`M0for(kZPa1#+p5gJL z(pYqZbZxP*h0il+g(TbJ)b4V!#4ScY`U|`X?}v6_V`-3a(m*<8hN?{WE{M}1riRLW zz{l{1jPc1`5jR*Lr!2P#wMq|o?N?0d**{+aF3gXXD*G7xP2}Js2lQm3QksB9-9`2X za_mrwkdeuc>PKa7zF&M9+=UbPaV@^6-4I-%uZg-iB4HoN_^OsJeRwtKcPCy}Z4fs4 zhR?XBea^5xe5sjHuxa3n^x>=+=l;A0#KF)%WOIj7^|vdD~aOlBP$l~$|7K{w@r@To0ldDyM^7j`%o3fVE6 zr=NatbfSgo*<_l3@N1{rzG_mgj0qAhQcrSn9RVV21KGfV_yN+BQ)R7gvZ|P8*nkcI zKZ^h!smiRGHycwJZ{tMp{nQi}`j1vcXT^Pw?}V&3Dh78LAvty4ua{=+_esi+@Jn){ zNbzVLSooBOt|vp{4Ey1O!1GKMzT=+v0{;;-xljf5^%G7tYOZMhWq*MD zU}=atb`;n2o&l`~W;9wE=^)mhq-irMDFAA~OKrDj_a)UV@Au`m#ljT0`0=P<_Z%w| zkG5rS)~!FpO#C>PD9K9tjAlIS%ItTYhGe|&^~Q-lWD{WWCP;MjKYqQp&!p{ZTf+Y1 z%E{`Xz%0}1>d6&s5KsODsEufuGan7HU^_Fjyz^jaJ+Q(M@VpA_w_xDfF`g?CD;{lR z#SFx|zKyZYC0Op@9#~Z}{PDVa)(kYhaK^)iHR7PTdf1X2X0z4`u0O>)>{K6qCse8B;WuR_uRz-qU4mg3V^&=@eUw6D14wYvk~)}O5Nwz z|Jdv#tGWqusKt;&GkqG+jjN!%Oif73tlEhHkx(l|wI(FBaHb?cHG4F^nh=mQa4{=) zKX~dQmG@N14T%^Os|yY6TMt+76QsOP+Ik!_2<`Sf%a+)4h~Ec!j$K>5GCes>P2CkP z`tpz`%Xz_NGxAXtv7Zid+=3nj8GvsKd4He3N*u1wc!&EB zN)4T#mfdkf!;m6*NG9iz{HW$8( za#0~q^IgaJ`8l@NPNQul3nb6dM|tMooWV~&rnc6{@{{J+1-|Okq2x6+Zo6bMD(g|(s6%tusBogmxB1kqvB4U zgC(CMlUHinAjQ*%Y-?5rr)^hl`o6Pf5UF7wyQb8(1Zkhk$jy}?X@eOfhe{4p>W2-F zq*R-Z^P>jlKvFHnuDjivsa+L9$FoVB?05FBNS@G6@F1DW(}rlJXKk`ec9-&J!hNaH_!k@9H<=SWIk3x zw9v$fI3lC`T`a_J>DTtwo(=HxBOkc9OSybyarx>sXF13_K*+x&yEfcw9uS*1i@Kb5 zOmvVqf}ag|ZD;Erwry9PE&wk18z=&Au>8HZyW#sRcleSg{+1bk1%qa_UcWcb8mfoR zi8}{5RJ{AOr2@Y71j-A9O`3)=*^RXno8!xf;pKs^!)e+~8g$zztLdCR+Y!wZnG3Kb zdspo->x`)BJuv4%2kly!QIhWExW!igDQH{&qY%=8VfgxNDv!LZshf+U3CO29W&}=Z ze?dC;^}%vbNEX(IjF!@`ckJRNCt%NRzkBk;;`VFD)pcO1?G*4mC9SzZR@`P$1A^F-9`@(#`~y}imM=1M zu$p5&0xJ*W0~~7`{l=hNbB8r(_ps%VaoET^C9cEDZ@XogaRW85jP!WykREB`HY@^;qU5(X9rHQ* z{=WWf1eL$OOZ8K+2y}gN72%r=L2g4B1-~(RYx{Go?v+>5zP$=ODAC-Z>)CKH@t&C| zkSCHVB$S`91>L5@g?AlH_)YMvd?sSAH`SeRtj9w`n;rycZMoku}(=l8JV1V%`d1Uk* z_CE+!pR`U5eO4lpzHZ+!Ir5&hgVpa6pu%Y!ch%oFAKnP1#42n*wa#+*2!H8+v-#Th z=Iqk0VR1<`pGc_BA;|0JRRI%1xpyxb5m5kIl@vt&a;T<}Km`#=DH|W?odj47jasSY zNG~>eH=2S91yJ|5%>;~`7BBWZdl=$6;MHwCdj){GsTg<5Iiv56fBP(g1i$Ak!*OjS z>Iad8Z~gDO!N=85XPU$J^9SF)M3tPCv+QXzVo-fqgG-L_q7cFgxQVl0!BPYL5Nr`P zjF{7n?K=PoH&odI*?d_7o@-&QmPy6&rBEUy`CpfoMNv_j?%O{YrO$Eh?>wS>y32_J|1FQ1>Gvzv z=Q!akh>r7Fgau!BcQL>+|4Ua*t0$+Xom1kmhCx4Em{Rhy+=exx)^nU=pWpIFQ1^8| zI&l{kIHM=crqji(_^meHF#6&*3x%^GBuyGrhh!F&G>E2~0K_ez1o6xxaRw0oly}0t z!9tNX2*cf!^M_PHBV36A36q;QM+f$;qdi z{5d)vix&i}|!ay`~s7(1FEd{B7Rx*7eA*i-PD4))s0Bn0rjG z^Qlsr+?;gcOv_@5;L9$z44a?%{Zzz{0?(#LvfkDlB0w)rP|3Pboy$chs1O8+F}%;8 zT1>@%1-Tsc=eR1hK2}`ibBU?Y%tm>yM_L(jpO~ukaQgJ$1ieS?zt>Hv}i&SQ?17F zWLS$#^85&Hwb#3JMLh9<-ATCxAPiGy*lLGM;zbzF0PI<9Jg}U}e7>c5!XPBsxa>v9 zHq7exIi4CCs!|@Yi>ci020XLfxT#YOAx^?gVnk>Q>Q+TZ^(=CfThJkXx>M@PYq`zl z%dlucn|;eAY=3|y%vU*p5>h-tcVvr{S>~~wij{!t&0SoyeC%R7_R9a=)N8uAS6)RK zQwgWOcJf8+hcF34ruFob`Ez_-4Sn8vt2Hrm4Cv{b>PerD$L;+pRicSqez)W{`ab2B zsnM*)*Be(#71d7UU*o$Eg6g$hck`z-e0OiCJ0JRd@DaQjO1HgtF}Qg=-{XD!bqyp; zL*-o^{Sx0}4c2WQ*!s3%aRBS}5|RH((5CpP4j))*pe) zhZDz|PnSdKvu>7EtGm0;sr_8&^jn}-?^pNmn@8jP(;9yzH0DROzs-a!d2OyU-KGs2 zhM6=vyUd+s`AmG;JY#5?p>>El*wEGF3S;oFGxW*&`uPVdOzW?pcWT~<16VU0_)MU) zD1?3!R-2vogsk}GPb)k~?#j|Ks?GQ~>TJ^%_Z2@>&d|O`$HubRG+N;SSr{GF7ua=$ z&Uc6bwIfNT-*`^mMD6ACir75I%hzFfuqZEV2*gEaByn9qYKv?A)lOZ|3EFOrxWSlRo`9Ik)z`f7o{u*n5{Yxc z9J7pO6aA4$K$wAgBRf!iN zKYD2giyC6&7A1Oc$eY~!*OkQr7V7ffb^P1;`E-0>={LRfjDhI)x8!?Ef*QGfS!ZW*_UB$!9Wie6>=7x%gCjWlkgX72h3f!61}vigr1$~#xBVr(YomGTZ5*6&3wh(lf2Gi4kNwVPM|?fijS=#c>hA(I2o zrzKHb_akaft&NeJ7`D}Wu7i;i_y6s@qpc9IKHmA?ALWP>Jq~C5%NI#}|9<}e{1F=) z8v0+pf%Cd5Ky#$L7xzuUsDn4CgjV7GYPy5gLB0Z$62!>+HZr9$IO@6=)RQB4BNZ?{ zlr9X`hwsgHq88sTORe?pe91z+3}v)M%|EK{DrY~q8kH=#ECz8x*C_az79l9aq5$=x z|LsA6xZ9SP$DBf4ZpK3#D$V9gKDXV2Weed{eE^Dn>t`Oa0p2wdWcqTiYL_S-JNO zx1dI3+cqQkzIXuLx4KL?*oala99#S0yV_beYaoMQgQy)^wO$L3XG3#*bxhn4tG?lQ zF`GaFr3n>o5ir2DD$6-N{L7v8;Xkrd5B}r|fE61rX0B&DqxQqjO9f z@eriMO%i{iuaMN%67qH;bs$w4bkaq(De}7t#qT{HH9)s~tmNnSy5c?&$k=0fouOVc zEf;)r^wJS}AUX^=J%F?a*i5R6Qu3rKE0JK2D|ybWVI2(EDF{bu?y zSW_-s-x+?=n@ZKXa$mG=iEqYiYl8{3I_(-DxedQPN(B#HZ8tD_poEJw3&gH<=4`gJ z{ER8DMV|hy`SI!IrWEI7XAu$x>9O?p`egd_jA0k@d%btxsr5wZ-jq}S*(41xfj;~+ zdno;p*=mPt1bXDNTpT=>D|v|1%fxAHf$T%&g)zQBo=kU9Fi4)mZ>~}`Cv>sbT)KKe z9Na})#&dRT%_Gh{4V}MTdmvfbrma5+y}$d)N`hSGhf%%U$ZF^+W;({~*FEm?J&KbU zOr*YvyA4bwbXy#Cf@DJ}J*I_9Ff707us;0C)gODE7|{t|ydAw^i{ROYf*w4HN$Dd5 zr|wy0()yDg(m;B&fnqx-_U1aIOI4Rb;=(FGs3Ch@%R}mMUD#~cq9NZ6taODa3^17^ zicRXilf&&}yjVDDyf~s;OD^w9eJmn$hrbY~VD}{JyYvKIamTvepa0aG%vDziw9iLCEpAUG4QFo*aoo z13Jia(PiJW)9kC(u+|GClj*f{({X21BOOD*DGIR;ns6!vg($Jk@{ zbI{o{vvp*69Frk(_bY1QQ_d}D-*A+BzoLaP%5~`b=Bse4!{tTx9rtzQiB7MC4Pi=# z_!oNrpJDR7I5&TFWSt#Wh#tqLNS+{%nzs*6qoVj8J8*SUAk_Y1N;6Ax{&i3}KoR$2 z)&s{SWZeQ5KhOk_Jo4SJmiSQHdb@yg!6p7i<@`rIQ*AcVz2RBC{ zy@jFC&>?6AD3jF?^4Y^xz)r?svF{=*g1P~vy}S-Vq_2jwdAaPKdd+f(KAkJpF~l0W zJ<0MQYH<=Rcyy&;wn&(uB!;NR#E=NufcM~#V2~O<&iIHLP~>!d+&YB4M@?`jgyiu? z(*SadrZ&cg01n&zP#huGb_&5HG(q%MIW85wq8rwi6tvWa@j`JvrOJdX-8=^`#ns8pkEO2e3jew)wNf+%%?SQkc(f^ku3BP65)s0R zLU3&g>3qB;0;DcLv^?uORtJIVB&zs3vitYczoD4Np6*w@T>3I9eb?zW1Cz~&>F2#) zsNbhBlYt1%ECc`ZA76HCI_cJ0cVY${o&0>}Jm%ltr$-^&r=2d;!Hg+XV1EU;OD_XI ztp~0h&c`A*t%dl&-q_t> zp`zV!z@4;>hZ+{YS0jsGd{y5x%*J;a{wX>QdP{u^IH>WuSe?#xZ>_yQLA(v)vfnW~ zZA^YRe|dUpfqkRfFbvAqe7Qo}#1xljdpADt#^t-`U5o-sK5qj)^0R!K9D3%^(8y{~ z+@Jw6PU@6}AfZd9IKfxOdj8c^&tu_s0 z2>U)RX$2`esVnZ`5Y$mekHOdmkvE2>us0NiPLWO#^9q*PAZ$d@^%DvV$EB>)We+!C z^Ss=8So%#!9bi2iP)$Ldfa_5Si;)#DcQOLACeH>G^=o5{eD%5}24^A89&w@lyq#Lx zr|{(cm@HO*-HWiP4gXQ;BM-8wll$36(~N8Er<=PxG`FahAg!s|Ey3*bO9+Vahk(#(*o!_$0W=exwO z{Shg*)@K;e4#~GggqKSaO|M;HpdKx_!rZI*xA6D-b4|`2opmVWZPce_<8Aqp&Dsj# zLlo_;M{}%{ePbtlV&I|o8cRyL$RIBsCEr6}t1DxYfLr1Xa)0kVenAr$5BesX`( zy$j4vxoA3S5{3K-c9O#EhbyabIT|yh@l+ zFDt`%6px+(MIN(f6U);Ym!a$IsN~mooN!ix&SQKfn12=<+ADnt%Kua_tFXvy@1^jh z`w_F_;k$Fb$afml8H2SSHsET}u6!HmnQ#eS8VEggIz&f~9ejO{cZ&QfukkNMZ-Chg zMcbH14!jh&N#P3+Y8qqa8vt15p=?5(YV{#fx9)q;f8=OF9UHBpigYc97!A?nFpxM; zr>WGenR@&J#Nhii*>8}YYH%JBBf;!r=@A6|5Oy7Yi8FcpwAj2}*VySs)eZ7Q%@y`u z@-MW#sqIc-pA5M{233VY{pC6QOPZ#PbO5uQemc|8re5>rf3)|VQBeeIn*u5-Nkv3S zBO-!CNs<|oppqm@92F4+bwDx#je(pq5**1ngJeb}gXAO#jN~M7NRxWD(R=Uip53#% z=X~Gp*&mns!}O`Hs_v?;dh30k=c#0cu;~XU{hp-40fp3&Yu-Pq-xG%z=!PTcPe~3V ziFjfC&USw8UkKTY)m$%@k4^ff0DxPA+;t2#< zPk}ho-0cQp;oZ$N7ds?;_TWvX`{PUU&X`9}n=H*M#%+4dO;0YaEgcN77WXq9Y^MG+ z_MNl4JQMshBj;krg-PSKxl1$>7Ek9GPmXDPBL+*HRpRJAY#%OhZ{_cb+l`@YXLHiT z4d*Br9@&N-q(cyAL!od`+ZP|=gA3UiLJ=zksxGvj$3#s<(msCm3m~qQ{8)6^64?!Q zcK#N0wq~~23+iD?Qv31kN3V7^Ar{6n^DFw?n&N49Xr;60Ww>eu)*ti{xlAdIKk=?b zv}%mPw4e^x6W~}{>!ZC;Y5H3(qmj0p;ozLzRWp_T1Hv}aZDp?3T_gV|tCFrHdxSWD zt~8JUUnYuPBVPvMIXMN63?^FxU+N+zuf;$6KuB)r>vt$93?V!^KV114FbP0g0p7FM zxOOlI)s}wnuBczNuJb$^4gEHfRe1pY7b$5gVSLScc~_NqK;16A8TWhSvzf;(j@(Gb zTjExY(R?HiaSmf}^tKfMz8CJ#IKbSuR7DQAqk-+Y0j8W^#!WkhyC>KCxxn@E$_l#g z-m4-u&Cto&k5;Y;H2hFKz-{ zI1?Ajn~5U#Xb;kyK=uz;$?kV!oQcAwZokjW@f5FS?s zIa3mbbl@3m+OSOA__)nl)?n)&KWGP6L^OlS@LI{i*UugBh#qQ_h3Ar-pC{10eIW55 zK`pGbi?dl6-tK$<6i1vPw}-t>bTXnO@>!W@Sz<3sVs)g#yR7(`Pb`cH0|loqVhW zaV6VrX)2ZOa#6mAQMo&%PDLIfvCdiM0z zxuD}M5vp+ksWgvW8*HWWm`LZ`Y{fu`wTEUwvdO51v@@O#gaz?2$TGM5b{}xiyqu&$WEH-{xm4 z=59CNEn1GKQ6aPh!|M@tyRX~y(PY8~!P|qdfuz1`z4D==D|!0x`dkV*SBejbS%0xG z6R>Tz+BMh4IO?uzJrXLJ=>T{I9 z7ni=)VXf(Eowmg8U(?@ z2nn^lglu)bA>55MdB0v6IBTEulKD%;wJYQhX5sUplU!KXA^jZUKJp5COnI^=vE zmR7b%+?Tmz?ap3Tg04s)I!&x7L1TjUZ-0&U$j11Bzt~^wAPZeaQTIi~$YP)za4kn} z-TbW6BFZRbzYr3N`Opt@3RY5<2%x7=w;PEp|Dqg6A4(bI4Y5A&!>Kw>IhRLy9pANq z&|3c_VRpj*Fee@VFelsoFEJ-Oh>ZUvnSTEdl1#xuY#*gMCcOBF_c_7UfhJ+lD7kN} zj@1VWuM_%w_A3_hbY3SpfouH5yf9LSx@x=TS`j(6eE9o^eY|FDiPs*y%NvdE+f`S9 zW7_;#UtXH^G0>J?h=szB2c040I|#^!YGT~>1$Ivuy(_wkSuYx5b0az~q|@{vH`I~w zTdwa0W3-2|*VMTkFb9*~)!4xhIOQWB+^nk&7`7GW^=&)386&f9woqKZvAaF*FN=ji z(fpMj*HF{0c(dm-hps-ChZm~}8{hlbt=*GRE`&VKiz~|*bXev~nV$u*){DtF~nv=+WNjjz>q&qe{nr^+65F&^?Fsi#mu}a=X{cO z-Rd%V@60OEtItb%asxYk5TE1Ez&+_aq&1lJnml@2Hu!4Bsf#@3@D0`?{AuK|mAg3` zru9h_?`>Z4_e}=F2&3TEg>H zt+WvBve&z(KBuPKI(GsR96h4lh)Y2ZSR%Dgq?LJuN5}NgR6m2j&hGU+z&Z7sp`UUlV#S`)f-0hLAZIB z)#IHNdAAp{hD_}Xi}VB2Vl{+nTiR&NtWm1UvRzLsSG^i;Kj816M%>D)crm))& zoqz(aW4h9B<9m>eX6t=J7S02HM~3YdF2U?&?GcV4^aH9`y>7>4r8Rs3_0p1jDzv_e zGYgT5GGspu{+I>F+e8oE6~W1}2R|yH2v5Mxc`jR|&xd%z3Xw<$CN%D51@mCv?V2_b zrlJaNGP~4nDbKF{uxcciw~WY6+A6k+!P(Tvp#FITP!PHH6+`6ZoQ6{vW=KWRg$Yqp{EqHyg&Gmi$&NoACS zH*e^fEnj;Pz-_2f7Eiaai$Z#SZt z^K@SRVJN}tAY6@uW-EMYgDXJ9*!s?O_Xk`gi&95XT47C+&?YgkO2X-_rl4PLC%1j$xJd zm#BN0`SnI@Ck_9>4o~4S0Td~X1JkJfQAnyUm&95htPz}Dr~yFILV33^f5IS;VBzXf zOWf(svi?~jR9|SWO}20y&su=IZ!bZx{NdNPJEM~5ECI{V>Piji)m*M_(QX0$8g{oI zX~ZEoiLtRpoRLJHp@#9*Z#KO?l^C@12pN&XRP) z?8H~c$}ayL$8J<2EA(^dR#+!>2qEYznc-g|MuFCDOUsuZUsr^_#&{pZAFS6SJ=%Y| z<-oUA<;M+DPdXNjRZr_LCxh2kljpP8UM+JjDEOEj>v7_4#Nnqly{j3?uGGVtL!3{1 z3gIvX;!S+5%d4c#%ouIH5}&<-?h=4xKleoWqng9uU4#G>Xp`@8Dd{6`xS{I{t42*- z8*8*{FO;oBJ|D9$<$t!w%u41SDbcF)XKX7v2KC&7=J9K*#bTnlb{V4fqk{`KJ+=0p zRzA`s=VD>KFxnXjoF)8hjPK`dj1xEk=o*RDfKW8hrdK*fAMd4u{Uy-)uofJ5SnxiiH>`}Xm1I+*GDpxv!d@qZ|xLfR4` zLM?BVcBEvRf2kB$m}$M(`m3nxN9#D?mw5A9M1>fwLghr&$nZ&>zbv|rv&nF(sLTevDZH1J7ds{VJU1mA8|A?)#RlR}xC3*7zLp~OrD@kFA8!O>E@&LC4 zKxC4o_?)5Sxr6Bfq`BCB2tqFJ$g}9IN!gU#qzi6yMW`~{JyS@sC*?f9e~S>()AcQY zIX)TLR7+f}#;eWWX_}DZ!!GsZFwhC89C{*uKCjS{K3LlQx!^yegkvZr(#_+ruuNnr zoOdpxF9-4>Y|-uMh3BX*qT`xgl=vv}%+pQDZYCKYHq#`c8ji&ZR!@|hzY=MBR>C14 z(G_+Cr=5Ocwc3qI8na(u5H4(*San7?b%(ADbrY#Mcr^s83~-~>+Wh1#2C}UbUhq&% zTUZ!a>GnhHE;y1G5MWk{=W5=yZT<`*R=%WW_KY=TQt*#izGnCSqKwWG27Iq~sIk5< z_rZ0CbUpW6=eLq$k#_~nFzFs-B_g`Utc7fKrNOqwd^;N zd%;SX8om1QWorg3j_h*M=xn3JWSTaZ> zV2$DN$)fXAbaK*<{z`(-5#k_({+yx8Xky)bWFz0~yN`R!DQP`;4T{GSMk;=LEQ+{I zhOi4QG`d!gM^Rr+GbNfrk^@X(+(Q+huNb6bUNhkv#v*wN!o&ZO5Rm_{+NbF`bm_K= zCC;I~)G5W*%b_TZd~Qn#ZzK;pii1iYs=&^F(5ZOZ%Li@gV^@w>I+)t5yL402%iUSu zk8}2Ez-?}nVS;!jAMF;ckfzbXjVn^k1g!ZO#=%<}OR$Nr@NqV(os~51ARe;x<`)#} z#alj&pIMoG(S#h_{6uDwKdnehq`!4rX6y^JlmtvaT#($x-QS8gdui$Fxu`ctUB zp@~SskWZizT8>+ zYbIbp%ZuaT;|Xy*gUYr9mCe;;Rq;3T5SQ=$PRXVxv-h2^sc#RbHhqkXYw>p8jPOY~ zpilS~;-iGr6z)^m!(vLp8rfD{gaW4yZ8ACD(yKDKbHA-C-3^M-pbRdLDm%K~YwOdyt|)a5!Ld(sTMaVRI_j%d=g5!&Ikfge{QA-o zkm}=!+pk7zAZPpHR?Y$KY}j%9^(M>kg9ZLEdo2I&trC@OZ+GJjT_*PK-ATY#z49Jy zWf$uyO)!qthIg(ncXpvn{Jv5c zc-dqN0(6t5G$5Ibf!sZ&py)m0z|OrKMu=}RYJSd+7)xNtdHa465RbcXMg-C-JyLV< zh+Qi&iZRtc%llNM3SR*l2|aF1dtk=~X;R#I8+}A@W=7!QIS`LtkfqAmHhpW;O$TKO zh?#Xf9z2Y^-eqSQ&hY`g)5RaM11Do+%6)yf%uR$A6g~A{&xJ`|M7WVKYZkBLZID_K zJD)>pVmNI2A~qA=Wt(`eWOZ`}u$e+r4smCenQZ+*3&U3(vgKd4>+0qK8S0k%Q*Lm) zJT^qtrk>V5p*unuKKC6KtD@lSFb=_Va;abA;FLdz$uB;C@LKj=GPCNDa6eVEZ2B}F znX=#W>em}sB=R#qGvy628v#QI#9F^oRpQ;4;y%mVZO30SD{mZWZ)AP|j^jIZyt983 zn!L?=nJkaO%jIhUmy8~~nU;%xorc#&wzNB1kx&x$A6{LMZ~a9Lm#UE4#Ptbx3zs1K zoowpO${HWmzvx;*J19eR4TKY?SbhoT0JKV#4`cAbHI`oN`&=p>0uSRw3Ik zEWzQ?DAeH=RWMDPVexSxp3e}#l4hRje2&iozq5{<>+4^jF~WQA)u;)a59v9_+U>cv zlENgd3qHp7bVDfoYF>YOfr7&C4P9}u3kh>$r&Px+qVW$}$Q0BZig0^J100Pf-mkZ9 zXovr6lBwlhmvJ18P>mwo;Ob{jX;+E2MWT5#Uq}Tv4q!d}r~NuS#J3|Ha#NPT>R0F9 zMzn;YJ0P(=J?+^O_ZAVOFroZiAUtG}Y%ftH!4&lPqJ9B6s0{4L*u&{p5g!>gJrNG@ z;+e@VWxwdOb=N&mh{_!%ST_Gloo`&qZVmMZgL@?P`iDiHbldo%hQ-TfliOx@Ji@qj zX>4tZtPw{bpykclbnXMh9mvQ0M}6tVDf`gi88)x-!$9o@30Dq3+FHO>9;A82vx$XK!Ivy9@f?~QPS0p#jI zt|fr9hKV(MSzlFHKFDWXC1E6(Ebmfr73mBQ#=q{9KZp$xu6^&b5Zmt(GpTi!BN;lbCSQR%i31n4hAwj@4Ja%*MU)w20;Py5pQ3wZKl zmif z4N(fb%fV1HNQ296Ag&@f{+_wjT_WQc!h;Lotu4Dz(B!_iuM=+j>fzy1Ai<(EnZE8| z?Uei-@p+=&tE=4Fh@H)aA2>(OIUPu8#eLHGJ;ybcBdWV?X9VXAB$o_HbiJUKS1|9u z^BWvjkVDESGn}XUYZ>(H1=y)=Mzs7xj3U!d_=6DSmw?J~{>)yye`O%@9}4k*L23yl zm#+U2cs>#n=82=ih}K0CkdVB}fFP>>b);aWQebuS>Jl~Ar|Yf#DPCHqHYbVw#;En>lF6gCVUS@GX1suZHX-|7AUH8Gmc z?I^%E2dT6~#GdbEKgvonO}{q6a#iNUIjL^^&n1C1VaMnrl#AXqyY)XhBOu!WY)7pr z`%Q6*Hs#n|pT(1~RXr0;3hs2u2hKXwIj?23#T1@TX6IR*dao^wK9@PnE68pc*WZ=8 zLlbOKaUdQ8SZAJ=3u)_m)P@p0I*)`W>Qi2tQq*36d#;PbnsC7FqSRhZ9_XMIn~$$Z zBZr5;y2V&DcW2Si1{uKiX5Ec^nDkSYo00Rx>daa3*5Oaj^nS|APoOR7uq>1E7ZeEn z;0PFI+A-Ks{$nwJKVoRGF$UYJ$*MB}?~U2l84cKvFn+hMQ(9`fd^y@^9We!WCI?z} zjDC8WC&*f0OsKX|7s((5jA)dlkwLb-PXuG5_ZtWiO=>!Ml~?_%U>c#FFH(SR9bt2Y zccn3%49;Z!U_kyrbH|Om1$dj+cWZ{8y}M@obzg#SiE>7I@<~5VO`M4r`!q3eB99v* z%O@9*OgoDp7JwfcZ>R2r;>dLdB*<8dymPV#1|`(e^E7RbHe0a4>RAQ0jB~ zd_PCdN*tIL@LO_-QNpT2X-V2FSO23(5*!)-&8epI6D|Wv*c8Fx8-160P?8#dDkly= z$)oq;YOGD@VqbFB7TZfvTM!C?O7@}fY6=n5`I|R4JzGD959xiS6If30HV*JPtx8~9 zzw6qVuG9W3@7KwTnb=nJ>L&4Plx=ilkZYsNigQZ*nU@chR^fXnybfth1sA1oTt9BIIh|nj3d7xU}r?1NR06e6m2fd&4i|&E%>9jIQF@H0yqaI(g zze})*J;Nf93D}Z-&#%MQVZ)RRQKe}?l|VGhmaeMcCz3u?)H8CZp3*zNU50ho{d7_W z>7D5>&hxj#qxN<5s1$;8PgDZIQH9*=C|y3=(nv6kM0Kwcs08lQ$fqByH;6r7<9g@%w--uRO`*AWqVfzJ>0rVcTbR>sc2j~--b?DBV(|AUe5Sfo| z^~EFO-#_F1xN@I-Limb0HfJ?ciSzM|7rESbj;7LwideVL@1NOzT?m?;I zhl%oUU$`?55wyAB)4PfOvmfa95YX{w!+;^rpT?*ND%nAE#&N17bOpDj3RW{H$*Pn7P`~$FhDEjnY)LKvkz2c z=$E%Ax`yMKP6~0hk{>JXNosr(ztHELOMc57nE@8aJi8zG;2x{p=^a!e6k7KpxX?H& zfBPW){AqgWH{4Ho-li*x3@1Qq(XCal?Gk(ey`}fa)C|WSD(WFLtNnI)oT8d{zwyHH ze&Pwn3HX%KFI9pul-y7SQ|uc9pv3vm=#Ly>RX~GsuH_j@Tk_k}>zIuz4(X%R$Wd5;S&^2O=Fni#(gCzz({OL zaQQAxLf)564)*Q6&lIV`pdnzJF>;wRUt<_Zf*Vo&pt0mPXDudA>myl@I(LM4n_Sp? zO4s7C_UG3Egslxs0(9S>^(kO_Mxj7x#fM*?qT$f2eEQZWP@kMf?pG)fQ#MG*C6{i7 zL{nbbsD9G;Lh;YSO1)ko!=>sHbhPx2_`2MUQ>D1ji|q9L=?ir$)UgvbE^2|R=bg%@ zti}V}$W4XSbh+eVU=_Yge_-Nr_rUDKnP07#=%e~E=wkAY@mhPr<2dKQu|9hDhi$3w z)sngwZgEv zipiSFvq^9Zv7rz^6jsxz~KDOk^TvBus-QjWh(&9J2&n z)=?5xn%9JT5*IgTuk#KNv%30;VS_i-o`>!}ahw}ewgp{##5JC2SYX|HE-^y&X5;E> zPTvy;@mHr1WM&}X2~%eUuy5c#w{7+yN7mbN=O7bS%R z-!CG^G%I314@&@Jp(0TY;#N=-lRwSF#eg+d99GKSzSw|_vEKX1z3vIj3~E~*j$(cb zGgxlJS_7$13j4Qh4B}!Kps}kmrn>g2y}@|!EUGyNq;U9jw>JyRsCvB zBgx|W!Pbu^87c%DNdLmF+Z~&!Lm1=ftV3N=)p$lOo}l2IKu@48Xe`+#&@l|_h~FP1 zWJ;|XjJ{=(DiXIB@8*(NM|9)f#1DcNS(M6^KowAbu|cX5;DJq`e^l5zKWuj|dY_kF zxX(~C0EVsRb}1eed!6#XEj9wRzX?Nw<(ogoGCRSY3pYux4q$m@@>~S$)(d~FFy>}g z$I`n?A;1U<-$#A2&+nsL5IyGUk74&4&zY%{#eu)c>*1DpN8S)B04es|uHn+>kat>1 z@V`mnrn}JY{9Wh6Tg};R8jtFweq{c`!kVLcx1_9{3d(sUKhmFTEwx~KKKRGr8z64K z`RbI$qm$lTefgGz<(Z$YU$6f$FCh~Xc1e!;)#~oj%um{67GeMO*@rQIj7(^>Gthpd zk@q-HGQpHFOsNo(4?(_QUI+ITuIA=Q16fTgus>>sX~>7+uGNq1xsEKXQlz)wTXy_| z$+|4>XBWu5!H#88c0Q9SZKYZPXz>~Y(wi$Iu*ATt+YyFl^gk2S_ApoPkd{>Mdncyj zhEkm8q(ojYWD)}Q8P2E@s5MpX4+HjfbO$DF$!gDzbGEhSM=iFmT@QPd9R909D+!&1 z&OOaH>f8q4h_LdAnyi?9rV!dEwY( zS+Y7|>{K_<6KhXhvt)lZix98@s{`)qiw~1j4j883rIMco3rO~_p&ygtUUsjtmcgj1 zsGsj0H<_oM)(QoIzATitS*1R^v8CNR7HD53ZB=UGLbpLYZU7`svgSIcjfQUmrQ^N~!pm%1w zH1vaLEY`2tBGaCHfk``gbyMJDfc5b3{a$8zx2R$~KVQnN5sZl(Lw9aoT8c2+##M_w zV*{xd`REFFj(A;t6={UjB4=R5gIfEZu<>o<7GpZ9$l7@_}iEQF!)T4)}GtN zzGhK7yQ7xZ-!UdJ{VyAc(lt7HLpJ#L8d8CGnk z(wdH2PzG~{Ry38L$2m&o_IVR%28$I8PeR|088-y3w0CQe5{A^UJo#5U?x=|aVNsW^ zE6Ap#j!5bkex)o~MBe`_Wd~l=dOAitA)SPtfKQ-nUsM8Ce8(?$2JV-LobPUN3#h1A zT)bdk_N9Qt`p=@P0HyH#5-1^GvaVwi&nT&hrjzVLTST{~W_}meO1GK{Y}t`dT8CsT zA_tzht|KBd{;b`}Kh+45*~*bop|)fet};z8Hz`^Bm)d0)X&NTbO5?M+I5Y7NB*2FZ z;`b6rA{pLW34;Z9j-{ZUi#_ns2|uvd&N-hgsi6MQAGg#zNdf<~-NX+r6Xze5v9i&n z$Ny~k*Tq2cyb@7eL^Ui_*sblWGu8uEvyV1>Vg2qHCcpsbe77nb%xU-JLRWhGe=R>3jHVtzTu>SA z;ElZ<5fo5M<)b*~ZA{c9aWmwjdHr*vba_KK7#WHQ28RKlORLM3asQ(H zb&f^)Ly9oydDQn3_a^Kmvs8}}ZXCqdHJ?@ik|ED}aGx-jQ6jZW| z?-^ZVkn|&gB=JN=_V0n!i`yn*yF==Rs-PvG3_90AuZ;^baCFbE!h296mwk-HfwD)H zz~f+i-3XY~Hb83b>?hf8h6tUTLOYX#?fOX?2PGUd0L=>Mx8}crr~^#8m%$+a2UwK@ z9UkR_z7f^}z6Z--9>He#-3F=-*m-!!`rtsP&}<6sLay{5ATiRFWe04av|9iI2^)d- zA>rc?U3)HmTgd5vxz#W~&w~eV6EE=(NZ1xJGx{9Lt3nG{Nt(S&F8J|#q6%rrFQ%`` zOB_fq`8#86vZ@`xrpS-o%$Wl>V4p2Ep-cx6E;#onXm;O#jmp;nE1!rFj(q!Bl(W4J^X%OPXmpi7V0(0cE;V92c!{o^BM>=XgG_R}ew41kBb7J}kK z)3A0!WG^#0A97eM>d>lk6g+#-l6Z+Ro=O4;%l{g_G6Cn!&An2b;7|xz3F`>{v*y@v z(Kjks8|_Qi?_tOMgBFDo-tYH4ZhpTXQPnOGKK~XSBn|QK_R)-OA9X~|Q8%oDml?M&#gP38Zsag8cu!D3%l)PfS`X@v zAEyRAIgfR8*WR@5klot-9Y0D#p*UL_@;%b+*UGqH5i_N4Lkf+)e*=|H-^*u>XOCT@ zumAUr|9(0D9y$LhU&nVv0xNEfVhfcW+|K>z?PdJoPtkpPy&~1?6VcB!85GM-4d6m< z@dV!CNi+xo?nl2@%1AQR%m@3;b9tdDB|?&V@60__teX}cibr1mk{u{>djg&tE^x_q zSbybI@iDP`SYzxM|69A;SV=Qgf=$>N)P?e@UobQg z;}NoC+%wZ*yZGJ4^!=6IcLdEg8TbbQ&$lksnps5Wb+#S(_>IP=Z#lOX%WF-eCdhT^&;jTIMkK3MmnGS*kCEsz|Nn^Sya4-q-|;v#g4SVGT8FfjromyoFr=$Yza_$kX{ P(BIwL+N#CM58wV5T|_oy literal 0 HcmV?d00001 From ac19f5f1910833884f9e93ce2bdd8b20bab025a3 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 12 Jul 2023 18:37:05 -0400 Subject: [PATCH 142/169] Update network/alsp/manager/manager_test.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/alsp/manager/manager_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 8563053b48d..faba4d3f6a3 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -281,16 +281,6 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { cfg := managerCfgFixture(t) - // this test is assessing the integration of the ALSP manager with the network. As the ALSP manager is an attribute - // of the network, we need to configure the ALSP manager via the network configuration, and let the network create - // the ALSP manager. - var victimSpamRecordCache alsp.SpamRecordCache - cfg.Opts = []alspmgr.MisbehaviorReportManagerOption{ - alspmgr.WithSpamRecordsCacheFactory(func(logger zerolog.Logger, size uint32, metrics module.HeroCacheMetrics) alsp.SpamRecordCache { - victimSpamRecordCache = internal.NewSpamRecordCache(size, logger, metrics, model.SpamRecordFactory()) - return victimSpamRecordCache - }), - } slashingMisbehaviors := []network.Misbehavior{ alsp.InvalidMessage, alsp.SenderEjected, alsp.UnauthorizedUnicastOnChannel, From c924c1c2ce89e746d3db112a26242bf3c28034f6 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 12 Jul 2023 18:38:57 -0400 Subject: [PATCH 143/169] Update network/alsp/manager/manager_test.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/alsp/manager/manager_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index faba4d3f6a3..1feada8792e 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -279,7 +279,6 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) // The test ensures that despite attempting on connections, no inbound or outbound connections between the victim and // the pruned spammer nodes are established. func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { - cfg := managerCfgFixture(t) slashingMisbehaviors := []network.Misbehavior{ From ca4fc5bc8931186db12a5d49b45b3507fe03ef23 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 12 Jul 2023 18:39:19 -0400 Subject: [PATCH 144/169] Update network/alsp/manager/manager_test.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/alsp/manager/manager_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 1feada8792e..c0442662ab7 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -290,7 +290,7 @@ func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, len(slashingMisbehaviors)+2, p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) - networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(cfg)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) victimNetwork, err := p2p.NewNetwork(networkCfg) require.NoError(t, err) From f67f007425330b9e0b1b61607c6f8d74159e1e69 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 12 Jul 2023 18:39:49 -0400 Subject: [PATCH 145/169] Update network/alsp/manager/manager_test.go Co-authored-by: Yahya Hassanzadeh, Ph.D. --- network/alsp/manager/manager_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index c0442662ab7..6c6db80f6bb 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -281,14 +281,13 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { - slashingMisbehaviors := []network.Misbehavior{ - alsp.InvalidMessage, alsp.SenderEjected, alsp.UnauthorizedUnicastOnChannel, - alsp.UnauthorizedPublishOnChannel, alsp.UnknownMsgType, - } - // create 1 victim node, 1 honest node and a node for each slashing violation - ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, len(slashingMisbehaviors)+2, - p2ptest.WithPeerManagerEnabled(p2ptest.PeerManagerConfigFixture(), nil)) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, + 7) // creates 7 nodes (1 victim, 1 honest, 5 spammer nodes one for each slashing violation). + mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) + networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) + victimNetwork, err := p2p.NewNetwork(networkCfg) + require.NoError(t, err) mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) victimNetwork, err := p2p.NewNetwork(networkCfg) From 07796e92950d9e446db0a24e2fc0d5d18ba31790 Mon Sep 17 00:00:00 2001 From: Khalil Claybon Date: Wed, 12 Jul 2023 18:41:42 -0400 Subject: [PATCH 146/169] Update manager_test.go --- network/alsp/manager/manager_test.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/network/alsp/manager/manager_test.go b/network/alsp/manager/manager_test.go index 6c6db80f6bb..b013688bf8b 100644 --- a/network/alsp/manager/manager_test.go +++ b/network/alsp/manager/manager_test.go @@ -8,8 +8,6 @@ import ( "testing" "time" - "github.com/onflow/flow-go/network/slashing" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -32,6 +30,7 @@ import ( "github.com/onflow/flow-go/network/mocknetwork" "github.com/onflow/flow-go/network/p2p" p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/utils/unittest" ) @@ -280,14 +279,8 @@ func TestHandleReportedMisbehavior_And_DisallowListing_Integration(t *testing.T) // the pruned spammer nodes are established. func TestHandleReportedMisbehavior_And_SlashingViolationsConsumer_Integration(t *testing.T) { - // create 1 victim node, 1 honest node and a node for each slashing violation - ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, - 7) // creates 7 nodes (1 victim, 1 honest, 5 spammer nodes one for each slashing violation). - mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) - networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) - victimNetwork, err := p2p.NewNetwork(networkCfg) - require.NoError(t, err) + ids, nodes, _ := testutils.LibP2PNodeForMiddlewareFixture(t, 7) // creates 7 nodes (1 victim, 1 honest, 5 spammer nodes one for each slashing violation). mws, _ := testutils.MiddlewareFixtures(t, ids, nodes, testutils.MiddlewareConfigFixture(t), mocknetwork.NewViolationsConsumer(t)) networkCfg := testutils.NetworkConfigFixture(t, *ids[0], ids, mws[0], p2p.WithAlspConfig(managerCfgFixture(t))) victimNetwork, err := p2p.NewNetwork(networkCfg) From 6228e63f66fa2c2be96dce2783f39d877b762d6f Mon Sep 17 00:00:00 2001 From: Faye Amacker <33205765+fxamacker@users.noreply.github.com> Date: Fri, 14 Jul 2023 15:33:24 -0500 Subject: [PATCH 147/169] Fix missing events in uploaded block data Previously, only last event is included in block data (repeated n times). This commit fixes the bug and includes all events. --- engine/execution/ingestion/uploader/model.go | 7 ++++--- .../execution/ingestion/uploader/model_test.go | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/engine/execution/ingestion/uploader/model.go b/engine/execution/ingestion/uploader/model.go index ba01f27ca28..fc39dd08393 100644 --- a/engine/execution/ingestion/uploader/model.go +++ b/engine/execution/ingestion/uploader/model.go @@ -29,9 +29,10 @@ func ComputationResultToBlockData(computationResult *execution.ComputationResult txResults[i] = &AllResults[i] } - events := make([]*flow.Event, 0) - for _, e := range computationResult.AllEvents() { - events = append(events, &e) + eventsList := computationResult.AllEvents() + events := make([]*flow.Event, len(eventsList)) + for i := 0; i < len(eventsList); i++ { + events[i] = &eventsList[i] } trieUpdates := make( diff --git a/engine/execution/ingestion/uploader/model_test.go b/engine/execution/ingestion/uploader/model_test.go index c58979eb44f..5f78824ebe4 100644 --- a/engine/execution/ingestion/uploader/model_test.go +++ b/engine/execution/ingestion/uploader/model_test.go @@ -11,6 +11,7 @@ import ( "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/pathfinder" "github.com/onflow/flow-go/ledger/complete" + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -29,12 +30,23 @@ func Test_ComputationResultToBlockDataConversion(t *testing.T) { assert.Equal(t, result, *blockData.TxResults[i]) } - // ramtin: warning returned events are not preserving orders, - // but since we are going to depricate this part of logic, - // I'm not going to spend more time fixing this mess + // Since returned events are not preserving orders, + // use map with event.ID() as key to confirm all events + // are included. allEvents := cr.AllEvents() require.Equal(t, len(allEvents), len(blockData.Events)) + eventsInBlockData := make(map[flow.Identifier]flow.Event) + for _, e := range blockData.Events { + eventsInBlockData[e.ID()] = *e + } + + for _, expectedEvent := range allEvents { + event, ok := eventsInBlockData[expectedEvent.ID()] + require.True(t, ok) + require.Equal(t, expectedEvent, event) + } + assert.Equal(t, len(expectedTrieUpdates), len(blockData.TrieUpdates)) assert.Equal(t, cr.CurrentEndState(), blockData.FinalStateCommitment) From f88f0979153f6511e61ce5ad48f05277d38d2cab Mon Sep 17 00:00:00 2001 From: Alexey Ivanov Date: Tue, 7 Feb 2023 16:58:44 -0800 Subject: [PATCH 148/169] Go 1.20 --- .github/workflows/bench.yml | 2 +- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- .github/workflows/test-monitor-flaky.yml | 2 +- .github/workflows/test-monitor-regular-skipped.yml | 2 +- .github/workflows/tools.yml | 2 +- cmd/Dockerfile | 4 ++-- cmd/testclient/go.mod | 2 +- crypto/Dockerfile | 2 +- crypto/go.mod | 2 +- go.mod | 2 +- insecure/go.mod | 2 +- integration/benchmark/cmd/manual/Dockerfile | 2 +- integration/go.mod | 2 +- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ada29474be7..fbe5bd1bf59 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -41,7 +41,7 @@ jobs: - name: Setup go uses: actions/setup-go@v3 with: - go-version: "1.19" + go-version: "1.20" cache: true - name: Build relic diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index eb28e840078..9079fb06a98 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Checkout repo uses: actions/checkout@v2 - name: Build relic diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5772ef5dcb3..e1c0484faa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: - 'v[0-9]+.[0-9]+' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index 3a5b47e2c2f..f6637edf0ae 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -5,7 +5,7 @@ on: branches: - '**/*flaky-test-debug*' env: - GO_VERSION: 1.19 + GO_VERSION: "1.20" #concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-flaky.yml b/.github/workflows/test-monitor-flaky.yml index fcf215b734e..442d71c3e07 100644 --- a/.github/workflows/test-monitor-flaky.yml +++ b/.github/workflows/test-monitor-flaky.yml @@ -13,7 +13,7 @@ on: env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/test-monitor-regular-skipped.yml b/.github/workflows/test-monitor-regular-skipped.yml index 74736a00431..d9f696ab87c 100644 --- a/.github/workflows/test-monitor-regular-skipped.yml +++ b/.github/workflows/test-monitor-regular-skipped.yml @@ -15,7 +15,7 @@ env: BIGQUERY_DATASET: production_src_flow_test_metrics BIGQUERY_TABLE: skipped_tests BIGQUERY_TABLE2: test_results - GO_VERSION: 1.19 + GO_VERSION: "1.20" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 2e297adb6ff..9f228f215ba 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.19' + go-version: "1.20" - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@v1 with: diff --git a/cmd/Dockerfile b/cmd/Dockerfile index fc4bcf7badb..d9d7800546c 100644 --- a/cmd/Dockerfile +++ b/cmd/Dockerfile @@ -3,7 +3,7 @@ #################################### ## (1) Setup the build environment -FROM golang:1.19-bullseye AS build-setup +FROM golang:1.20-bullseye AS build-setup RUN apt-get update RUN apt-get -y install cmake zip @@ -71,7 +71,7 @@ RUN --mount=type=ssh \ RUN chmod a+x /app/app ## (4) Add the statically linked debug binary to a distroless image configured for debugging -FROM golang:1.19-bullseye as debug +FROM golang:1.20-bullseye as debug RUN go install github.com/go-delve/delve/cmd/dlv@latest diff --git a/cmd/testclient/go.mod b/cmd/testclient/go.mod index 0a02e69ad42..dbe66a78fb5 100644 --- a/cmd/testclient/go.mod +++ b/cmd/testclient/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/cmd/testclient -go 1.19 +go 1.20 require ( github.com/onflow/flow-go-sdk v0.4.1 diff --git a/crypto/Dockerfile b/crypto/Dockerfile index 37a0b373171..d75e9543de4 100644 --- a/crypto/Dockerfile +++ b/crypto/Dockerfile @@ -1,6 +1,6 @@ # gcr.io/dl-flow/golang-cmake -FROM golang:1.19-buster +FROM golang:1.20-buster RUN apt-get update RUN apt-get -y install cmake zip RUN go install github.com/axw/gocov/gocov@latest diff --git a/crypto/go.mod b/crypto/go.mod index c7fe54f9ff5..9895e1c35db 100644 --- a/crypto/go.mod +++ b/crypto/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/crypto -go 1.19 +go 1.20 require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 diff --git a/go.mod b/go.mod index ef9c29a3a43..7262dd6f144 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go -go 1.19 +go 1.20 require ( cloud.google.com/go/compute/metadata v0.2.3 diff --git a/insecure/go.mod b/insecure/go.mod index fba888f2997..1c6985ffbc2 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/insecure -go 1.19 +go 1.20 require ( github.com/golang/protobuf v1.5.3 diff --git a/integration/benchmark/cmd/manual/Dockerfile b/integration/benchmark/cmd/manual/Dockerfile index 1ad38985a43..58f2b71d42b 100644 --- a/integration/benchmark/cmd/manual/Dockerfile +++ b/integration/benchmark/cmd/manual/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:experimental # NOTE: Must be run in the context of the repo's root directory -FROM golang:1.19-buster AS build-setup +FROM golang:1.20-buster AS build-setup RUN apt-get update RUN apt-get -y install cmake zip diff --git a/integration/go.mod b/integration/go.mod index f59d02427b4..75f2e2ed45f 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/flow-go/integration -go 1.19 +go 1.20 require ( cloud.google.com/go/bigquery v1.48.0 From 0bd284c8c92f11a5dd9b3602880ff218a97e75e2 Mon Sep 17 00:00:00 2001 From: Kay-Zee Date: Wed, 8 Feb 2023 13:13:38 -0800 Subject: [PATCH 149/169] Update golangci-lint --- .github/workflows/ci.yml | 2 +- .github/workflows/flaky-test-debug.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1c0484faa4..26d14496fb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 diff --git a/.github/workflows/flaky-test-debug.yml b/.github/workflows/flaky-test-debug.yml index f6637edf0ae..8058a656f29 100644 --- a/.github/workflows/flaky-test-debug.yml +++ b/.github/workflows/flaky-test-debug.yml @@ -36,7 +36,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.49 + version: v1.51 args: -v --build-tags relic working-directory: ${{ matrix.dir }} # https://github.com/golangci/golangci-lint-action/issues/244 From 81bc2d873e85eeceaf0d8c23f20fc4a4d7edd564 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 17 Jul 2023 11:54:35 -0600 Subject: [PATCH 150/169] update more go1.20 deprecated functions --- engine/access/state_stream/backend_executiondata_test.go | 3 --- fvm/crypto/hash_test.go | 2 +- ledger/common/testutils/testutils.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/engine/access/state_stream/backend_executiondata_test.go b/engine/access/state_stream/backend_executiondata_test.go index b619a94e322..361cb64aa80 100644 --- a/engine/access/state_stream/backend_executiondata_test.go +++ b/engine/access/state_stream/backend_executiondata_test.go @@ -3,7 +3,6 @@ package state_stream import ( "context" "fmt" - "math/rand" "testing" "time" @@ -65,8 +64,6 @@ func TestBackendExecutionDataSuite(t *testing.T) { } func (s *BackendExecutionDataSuite) SetupTest() { - rand.Seed(time.Now().UnixNano()) - logger := unittest.Logger() s.state = protocolmock.NewState(s.T()) diff --git a/fvm/crypto/hash_test.go b/fvm/crypto/hash_test.go index bb9bb64172b..58d15d19b17 100644 --- a/fvm/crypto/hash_test.go +++ b/fvm/crypto/hash_test.go @@ -1,7 +1,7 @@ package crypto_test import ( - "math/rand" + "crypto/rand" "testing" "crypto/sha256" diff --git a/ledger/common/testutils/testutils.go b/ledger/common/testutils/testutils.go index ab30000c47c..e0e100ee46c 100644 --- a/ledger/common/testutils/testutils.go +++ b/ledger/common/testutils/testutils.go @@ -206,7 +206,7 @@ func RandomValues(n int, minByteSize, maxByteSize int) []l.Value { byteSize = minByteSize + rand.Intn(maxByteSize-minByteSize) } value := make([]byte, byteSize) - _, err := rand.Read(value) + _, err := crand.Read(value) if err != nil { panic("random generation failed") } From 79c011b6a1eaf1f55a78b5810ff335411a60efb2 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 17 Jul 2023 13:08:44 -0600 Subject: [PATCH 151/169] remove math/rand.Read calls --- utils/unittest/fixtures.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index f5d454d01b0..aea81107d29 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -417,7 +417,7 @@ func BlockHeaderFixture(opts ...func(header *flow.Header)) *flow.Header { func CidFixture() cid.Cid { data := make([]byte, 1024) - _, _ = rand.Read(data) + _, _ = crand.Read(data) return blocks.NewBlock(data).Cid() } @@ -2451,7 +2451,7 @@ func ChunkExecutionDataFixture(t *testing.T, minSize int, opts ...func(*executio } v := make([]byte, size) - _, err := rand.Read(v) + _, err := crand.Read(v) require.NoError(t, err) k, err := ced.TrieUpdate.Payloads[0].Key() From ef8670563b9f0b0802c1c52b63e526e0cd8b107a Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 17 Jul 2023 16:10:17 -0600 Subject: [PATCH 152/169] remove unused function --- utils/unittest/fixtures.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index aea81107d29..5f3d5f86e95 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -9,8 +9,6 @@ import ( "testing" "time" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" @@ -415,12 +413,6 @@ func BlockHeaderFixture(opts ...func(header *flow.Header)) *flow.Header { return header } -func CidFixture() cid.Cid { - data := make([]byte, 1024) - _, _ = crand.Read(data) - return blocks.NewBlock(data).Cid() -} - func BlockHeaderFixtureOnChain( chainID flow.ChainID, opts ...func(header *flow.Header), From 6b9cce5e8250e4f2f1f6f1b07f6f3f450a8f28f5 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 17 Jul 2023 16:29:15 -0600 Subject: [PATCH 153/169] use assert.Zero --- engine/verification/utils/unittest/helper.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/verification/utils/unittest/helper.go b/engine/verification/utils/unittest/helper.go index fee800b34e9..62f26cd7f70 100644 --- a/engine/verification/utils/unittest/helper.go +++ b/engine/verification/utils/unittest/helper.go @@ -595,10 +595,10 @@ func withConsumers(t *testing.T, } // verifies memory resources are cleaned up all over pipeline - assert.Equal(t, verNode.BlockConsumer.Size(), uint(0)) - assert.Equal(t, verNode.ChunkConsumer.Size(), uint(0)) - assert.Equal(t, verNode.ChunkStatuses.Size(), uint(0)) - assert.Equal(t, verNode.ChunkRequests.Size(), uint(0)) + assert.Zero(t, verNode.BlockConsumer.Size()) + assert.Zero(t, verNode.ChunkConsumer.Size()) + assert.Zero(t, verNode.ChunkStatuses.Size()) + assert.Zero(t, verNode.ChunkRequests.Size()) } // bootstrapSystem is a test helper that bootstraps a flow system with node of each main roles (except execution nodes that are two). From e7dc893cab579f8b1a41da705b36fe5d3e72ee08 Mon Sep 17 00:00:00 2001 From: "Yahya Hassanzadeh, Ph.D" Date: Tue, 18 Jul 2023 14:17:12 -0700 Subject: [PATCH 154/169] [Networking] Handling iHave overpromising part-1 (#4556) * adds scoring parameters for mesh message delivery * refactors config-based approach to override-based approach * generates mocks * adds a documentation * extends godoc * adds test skeleton * fixes test * implements under-performing test * adds test for under-delivery in two topics * refactors test fixture * wip * err fix * applies refactoring * fixes test * parallelize the fixture * adds logs for overriding parameters * adds logging for network type to builder * fmt * renames a fixture function * adds warn logging for overriding score parameters * revises godocs * adds readme * fixes scoring test * fixing lint * lint fix * improves test quality * Update network/p2p/builder.go Co-authored-by: Tarak Ben Youssef <50252200+tarakby@users.noreply.github.com> --------- Co-authored-by: Tarak Ben Youssef <50252200+tarakby@users.noreply.github.com> --- insecure/corruptlibp2p/spam_test.go | 6 +- .../test/gossipsub/rpc_inspector/utils.go | 6 +- .../validation_inspector_test.go | 13 +- .../test/gossipsub/scoring/scoring_test.go | 375 +++++++++++++++++- network/network.go | 11 + network/p2p/builder.go | 64 ++- .../p2p/connection/connection_gater_test.go | 13 +- network/p2p/connection/peerManager.go | 2 +- network/p2p/mock/gossip_sub_builder.go | 22 +- network/p2p/mock/node_builder.go | 6 +- .../p2pbuilder/gossipsub/gossipSubBuilder.go | 69 ++-- network/p2p/p2pbuilder/libp2pNodeBuilder.go | 31 +- network/p2p/p2pnode/gossipSubAdapter.go | 1 + network/p2p/scoring/README.md | 79 ++++ network/p2p/scoring/app_score_test.go | 14 +- network/p2p/scoring/score_option.go | 145 ++++++- network/p2p/scoring/scoring_test.go | 16 +- .../scoring/subscription_validator_test.go | 6 +- network/p2p/test/fixtures.go | 148 +++++-- .../p2p/tracer/gossipSubScoreTracer_test.go | 4 +- utils/logging/consts.go | 12 +- 21 files changed, 857 insertions(+), 186 deletions(-) diff --git a/insecure/corruptlibp2p/spam_test.go b/insecure/corruptlibp2p/spam_test.go index 06f6183c03e..2886b598c66 100644 --- a/insecure/corruptlibp2p/spam_test.go +++ b/insecure/corruptlibp2p/spam_test.go @@ -71,9 +71,9 @@ func TestSpam_IHave(t *testing.T) { // this is vital as the spammer will circumvent the normal pubsub subscription mechanism and send iHAVE messages directly to the victim. // without a prior connection established, directly spamming pubsub messages may cause a race condition in the pubsub implementation. p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) // prepare to spam - generate iHAVE control messages diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go index 164634236bc..2307c57f0ab 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/utils.go @@ -20,9 +20,9 @@ func startNodesAndEnsureConnected(t *testing.T, ctx irrecoverable.SignalerContex // this is vital as the spammer will circumvent the normal pubsub subscription mechanism and send iHAVE messages directly to the victim. // without a prior connection established, directly spamming pubsub messages may cause a race condition in the pubsub implementation. p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) - return unittest.ProposalFixture(), blockTopic + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) } diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 5917ceee31e..3dd873d0e7c 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -1145,7 +1145,7 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), ) ids := flow.IdentityList{&victimId, &spammer.SpammerId} @@ -1196,9 +1196,9 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) // as nodes started fresh and no spamming has happened yet, the nodes should be able to exchange messages on the topic. - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) - return unittest.ProposalFixture(), blockTopic + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) // prepares spam graft and prune messages with different strategies. @@ -1228,9 +1228,8 @@ func TestGossipSubSpamMitigationIntegration(t *testing.T) { // now we expect the detection and mitigation to kick in and the victim node to disconnect from the spammer node. // so the spammer and victim nodes should not be able to exchange messages on the topic. - p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkID) - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) } diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index ec024775cf0..2fc1135b5bb 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -5,18 +5,21 @@ import ( "testing" "time" + pubsub "github.com/libp2p/go-libp2p-pubsub" pubsub_pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/insecure/corruptlibp2p" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/messages" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/scoring" p2ptest "github.com/onflow/flow-go/network/p2p/test" + validator "github.com/onflow/flow-go/network/validator/pubsub" "github.com/onflow/flow-go/utils/unittest" ) @@ -107,7 +110,7 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun idProvider, p2ptest.WithRole(role), p2ptest.WithPeerScoreTracerInterval(1*time.Second), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), ) idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() @@ -122,8 +125,8 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) // checks end-to-end message delivery works on GossipSub - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) totalSpamMessages := 20 @@ -167,7 +170,369 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun // ensure that the topic snapshot of the spammer contains a record of at least (60%) of the spam messages sent. The 60% is to account for the messages that were delivered before the score was updated, after the spammer is PRUNED, as well as to account for decay. require.True(t, blkTopicSnapshot.InvalidMessageDeliveries > 0.6*float64(totalSpamMessages), "invalid message deliveries must be greater than %f. invalid message deliveries: %f", 0.9*float64(totalSpamMessages), blkTopicSnapshot.InvalidMessageDeliveries) - p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, func() (interface{}, channels.Topic) { - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{victimNode}, []p2p.LibP2PNode{spammer.SpammerNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic tests that when a peer is under-performing in a topic mesh, its score is (slightly) penalized. +func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + underPerformerNode, underPerformerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", underPerformerNode.Host().ID()).Return(&underPerformerId, true).Maybe() + ids := flow.IdentityList{&underPerformerId, &thisId} + nodes := []p2p.LibP2PNode{underPerformerNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // initially both nodes should be able to publish and receive messages from each other in the topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // Also initially the under-performing node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and the under-performing node is considered staked + // as it is in the id provider of thisNode. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + return true + }, 1*time.Second, 100*time.Millisecond) + + // however, after one decay interval, we expect the score of the under-performing node to be penalized by -0.05 * MaxAppSpecificReward as + // it has not been able to deliver messages to this node in the topic mesh since the past decay interval. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore > 0.96*scoring.MaxAppSpecificReward { // score must be penalized by -0.05 * MaxAppSpecificReward. + // 0.96 is to account for floating point errors. + return false + } + if underPerformingNodeScore < scoring.DefaultGossipThreshold { // even the node is slightly penalized, it should still be able to gossip with this node. + return false + } + if underPerformingNodeScore < scoring.DefaultPublishThreshold { // even the node is slightly penalized, it should still be able to publish to this node. + return false + } + if underPerformingNodeScore < scoring.DefaultGraylistThreshold { // even the node is slightly penalized, it should still be able to establish rpc connection with this node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + // even though the under-performing node is penalized, it should still be able to publish and receive messages from this node in the topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics tests that when a peer is under-performing in two topics, it is penalized in both topics. +func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + dkgTopic := channels.TopicFromChannel(channels.DKGCommittee, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + dkgTopicOverrideParams := scoring.DefaultTopicScoreParams() + dkgTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + dkgTopic: dkgTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + underPerformerNode, underPerformerId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", underPerformerNode.Host().ID()).Return(&underPerformerId, true).Maybe() + ids := flow.IdentityList{&underPerformerId, &thisId} + nodes := []p2p.LibP2PNode{underPerformerNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // subscribe to the topics. + for _, node := range nodes { + for _, topic := range []channels.Topic{blockTopic, dkgTopic} { + _, err := node.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + } + } + + // Initially the under-performing node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and the under-performing node is considered staked + // as it is in the id provider of thisNode. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + return true + }, 2*time.Second, 100*time.Millisecond) + + // No message delivery happens intentionally, so that the under-performing node is penalized. + + // however, after one decay interval, we expect the score of the under-performing node to be penalized by ~ 2 * -0.05 * MaxAppSpecificReward. + require.Eventually(t, func() bool { + underPerformingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(underPerformerNode.Host().ID()) + if !ok { + return false + } + if underPerformingNodeScore > 0.91*scoring.MaxAppSpecificReward { // score must be penalized by ~ 2 * -0.05 * MaxAppSpecificReward. + // 0.91 is to account for the floating point errors. + return false + } + if underPerformingNodeScore < scoring.DefaultGossipThreshold { // even the node is slightly penalized, it should still be able to gossip with this node. + return false + } + if underPerformingNodeScore < scoring.DefaultPublishThreshold { // even the node is slightly penalized, it should still be able to publish to this node. + return false + } + if underPerformingNodeScore < scoring.DefaultGraylistThreshold { // even the node is slightly penalized, it should still be able to establish rpc connection with this node. + return false + } + + return true + }, 3*time.Second, 100*time.Millisecond) + + // even though the under-performing node is penalized, it should still be able to publish and receive messages from this node in both topic meshes. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, dkgTopic, 1, func() interface{} { + return unittest.DKGMessageFixture() + }) +} + +// TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted tests that replayed messages will not be counted towards the mesh message deliveries. +func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + + idProvider := mock.NewIdentityProvider(t) + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + thisNode, thisId := p2ptest.NodeFixture( // this node is the one that will be penalizing the under-performer node. + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride( + &p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + replayingNode, replayingId := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + ) + + idProvider.On("ByPeerID", thisNode.Host().ID()).Return(&thisId, true).Maybe() + idProvider.On("ByPeerID", replayingNode.Host().ID()).Return(&replayingId, true).Maybe() + ids := flow.IdentityList{&replayingId, &thisId} + nodes := []p2p.LibP2PNode{replayingNode, thisNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // initially both nodes should be able to publish and receive messages from each other in the block topic mesh. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // Initially the replaying node should have a score that is at least equal to the MaxAppSpecificReward. + // The reason is in our scoring system, we reward the staked nodes by MaxAppSpecificReward, and initially every node is considered staked + // as it is in the id provider of thisNode. + initialReplayingNodeScore := float64(0) + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + if replayingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // replaying node acts honestly and sends 200 block proposals on the topic mesh. This is twice the + // defaultTopicMeshMessageDeliveryThreshold, which prevents the replaying node to be penalized. + proposalList := make([]*messages.BlockProposal, 200) + for i := 0; i < len(proposalList); i++ { + proposalList[i] = unittest.ProposalFixture() + } + i := -1 + p2ptest.EnsurePubsubMessageExchangeFromNode(t, ctx, replayingNode, thisNode, blockTopic, len(proposalList), func() interface{} { + i += 1 + return proposalList[i] + }) + + // as the replaying node is not penalized, we expect its score to be equal to the initial score. + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + if replayingNodeScore < scoring.MaxAppSpecificReward { + // ensure the score is high enough so that gossip is routed by victim node to spammer node. + return false + } + if replayingNodeScore != initialReplayingNodeScore { + // ensure the score is not penalized. + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // now the replaying node acts maliciously and just replays the same messages again. + i = -1 + p2ptest.EnsureNoPubsubMessageExchange(t, ctx, []p2p.LibP2PNode{replayingNode}, []p2p.LibP2PNode{thisNode}, blockTopic, len(proposalList), func() interface{} { + i += 1 + return proposalList[i] + }) + + // since the last decay interval, the replaying node has not delivered anything new, so its score should be penalized for under-performing. + require.Eventually(t, func() bool { + replayingNodeScore, ok := thisNode.PeerScoreExposer().GetScore(replayingNode.Host().ID()) + if !ok { + return false + } + + if replayingNodeScore >= initialReplayingNodeScore { + // node must be penalized for just replaying the same messages. + return false + } + + if replayingNodeScore >= scoring.MaxAppSpecificReward { + // node must be penalized for just replaying the same messages. + return false + } + + // following if-statements check that even though the node is penalized, it is not penalized too much, and + // can still participate in the network. We don't desire to disallow list a node for just under-performing. + if replayingNodeScore < scoring.DefaultGossipThreshold { + return false + } + + if replayingNodeScore < scoring.DefaultPublishThreshold { + return false + } + + if replayingNodeScore < scoring.DefaultGraylistThreshold { + return false + } + + initialReplayingNodeScore = replayingNodeScore + return true + }, 2*time.Second, 100*time.Millisecond) + + // even though the replaying node is penalized, it should still be able to publish and receive messages from this node in both topic meshes. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) } diff --git a/network/network.go b/network/network.go index 4f77892b666..38896633e4d 100644 --- a/network/network.go +++ b/network/network.go @@ -13,6 +13,17 @@ import ( // and private (i.e., staked) networks. type NetworkingType uint8 +func (t NetworkingType) String() string { + switch t { + case PrivateNetwork: + return "private" + case PublicNetwork: + return "public" + default: + return "unknown" + } +} + const ( // PrivateNetwork indicates that the staked private-side of the Flow blockchain that nodes can only join and leave // with a staking requirement. diff --git a/network/p2p/builder.go b/network/p2p/builder.go index 3bd8e278716..b856f931a29 100644 --- a/network/p2p/builder.go +++ b/network/p2p/builder.go @@ -28,7 +28,6 @@ type GossipSubAdapterConfigFunc func(*BasePubSubAdapterConfig) PubSubAdapterConf // GossipSubBuilder provides a builder pattern for creating a GossipSub pubsub system. type GossipSubBuilder interface { - PeerScoringBuilder // SetHost sets the host of the builder. // If the host has already been set, a fatal error is logged. SetHost(host.Host) @@ -45,9 +44,16 @@ type GossipSubBuilder interface { // We expect the node to initialize with a default gossipsub config. Hence, this function overrides the default config. SetGossipSubConfigFunc(GossipSubAdapterConfigFunc) - // SetGossipSubPeerScoring sets the gossipsub peer scoring of the builder. - // If the gossipsub peer scoring flag has already been set, a fatal error is logged. - SetGossipSubPeerScoring(bool) + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. + // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. + // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. + // Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. + // Args: + // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. + // Returns: + // none + EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. // If the gossipsub score tracer interval has already been set, a fatal error is logged. @@ -81,16 +87,6 @@ type GossipSubBuilder interface { Build(irrecoverable.SignalerContext) (PubSubAdapter, error) } -type PeerScoringBuilder interface { - // SetTopicScoreParams sets the topic score parameters for the given topic. - // If the topic score parameters have already been set for the given topic, it is overwritten. - SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) - - // SetAppSpecificScoreParams sets the application specific score parameters for the given topic. - // If the application specific score parameters have already been set for the given topic, it is overwritten. - SetAppSpecificScoreParams(func(peer.ID) float64) -} - // GossipSubRpcInspectorSuiteFactoryFunc is a function that creates a new RPC inspector suite. It is used to create // RPC inspectors for the gossipsub protocol. The RPC inspectors are used to inspect and validate // incoming RPC messages before they are processed by the gossipsub protocol. @@ -123,11 +119,16 @@ type NodeBuilder interface { SetConnectionGater(ConnectionGater) NodeBuilder SetRoutingSystem(func(context.Context, host.Host) (routing.Routing, error)) NodeBuilder - // EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. - // Arguments: - // - module.IdentityProvider: the identity provider for the node (must be set before calling this method). - // - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. - EnableGossipSubPeerScoring(*PeerScoringConfig) NodeBuilder + // EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. + // Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. + // Anything that is left to nil or zero value in the override will be ignored and the default value will be used. + // Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. + // Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. + // Args: + // - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. + // Returns: + // none + EnableGossipSubScoringWithOverride(*PeerScoringConfigOverride) NodeBuilder SetCreateNode(CreateNodeFunc) NodeBuilder SetGossipSubFactory(GossipSubFactoryFunc, GossipSubAdapterConfigFunc) NodeBuilder SetStreamCreationRetryInterval(time.Duration) NodeBuilder @@ -138,10 +139,31 @@ type NodeBuilder interface { Build() (LibP2PNode, error) } -// PeerScoringConfig is a configuration for peer scoring parameters for a GossipSub pubsub system. -type PeerScoringConfig struct { +// PeerScoringConfigOverride is a structure that is used to carry over the override values for peer scoring configuration. +// Any attribute that is set in the override will override the default peer scoring config. +// Typically, we are not recommending to override the default peer scoring config in production unless you know what you are doing. +type PeerScoringConfigOverride struct { // TopicScoreParams is a map of topic score parameters for each topic. + // Override criteria: any topic (i.e., key in the map) will override the default topic score parameters for that topic and + // the corresponding value in the map will be used instead of the default value. + // If you don't want to override topic score params for a given topic, simply don't include that topic in the map. + // If the map is nil, the default topic score parameters are used for all topics. TopicScoreParams map[channels.Topic]*pubsub.TopicScoreParams + // AppSpecificScoreParams is a function that returns the application specific score parameters for a given peer. + // Override criteria: if the function is not nil, it will override the default application specific score parameters. + // If the function is nil, the default application specific score parameters are used. AppSpecificScoreParams func(peer.ID) float64 + + // DecayInterval is the interval over which we decay the effect of past behavior, so that + // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval + // that GossipSub uses to refresh the scores of all peers. + // Override criteria: if the value is not zero, it will override the default decay interval. + // If the value is zero, the default decay interval is used. + DecayInterval time.Duration } + +// PeerScoringConfigNoOverride is a default peer scoring configuration for a GossipSub pubsub system. +// It is set to nil, which means that no override is done to the default peer scoring configuration. +// It is the recommended way to use the default peer scoring configuration. +var PeerScoringConfigNoOverride = (*PeerScoringConfigOverride)(nil) diff --git a/network/p2p/connection/connection_gater_test.go b/network/p2p/connection/connection_gater_test.go index 0277fc6b632..cd240b5293b 100644 --- a/network/p2p/connection/connection_gater_test.go +++ b/network/p2p/connection/connection_gater_test.go @@ -396,19 +396,20 @@ func TestConnectionGater_Disallow_Integration(t *testing.T) { func ensureCommunicationSilenceAmongGroups(t *testing.T, ctx context.Context, sporkId flow.Identifier, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode) { // ensures no connection, unicast, or pubsub going to the disallow-listed nodes p2ptest.EnsureNotConnectedBetweenGroups(t, ctx, groupA, groupB) - p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, groupA, groupB, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, groupA, groupB, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) p2pfixtures.EnsureNoStreamCreationBetweenGroups(t, ctx, groupA, groupB) } // ensureCommunicationOverAllProtocols ensures that all nodes are connected to each other, and they can exchange messages over the pubsub and unicast. func ensureCommunicationOverAllProtocols(t *testing.T, ctx context.Context, sporkId flow.Identifier, nodes []p2p.LibP2PNode, inbounds []chan string) { + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) p2pfixtures.EnsureMessageExchangeOverUnicast(t, ctx, nodes, inbounds, p2pfixtures.LongStringMessageFactoryFixture(t)) } diff --git a/network/p2p/connection/peerManager.go b/network/p2p/connection/peerManager.go index 211d0ee2f97..11fe502a07c 100644 --- a/network/p2p/connection/peerManager.go +++ b/network/p2p/connection/peerManager.go @@ -42,7 +42,7 @@ type PeerManager struct { // and it uses the connector to actually connect or disconnect from peers. func NewPeerManager(logger zerolog.Logger, updateInterval time.Duration, connector p2p.PeerUpdater) *PeerManager { pm := &PeerManager{ - logger: logger, + logger: logger.With().Str("component", "peer-manager").Logger(), connector: connector, peerRequestQ: make(chan struct{}, 1), peerUpdateInterval: updateInterval, diff --git a/network/p2p/mock/gossip_sub_builder.go b/network/p2p/mock/gossip_sub_builder.go index 2146f922c9b..08d82bd03c6 100644 --- a/network/p2p/mock/gossip_sub_builder.go +++ b/network/p2p/mock/gossip_sub_builder.go @@ -4,16 +4,12 @@ package mockp2p import ( host "github.com/libp2p/go-libp2p/core/host" - channels "github.com/onflow/flow-go/network/channels" - irrecoverable "github.com/onflow/flow-go/module/irrecoverable" mock "github.com/stretchr/testify/mock" p2p "github.com/onflow/flow-go/network/p2p" - peer "github.com/libp2p/go-libp2p/core/peer" - pubsub "github.com/libp2p/go-libp2p-pubsub" routing "github.com/libp2p/go-libp2p/core/routing" @@ -52,13 +48,13 @@ func (_m *GossipSubBuilder) Build(_a0 irrecoverable.SignalerContext) (p2p.PubSub return r0, r1 } -// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) { +// EnableGossipSubScoringWithOverride provides a mock function with given fields: _a0 +func (_m *GossipSubBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringConfigOverride) { _m.Called(_a0) } -// SetAppSpecificScoreParams provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetAppSpecificScoreParams(_a0 func(peer.ID) float64) { +// OverrideDefaultRpcInspectorSuiteFactory provides a mock function with given fields: _a0 +func (_m *GossipSubBuilder) OverrideDefaultRpcInspectorSuiteFactory(_a0 p2p.GossipSubRpcInspectorSuiteFactoryFunc) { _m.Called(_a0) } @@ -72,11 +68,6 @@ func (_m *GossipSubBuilder) SetGossipSubFactory(_a0 p2p.GossipSubFactoryFunc) { _m.Called(_a0) } -// SetGossipSubPeerScoring provides a mock function with given fields: _a0 -func (_m *GossipSubBuilder) SetGossipSubPeerScoring(_a0 bool) { - _m.Called(_a0) -} - // SetGossipSubScoreTracerInterval provides a mock function with given fields: _a0 func (_m *GossipSubBuilder) SetGossipSubScoreTracerInterval(_a0 time.Duration) { _m.Called(_a0) @@ -102,11 +93,6 @@ func (_m *GossipSubBuilder) SetSubscriptionFilter(_a0 pubsub.SubscriptionFilter) _m.Called(_a0) } -// SetTopicScoreParams provides a mock function with given fields: topic, topicScoreParams -func (_m *GossipSubBuilder) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { - _m.Called(topic, topicScoreParams) -} - type mockConstructorTestingTNewGossipSubBuilder interface { mock.TestingT Cleanup(func()) diff --git a/network/p2p/mock/node_builder.go b/network/p2p/mock/node_builder.go index 97ab398f37a..15bb6c10306 100644 --- a/network/p2p/mock/node_builder.go +++ b/network/p2p/mock/node_builder.go @@ -55,12 +55,12 @@ func (_m *NodeBuilder) Build() (p2p.LibP2PNode, error) { return r0, r1 } -// EnableGossipSubPeerScoring provides a mock function with given fields: _a0 -func (_m *NodeBuilder) EnableGossipSubPeerScoring(_a0 *p2p.PeerScoringConfig) p2p.NodeBuilder { +// EnableGossipSubScoringWithOverride provides a mock function with given fields: _a0 +func (_m *NodeBuilder) EnableGossipSubScoringWithOverride(_a0 *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { ret := _m.Called(_a0) var r0 p2p.NodeBuilder - if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfig) p2p.NodeBuilder); ok { + if rf, ok := ret.Get(0).(func(*p2p.PeerScoringConfigOverride) p2p.NodeBuilder); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { diff --git a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go index 89b1351691f..3e69590dc17 100644 --- a/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/p2pbuilder/gossipsub/gossipSubBuilder.go @@ -7,7 +7,6 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/rs/zerolog" @@ -17,7 +16,6 @@ import ( "github.com/onflow/flow-go/module/mempool/queue" "github.com/onflow/flow-go/module/metrics" "github.com/onflow/flow-go/network" - "github.com/onflow/flow-go/network/channels" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/distributor" "github.com/onflow/flow-go/network/p2p/inspector" @@ -29,6 +27,7 @@ import ( "github.com/onflow/flow-go/network/p2p/scoring" "github.com/onflow/flow-go/network/p2p/tracer" "github.com/onflow/flow-go/network/p2p/utils" + "github.com/onflow/flow-go/utils/logging" ) // The Builder struct is used to configure and create a new GossipSub pubsub system. @@ -92,14 +91,42 @@ func (g *Builder) SetGossipSubConfigFunc(gossipSubConfigFunc p2p.GossipSubAdapte g.gossipSubConfigFunc = gossipSubConfigFunc } -// SetGossipSubPeerScoring sets the gossipsub peer scoring of the builder. -// If the gossipsub peer scoring flag has already been set, a fatal error is logged. -func (g *Builder) SetGossipSubPeerScoring(gossipSubPeerScoring bool) { - if g.gossipSubPeerScoring { - g.logger.Fatal().Msg("gossipsub peer scoring has already been set") +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. +// Returns: +// none +func (g *Builder) EnableGossipSubScoringWithOverride(override *p2p.PeerScoringConfigOverride) { + g.gossipSubPeerScoring = true // TODO: we should enable peer scoring by default. + if override == nil { return } - g.gossipSubPeerScoring = gossipSubPeerScoring + if override.AppSpecificScoreParams != nil { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("overriding app specific score params for gossipsub") + g.scoreOptionConfig.OverrideAppSpecificScoreFunction(override.AppSpecificScoreParams) + } + if override.TopicScoreParams != nil { + for topic, params := range override.TopicScoreParams { + topicLogger := utils.TopicScoreParamsLogger(g.logger, topic.String(), params) + topicLogger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("overriding topic score params for gossipsub") + g.scoreOptionConfig.OverrideTopicScoreParams(topic, params) + } + } + if override.DecayInterval > 0 { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Dur("decay_interval", override.DecayInterval). + Msg("overriding decay interval for gossipsub") + g.scoreOptionConfig.OverrideDecayInterval(override.DecayInterval) + } } // SetGossipSubScoreTracerInterval sets the gossipsub score tracer interval of the builder. @@ -132,21 +159,6 @@ func (g *Builder) SetRoutingSystem(routingSystem routing.Routing) { g.routingSystem = routingSystem } -// SetTopicScoreParams sets the topic score params of the builder. -// There is a default topic score parameters that is used if this function is not called for a topic. -// However, if this function is called multiple times for a topic, the last topic score params will be used. -// Note: calling this function will override the default topic score params for the topic. Don't call this function -// unless you know what you are doing. -func (g *Builder) SetTopicScoreParams(topic channels.Topic, topicScoreParams *pubsub.TopicScoreParams) { - g.scoreOptionConfig.OverrideTopicScoreParams(topic, topicScoreParams) -} - -// SetAppSpecificScoreParams sets the app specific score params of the builder. -// There is no default app specific score function. However, if this function is called multiple times, the last function will be used. -func (g *Builder) SetAppSpecificScoreParams(f func(peer.ID) float64) { - g.scoreOptionConfig.SetAppSpecificScoreFunction(f) -} - // OverrideDefaultRpcInspectorSuiteFactory overrides the default rpc inspector suite factory. // Note: this function should only be used for testing purposes. Never override the default rpc inspector suite factory unless you know what you are doing. func (g *Builder) OverrideDefaultRpcInspectorSuiteFactory(factory p2p.GossipSubRpcInspectorSuiteFactoryFunc) { @@ -173,7 +185,11 @@ func NewGossipSubBuilder( idProvider module.IdentityProvider, rpcInspectorConfig *p2pconf.GossipSubRPCInspectorsConfig, ) *Builder { - lg := logger.With().Str("component", "gossipsub").Logger() + lg := logger.With(). + Str("component", "gossipsub"). + Str("network-type", networkType.String()). + Logger() + b := &Builder{ logger: lg, metricsCfg: metricsCfg, @@ -186,6 +202,7 @@ func NewGossipSubBuilder( rpcInspectorConfig: rpcInspectorConfig, rpcInspectorSuiteFactory: defaultInspectorSuite(), } + return b } @@ -310,6 +327,10 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e gossipSubConfigs.WithScoreTracer(scoreTracer) } + } else { + g.logger.Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("gossipsub peer scoring is disabled") } if g.gossipSubTracer != nil { diff --git a/network/p2p/p2pbuilder/libp2pNodeBuilder.go b/network/p2p/p2pbuilder/libp2pNodeBuilder.go index 8e550b4fa94..e27cbc5bedd 100644 --- a/network/p2p/p2pbuilder/libp2pNodeBuilder.go +++ b/network/p2p/p2pbuilder/libp2pNodeBuilder.go @@ -150,22 +150,17 @@ func (builder *LibP2PNodeBuilder) SetGossipSubFactory(gf p2p.GossipSubFactoryFun return builder } -// EnableGossipSubPeerScoring enables peer scoring for the GossipSub pubsub system. -// Arguments: -// - *PeerScoringConfig: the peer scoring configuration for the GossipSub pubsub system. If nil, the default configuration is used. -func (builder *LibP2PNodeBuilder) EnableGossipSubPeerScoring(config *p2p.PeerScoringConfig) p2p.NodeBuilder { - builder.gossipSubBuilder.SetGossipSubPeerScoring(true) - if config != nil { - if config.AppSpecificScoreParams != nil { - builder.gossipSubBuilder.SetAppSpecificScoreParams(config.AppSpecificScoreParams) - } - if config.TopicScoreParams != nil { - for topic, params := range config.TopicScoreParams { - builder.gossipSubBuilder.SetTopicScoreParams(topic, params) - } - } - } - +// EnableGossipSubScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Production Tip: use PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use PeerScoringConfigNoOverride for production. +// Returns: +// none +func (builder *LibP2PNodeBuilder) EnableGossipSubScoringWithOverride(config *p2p.PeerScoringConfigOverride) p2p.NodeBuilder { + builder.gossipSubBuilder.EnableGossipSubScoringWithOverride(config) return builder } @@ -491,8 +486,8 @@ func DefaultNodeBuilder( SetRateLimiterDistributor(uniCfg.RateLimiterDistributor) if gossipCfg.PeerScoring { - // currently, we only enable peer scoring with default parameters. So, we set the score parameters to nil. - builder.EnableGossipSubPeerScoring(nil) + // In production, we never override the default scoring config. + builder.EnableGossipSubScoringWithOverride(p2p.PeerScoringConfigNoOverride) } meshTracerCfg := &tracer.GossipSubMeshTracerConfig{ diff --git a/network/p2p/p2pnode/gossipSubAdapter.go b/network/p2p/p2pnode/gossipSubAdapter.go index 861093993cc..59bd2f2d65a 100644 --- a/network/p2p/p2pnode/gossipSubAdapter.go +++ b/network/p2p/p2pnode/gossipSubAdapter.go @@ -158,6 +158,7 @@ func (g *GossipSubAdapter) Join(topic string) (p2p.Topic, error) { topicParamsLogger.Info().Msg("joined topic with score params set") } else { g.logger.Warn(). + Bool(logging.KeyNetworkingSecurity, true). Str("topic", topic). Msg("joining topic without score params, this is not recommended from a security perspective") } diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md index 622ecadd3fe..a04e93fa741 100644 --- a/network/p2p/scoring/README.md +++ b/network/p2p/scoring/README.md @@ -93,6 +93,85 @@ scoreOption := NewScoreOption(config) 16. `defaultTopicInvalidMessageDeliveriesWeight` is set to -1.0 and is used to penalize peers that send invalid messages by applying it to the square of the number of such messages. A message is considered invalid if it is not properly signed. A peer will be disconnected if it sends around 14 invalid messages within a gossipsub heartbeat interval. 17. `defaultTopicInvalidMessageDeliveriesDecay` is a decay factor set to 0.99. It is used to reduce the number of invalid message deliveries counted against a peer by 1% at each heartbeat interval. This prevents the peer from being disconnected if it stops sending invalid messages. The heartbeat interval in the gossipsub scoring system is set to 1 minute by default. +## GossipSub Message Delivery Scoring +This section provides an overview of the GossipSub message delivery scoring mechanism used in the Flow network. +It's designed to maintain an efficient, secure and stable peer-to-peer network by scoring each peer based on their message delivery performance. +The system ensures the reliability of message propagation by scoring peers, which discourages malicious behaviors and enhances overall network performance. + +### Comprehensive System Overview +The GossipSub message delivery scoring mechanism used in the Flow network is an integral component of its P2P communication model. +It is designed to monitor and incentivize appropriate network behaviors by attributing scores to peers based on their message delivery performance. +This scoring system is fundamental to ensure that messages are reliably propagated across the network, creating a robust P2P communication infrastructure. + +The scoring system is per topic, which means it tracks the efficiency of peers in delivering messages in each specific topic they are participating in. +These per-topic scores then contribute to an overall score for each peer, providing a comprehensive view of a peer's effectiveness within the network. +In GossipSub, a crucial aspect of a peer's responsibility is to relay messages effectively to other nodes in the network. +The role of the scoring mechanism is to objectively assess a peer's efficiency in delivering these messages. +It takes into account several factors to determine the effectiveness of the peers. + +1. **Message Delivery Rate** - A peer's ability to deliver messages quickly is a vital metric. Slow delivery could lead to network lags and inefficiency. +2. **Message Delivery Volume** - A peer's capacity to deliver a large number of messages accurately and consistently. +3. **Continuity of Performance** - The scoring mechanism tracks not only the rate and volume of the messages but also the consistency in a peer's performance over time. +4. **Prevention of Malicious Behaviors** - The scoring system also helps in mitigating potential network attacks such as spamming and message replay attacks. + +The system utilizes several parameters to maintain and adjust the scores of the peers: +- `defaultTopicMeshMessageDeliveriesDecay`(value: 0.5): This parameter dictates how rapidly a peer's message delivery count decays with time. With a value of 0.5, it indicates a 50% decay at each decay interval. This mechanism ensures that past performances do not disproportionately impact the current score of the peer. +- `defaultTopicMeshMessageDeliveriesCap` (value: 1000): This parameter sets an upper limit on the number of message deliveries that can contribute to the score of a peer in a topic. With a cap set at 1000, it prevents the score from being overly influenced by large volumes of message deliveries, providing a balanced assessment of peer performance. +- `defaultTopicMeshMessageDeliveryThreshold` (value: 0.1 * `defaultTopicMeshMessageDeliveriesCap`): This threshold serves to identify under-performing peers. If a peer's message delivery count is below this threshold in a topic, the peer's score is penalized. This encourages peers to maintain a minimum level of performance. +- `defaultTopicMeshMessageDeliveriesWeight` (value: -0.05 * `MaxAppSpecificReward` / (`defaultTopicMeshMessageDeliveryThreshold` ^ 2) = 5^-4): This weight is applied when penalizing under-performing peers. The penalty is proportional to the square of the difference between the actual message deliveries and the threshold, multiplied by this weight. +- `defaultMeshMessageDeliveriesWindow` (value: `defaultDecayInterval` = 1 minute): This parameter defines the time window within which a message delivery is counted towards the score. This window is set to the decay interval, preventing replay attacks and counting only unique message deliveries. +- `defaultMeshMessageDeliveriesActivation` (value: 2 * `defaultDecayInterval` = 2 minutes): This time interval is the grace period before the scoring system starts tracking a new peer's performance. It accounts for the time it takes for a new peer to fully integrate into the network. + +By continually updating and adjusting the scores of peers based on these parameters, the GossipSub message delivery scoring mechanism ensures a robust, efficient, and secure P2P network. + +### Examples + +#### Scenario 1: Peer A Delivers Messages Within Cap and Above Threshold +Let's assume a Peer A that consistently delivers 500 messages per decay interval. This is within the `defaultTopicMeshMessageDeliveriesCap` (1000) and above the `defaultTopicMeshMessageDeliveryThreshold` (100). +As Peer A's deliveries are above the threshold and within the cap, its score will not be penalized. Instead, it will be maintained, promoting healthy network participation. + +#### Scenario 2: Peer B Delivers Messages Below Threshold +Now, assume Peer B delivers 50 messages per decay interval, below the `defaultTopicMeshMessageDeliveryThreshold` (100). +In this case, the score of Peer B will be penalized because its delivery rate is below the threshold. The penalty is calculated as `-|w| * (actual - threshold)^2`, where `w` is the weight (`defaultTopicMeshMessageDeliveriesWeight`), `actual` is the actual messages delivered (50), and `threshold` is the delivery threshold (100). + +#### Scenario 3: Peer C Delivers Messages Exceeding the Cap +Consider Peer C, which delivers 1500 messages per decay interval, exceeding the `defaultTopicMeshMessageDeliveriesCap` (1000). +In this case, even though Peer C is highly active, its score will not increase further once it hits the cap (1000). This is to avoid overemphasis on high delivery counts, which could skew the scoring system. + +#### Scenario 4: Peer D Joins a Topic Mesh +When a new Peer D joins a topic mesh, it will be given a grace period of `defaultMeshMessageDeliveriesActivation` (2 decay intervals) before its message delivery performance is tracked. This grace period allows the peer to set up and begin receiving messages from the network. +Remember, the parameters and scenarios described here aim to maintain a stable, efficient, and secure peer-to-peer network by carefully tracking and scoring each peer's message delivery performance. + +#### Scenario 5: Message Delivery Decay +To better understand how the message delivery decay (`defaultTopicMeshMessageDeliveriesDecay`) works in the GossipSub protocol, let's examine a hypothetical scenario. +Let's say we have a peer named `Peer A` who is actively participating in `Topic X`. `Peer A` has successfully delivered 800 messages in `Topic X` over a given time period. +**Initial State**: At this point, `Peer A`'s message delivery count for `Topic X` is 800. Now, the decay interval elapses without `Peer A` delivering any new messages in `Topic X`. +**After One Decay Interval**: Given that our `defaultTopicMeshMessageDeliveriesDecay` value is 0.5, after one decay interval, `Peer A`'s message delivery count for `Topic X` will decay by 50%. Therefore, `Peer A`'s count is now: + + 800 (previous message count) * 0.5 (decay factor) = 400 + +**After Two Decay Intervals** +If `Peer A` still hasn't delivered any new messages in `Topic X` during the next decay interval, the decay is applied again, further reducing the message delivery count: + + 400 (current message count) * 0.5 (decay factor) = 200 +And this process will continue at every decay interval, halving `Peer A`'s message delivery count for `Topic X` until `Peer A` delivers new messages in `Topic X` or the count reaches zero. +This decay process ensures that a peer cannot rest on its past deliveries; it must continually contribute to the network to maintain its score. +It helps maintain a lively and dynamic network environment, incentivizing constant active participation from all peers. + +### Scenario 6: Replay Attack +## Example Scenario: Preventing Replay Attacks +The `defaultMeshMessageDeliveriesWindow` and `defaultMeshMessageDeliveriesActivation` parameters play a crucial role in preventing replay attacks in the GossipSub protocol. Let's illustrate this with an example. +Consider a scenario where we have three peers: `Peer A`, `Peer B`, and `Peer C`. All three peers are active participants in `Topic X`. +**Initial State**: At Time = 0: `Peer A` generates and broadcasts a new message `M` in `Topic X`. `Peer B` and `Peer C` receive this message from `Peer A` and update their message caches accordingly. +**After Few Seconds**: At Time = 30 seconds: `Peer B`, with malicious intent, tries to rebroadcast the same message `M` back into `Topic X`. +Given that our `defaultMeshMessageDeliveriesWindow` value is equal to the decay interval (let's assume 1 minute), `Peer C` would have seen the original message `M` from `Peer A` less than one minute ago. +This is within the `defaultMeshMessageDeliveriesWindow`. Because `Peer A` (the original sender) is different from `Peer B` (the current sender), this delivery will be counted towards `Peer B`'s message delivery score in `Topic X`. +**After One Minute**: At Time = 61 seconds: `Peer B` tries to rebroadcast the same message `M` again. +Now, more than a minute has passed since `Peer C` first saw the message `M` from `Peer A`. This is outside the `defaultMeshMessageDeliveriesWindow`. +Therefore, the message `M` from `Peer B` will not count towards `Peer B`'s message delivery score in `Topic X` and `Peer B` still needs to fill up its threshold of message delivery in order not to be penalized for under-performing. +This effectively discouraging replay attacks of messages older than the `defaultMeshMessageDeliveriesWindow`. +This mechanism, combined with other parameters, helps maintain the security and efficiency of the network by discouraging harmful behaviors such as message replay attacks. + ## Customization The scoring mechanism can be easily customized to suit the needs of the Flow network. This includes changing the scoring parameters, thresholds, and the scoring function itself. You can customize the scoring parameters and thresholds by using the various setter methods provided in the `ScoreOptionConfig` object. Additionally, you can provide a custom app-specific scoring function through the `SetAppSpecificScoreFunction` method. diff --git a/network/p2p/scoring/app_score_test.go b/network/p2p/scoring/app_score_test.go index 50e0379116e..8e2a1ae1bb8 100644 --- a/network/p2p/scoring/app_score_test.go +++ b/network/p2p/scoring/app_score_test.go @@ -35,15 +35,15 @@ func TestFullGossipSubConnectivity(t *testing.T) { groupOneNodes, groupOneIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) groupTwoNodes, groupTwoIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, p2ptest.WithRole(flow.RoleCollection), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) accessNodeGroup, accessNodeIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 5, idProvider, p2ptest.WithRole(flow.RoleAccess), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) ids := append(append(groupOneIds, groupTwoIds...), accessNodeIds...) nodes := append(append(groupOneNodes, groupTwoNodes...), accessNodeGroup...) @@ -150,7 +150,7 @@ func testGossipSubMessageDeliveryUnderNetworkPartition(t *testing.T, honestPeerS // two (honest) consensus nodes opts := []p2ptest.NodeFixtureParameterOption{p2ptest.WithRole(flow.RoleConsensus)} if honestPeerScoring { - opts = append(opts, p2ptest.WithPeerScoringEnabled(idProvider)) + opts = append(opts, p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) } con1Node, con1Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, opts...) con2Node, con2Id := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, opts...) @@ -162,11 +162,11 @@ func testGossipSubMessageDeliveryUnderNetworkPartition(t *testing.T, honestPeerS accessNodeGroup, accessNodeIds := p2ptest.NodesFixture(t, sporkId, t.Name(), 30, idProvider, p2ptest.WithRole(flow.RoleAccess), - p2ptest.WithPeerScoringEnabled(idProvider), // overrides the default peer scoring parameters to mute GossipSub traffic from/to honest nodes. - p2ptest.WithPeerScoreParamsOption(&p2p.PeerScoringConfig{ + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: maliciousAppSpecificScore(flow.IdentityList{&con1Id, &con2Id}), - })) + }), + ) allNodes := append([]p2p.LibP2PNode{con1Node, con2Node}, accessNodeGroup...) allIds := append([]*flow.Identity{&con1Id, &con2Id}, accessNodeIds...) diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index c743b3efa33..cb7c5e3a4d4 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -1,6 +1,7 @@ package scoring import ( + "fmt" "time" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -138,10 +139,77 @@ const ( // are churners, i.e., peers that join and leave a topic mesh frequently. defaultTopicTimeInMesh = time.Hour - // defaultTopicWeight is the default weight of a topic in the GossipSub scoring system. The overall score of a peer in a topic mesh is - // multiplied by the weight of the topic when calculating the overall score of the peer. + // defaultTopicWeight is the default weight of a topic in the GossipSub scoring system. + // The overall score of a peer in a topic mesh is multiplied by the weight of the topic when calculating the overall score of the peer. // We set it to 1.0, which means that the overall score of a peer in a topic mesh is not affected by the weight of the topic. defaultTopicWeight = 1.0 + + // defaultTopicMeshMessageDeliveriesDecay is applied to the number of actual message deliveries in a topic mesh + // at each decay interval (i.e., defaultDecayInterval). + // It is used to decay the number of actual message deliveries, and prevents past message + // deliveries from affecting the current score of the peer. + // As the decay interval is 1 minute, we set it to 0.5, which means that the number of actual message + // deliveries will decay by 50% at each decay interval. + defaultTopicMeshMessageDeliveriesDecay = .5 + + // defaultTopicMeshMessageDeliveriesCap is the maximum number of actual message deliveries in a topic + // mesh that is used to calculate the score of a peer in that topic mesh. + // We set it to 1000, which means that the maximum number of actual message deliveries in a + // topic mesh that is used to calculate the score of a peer in that topic mesh is 1000. + // This is to prevent the score of a peer in a topic mesh from being affected by a large number of actual + // message deliveries and also affect the score of the peer in other topic meshes. + // When the total delivered messages in a topic mesh exceeds this value, the score of the peer in that topic + // mesh will not be affected by the actual message deliveries in that topic mesh. + // Moreover, this does not allow the peer to accumulate a large number of actual message deliveries in a topic mesh + // and then start under-performing in that topic mesh without being penalized. + defaultTopicMeshMessageDeliveriesCap = 1000 + + // defaultTopicMeshMessageDeliveriesThreshold is the threshold for the number of actual message deliveries in a + // topic mesh that is used to calculate the score of a peer in that topic mesh. + // If the number of actual message deliveries in a topic mesh is less than this value, + // the peer will be penalized by square of the difference between the actual message deliveries and the threshold, + // i.e., -w * (actual - threshold)^2 where `actual` and `threshold` are the actual message deliveries and the + // threshold, respectively, and `w` is the weight (i.e., defaultTopicMeshMessageDeliveriesWeight). + // We set it to 0.1 * defaultTopicMeshMessageDeliveriesCap, which means that if a peer delivers less tha 10% of the + // maximum number of actual message deliveries in a topic mesh, it will be considered as an under-performing peer + // in that topic mesh. + defaultTopicMeshMessageDeliveryThreshold = 0.1 * defaultTopicMeshMessageDeliveriesCap + + // defaultTopicMeshDeliveriesWeight is the weight for applying penalty when a peer is under-performing in a topic mesh. + // Upon every decay interval, if the number of actual message deliveries is less than the topic mesh message deliveries threshold + // (i.e., defaultTopicMeshMessageDeliveriesThreshold), the peer will be penalized by square of the difference between the actual + // message deliveries and the threshold, multiplied by this weight, i.e., -w * (actual - threshold)^2 where w is the weight, and + // `actual` and `threshold` are the actual message deliveries and the threshold, respectively. + // We set this value to be - 0.05 MaxAppSpecificReward / (defaultTopicMeshMessageDeliveriesThreshold^2). This guarantees that even if a peer + // is not delivering any message in a topic mesh, it will not be disconnected. + // Rather, looses part of the MaxAppSpecificReward that is awarded by our app-specific scoring function to all staked + // nodes by default will be withdrawn, and the peer will be slightly penalized. In other words, under-performing in a topic mesh + // will drop the overall score of a peer by 5% of the MaxAppSpecificReward that is awarded by our app-specific scoring function. + // It means that under-performing in a topic mesh will not cause a peer to be disconnected, but it will cause the peer to lose + // its MaxAppSpecificReward that is awarded by our app-specific scoring function. + // At this point, we do not want to disconnect a peer only because it is under-performing in a topic mesh as it might be + // causing a false positive network partition. + // TODO: we must increase the penalty for under-performing in a topic mesh in the future, and disconnect the peer if it is under-performing. + defaultTopicMeshMessageDeliveriesWeight = -0.05 * MaxAppSpecificReward / (defaultTopicMeshMessageDeliveryThreshold * defaultTopicMeshMessageDeliveryThreshold) + + // defaultMeshMessageDeliveriesWindow is the window size is time interval that we count a delivery of an already + // seen message towards the score of a peer in a topic mesh. The delivery is counted + // by GossipSub only if the previous sender of the message is different from the current sender. + // We set it to the decay interval of the GossipSub scoring system, which is 1 minute. + // It means that if a peer delivers a message that it has already seen less than one minute ago, + // the delivery will be counted towards the score of the peer in a topic mesh only if the previous sender of the message. + // This also prevents replay attacks of messages that are older than one minute. As replayed messages will not + // be counted towards the actual message deliveries of a peer in a topic mesh. + defaultMeshMessageDeliveriesWindow = defaultDecayInterval + + // defaultMeshMessageDeliveryActivation is the time interval that we wait for a new peer that joins a topic mesh + // till start counting the number of actual message deliveries of that peer in that topic mesh. + // We set it to 2 * defaultDecayInterval, which means that we wait for 2 decay intervals before start counting + // the number of actual message deliveries of a peer in a topic mesh. + // With a default decay interval of 1 minute, it means that we wait for 2 minutes before start counting the + // number of actual message deliveries of a peer in a topic mesh. This is to account for + // the time that it takes for a peer to start up and receive messages from other peers in the topic mesh. + defaultMeshMessageDeliveriesActivation = 2 * defaultDecayInterval ) // ScoreOption is a functional option for configuring the peer scoring system. @@ -160,6 +228,7 @@ type ScoreOptionConfig struct { cacheSize uint32 cacheMetrics module.HeroCacheMetrics appScoreFunc func(peer.ID) float64 + decayInterval time.Duration // the decay interval, when is set to 0, the default value will be used. topicParams []func(map[string]*pubsub.TopicScoreParams) registerNotificationConsumerFunc func(p2p.GossipSubInvCtrlMsgNotifConsumer) } @@ -189,12 +258,12 @@ func (c *ScoreOptionConfig) SetCacheMetrics(metrics module.HeroCacheMetrics) { c.cacheMetrics = metrics } -// SetAppSpecificScoreFunction sets the app specific penalty function for the penalty option. +// OverrideAppSpecificScoreFunction sets the app specific penalty function for the penalty option. // It is used to calculate the app specific penalty of a peer. // If the app specific penalty function is not set, the default one is used. // Note that it is always safer to use the default one, unless you know what you are doing. // It is safe to call this method multiple times, the last call will be used. -func (c *ScoreOptionConfig) SetAppSpecificScoreFunction(appSpecificScoreFunction func(peer.ID) float64) { +func (c *ScoreOptionConfig) OverrideAppSpecificScoreFunction(appSpecificScoreFunction func(peer.ID) float64) { c.appScoreFunc = appSpecificScoreFunction } @@ -214,6 +283,21 @@ func (c *ScoreOptionConfig) SetRegisterNotificationConsumerFunc(f func(p2p.Gossi c.registerNotificationConsumerFunc = f } +// OverrideDecayInterval overrides the decay interval for the penalty option. It is used to override the default +// decay interval for the penalty option. The decay interval is the time interval that the decay values are applied and +// peer scores are updated. +// Note: It is always recommended to use the default value unless you know what you are doing. Hence, calling this method +// is not recommended in production. +// Args: +// +// interval: the decay interval. +// +// Returns: +// none +func (c *ScoreOptionConfig) OverrideDecayInterval(interval time.Duration) { + c.decayInterval = interval +} + // NewScoreOption creates a new penalty option with the given configuration. func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { throttledSampler := logging.BurstSampler(MaxDebugLogs, time.Second) @@ -239,14 +323,27 @@ func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { logger: logger, validator: validator, peerScoreParams: defaultPeerScoreParams(), + appScoreFunc: scoreRegistry.AppSpecificScoreFunc(), } // set the app specific penalty function for the penalty option // if the app specific penalty function is not set, use the default one - if cfg.appScoreFunc == nil { - s.appScoreFunc = scoreRegistry.AppSpecificScoreFunc() - } else { + if cfg.appScoreFunc != nil { s.appScoreFunc = cfg.appScoreFunc + s.logger. + Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Msg("app specific score function is overridden") + } + + if cfg.decayInterval > 0 { + // overrides the default decay interval if the decay interval is set. + s.peerScoreParams.DecayInterval = cfg.decayInterval + s.logger. + Warn(). + Str(logging.KeyNetworkingSecurity, "true"). + Dur("decay_interval_ms", cfg.decayInterval). + Msg("decay interval is overridden") } // registers the score registry as the consumer of the invalid control message notifications @@ -308,7 +405,7 @@ func (s *ScoreOption) preparePeerScoreThresholds() { func (s *ScoreOption) TopicScoreParams(topic *pubsub.Topic) *pubsub.TopicScoreParams { params, exists := s.peerScoreParams.Topics[topic.String()] if !exists { - return defaultTopicScoreParams() + return DefaultTopicScoreParams() } return params } @@ -320,7 +417,8 @@ func defaultPeerScoreParams() *pubsub.PeerScoreParams { // atomic validation fails initialization if any parameter is not set. SkipAtomicValidation: true, // DecayInterval is the interval over which we decay the effect of past behavior. So that - // a good or bad behavior will not have a permanent effect on the penalty. + // a good or bad behavior will not have a permanent effect on the penalty. It is also interval + // that GossipSub refreshes the scores of all peers. DecayInterval: defaultDecayInterval, // DecayToZero defines the maximum value below which a peer scoring counter is reset to zero. // This is to prevent the counter from decaying to a very small value. @@ -332,13 +430,26 @@ func defaultPeerScoreParams() *pubsub.PeerScoreParams { } } -// defaultTopicScoreParams returns the default score params for topics. -func defaultTopicScoreParams() *pubsub.TopicScoreParams { - return &pubsub.TopicScoreParams{ - TopicWeight: defaultTopicWeight, - SkipAtomicValidation: defaultTopicSkipAtomicValidation, - InvalidMessageDeliveriesWeight: defaultTopicInvalidMessageDeliveriesWeight, - InvalidMessageDeliveriesDecay: defaultTopicInvalidMessageDeliveriesDecay, - TimeInMeshQuantum: defaultTopicTimeInMesh, +// DefaultTopicScoreParams returns the default score params for topics. +func DefaultTopicScoreParams() *pubsub.TopicScoreParams { + p := &pubsub.TopicScoreParams{ + TopicWeight: defaultTopicWeight, + SkipAtomicValidation: defaultTopicSkipAtomicValidation, + InvalidMessageDeliveriesWeight: defaultTopicInvalidMessageDeliveriesWeight, + InvalidMessageDeliveriesDecay: defaultTopicInvalidMessageDeliveriesDecay, + TimeInMeshQuantum: defaultTopicTimeInMesh, + MeshMessageDeliveriesWeight: defaultTopicMeshMessageDeliveriesWeight, + MeshMessageDeliveriesDecay: defaultTopicMeshMessageDeliveriesDecay, + MeshMessageDeliveriesCap: defaultTopicMeshMessageDeliveriesCap, + MeshMessageDeliveriesThreshold: defaultTopicMeshMessageDeliveryThreshold, + MeshMessageDeliveriesWindow: defaultMeshMessageDeliveriesWindow, + MeshMessageDeliveriesActivation: defaultMeshMessageDeliveriesActivation, } + + if p.MeshMessageDeliveriesWeight >= 0 { + // GossipSub also does a validation, but we want to panic as early as possible. + panic(fmt.Sprintf("invalid mesh message deliveries weight %f", p.MeshMessageDeliveriesWeight)) + } + + return p } diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index db43c59a055..47e6f27cb57 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -92,7 +92,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.OverrideGossipSubRpcInspectorSuiteFactory(func(zerolog.Logger, flow.Identifier, *p2pconf.GossipSubRPCInspectorsConfig, @@ -110,7 +110,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { t.Name(), idProvider, p2ptest.WithRole(flow.RoleConsensus), - p2ptest.WithPeerScoringEnabled(idProvider)) + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride)) ids := flow.IdentityList{&id1, &id2} nodes := []p2p.LibP2PNode{node1, node2} @@ -128,11 +128,10 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) - + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) // checks end-to-end message delivery works on GossipSub - p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) // now simulates node2 spamming node1 with invalid gossipsub control messages. @@ -146,8 +145,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { } // checks no GossipSub message exchange should no longer happen between node1 and node2. - p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}, func() (interface{}, channels.Topic) { - blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) - return unittest.ProposalFixture(), blockTopic + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{node1}, []p2p.LibP2PNode{node2}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() }) } diff --git a/network/p2p/scoring/subscription_validator_test.go b/network/p2p/scoring/subscription_validator_test.go index 3bba66c6199..549006b3bde 100644 --- a/network/p2p/scoring/subscription_validator_test.go +++ b/network/p2p/scoring/subscription_validator_test.go @@ -178,20 +178,20 @@ func TestSubscriptionValidator_Integration(t *testing.T) { conNode, conId := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleConsensus)) // two verification node. verNode1, verId1 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) verNode2, verId2 := p2ptest.NodeFixture(t, sporkId, t.Name(), idProvider, p2ptest.WithLogger(unittest.Logger()), - p2ptest.WithPeerScoringEnabled(idProvider), + p2ptest.EnablePeerScoringWithOverride(p2p.PeerScoringConfigNoOverride), p2ptest.WithRole(flow.RoleVerification)) ids := flow.IdentityList{&conId, &verId1, &verId2} diff --git a/network/p2p/test/fixtures.go b/network/p2p/test/fixtures.go index 5ee82dd4ab5..50ecb5a568f 100644 --- a/network/p2p/test/fixtures.go +++ b/network/p2p/test/fixtures.go @@ -5,6 +5,7 @@ import ( "context" "crypto/rand" crand "math/rand" + "sync" "testing" "time" @@ -145,7 +146,7 @@ func NodeFixture( } if parameters.PeerScoringEnabled { - builder.EnableGossipSubPeerScoring(parameters.PeerScoreConfig) + builder.EnableGossipSubScoringWithOverride(parameters.PeerScoringConfigOverride) } if parameters.GossipSubFactory != nil && parameters.GossipSubConfig != nil { @@ -199,7 +200,7 @@ type NodeFixtureParameters struct { Logger zerolog.Logger PeerScoringEnabled bool IdProvider module.IdentityProvider - PeerScoreConfig *p2p.PeerScoringConfig + PeerScoringConfigOverride *p2p.PeerScoringConfigOverride PeerManagerConfig *p2pconfig.PeerManagerConfig PeerProvider p2p.PeersProvider // peer manager parameter ConnGater p2p.ConnectionGater @@ -240,10 +241,21 @@ func WithCreateStreamRetryDelay(delay time.Duration) NodeFixtureParameterOption } } -func WithPeerScoringEnabled(idProvider module.IdentityProvider) NodeFixtureParameterOption { +// EnablePeerScoringWithOverride enables peer scoring for the GossipSub pubsub system with the given override. +// Any existing peer scoring config attribute that is set in the override will override the default peer scoring config. +// Anything that is left to nil or zero value in the override will be ignored and the default value will be used. +// Note: it is not recommended to override the default peer scoring config in production unless you know what you are doing. +// Default Use Tip: use p2p.PeerScoringConfigNoOverride as the argument to this function to enable peer scoring without any override. +// Args: +// - PeerScoringConfigOverride: override for the peer scoring config- Recommended to use p2p.PeerScoringConfigNoOverride for production or when +// you don't want to override the default peer scoring config. +// +// Returns: +// - NodeFixtureParameterOption: a function that can be passed to the NodeFixture function to enable peer scoring. +func EnablePeerScoringWithOverride(override *p2p.PeerScoringConfigOverride) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { p.PeerScoringEnabled = true - p.IdProvider = idProvider + p.PeerScoringConfigOverride = override } } @@ -308,9 +320,9 @@ func WithRole(role flow.Role) NodeFixtureParameterOption { } } -func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfig) NodeFixtureParameterOption { +func WithPeerScoreParamsOption(cfg *p2p.PeerScoringConfigOverride) NodeFixtureParameterOption { return func(p *NodeFixtureParameters) { - p.PeerScoreConfig = cfg + p.PeerScoringConfigOverride = cfg } } @@ -549,17 +561,20 @@ func EnsureStreamCreationInBothDirections(t *testing.T, ctx context.Context, nod } // EnsurePubsubMessageExchange ensures that the given connected nodes exchange the given message on the given channel through pubsub. -// Note: TryConnectionAndEnsureConnected() must be called to connect all nodes before calling this function. -func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { - _, topic := messageFactory() - +// Args: +// - nodes: the nodes to exchange messages +// - ctx: the context- the test will fail if the context expires. +// - topic: the topic to exchange messages on +// - count: the number of messages to exchange from each node. +// - messageFactory: a function that creates a unique message to be published by the node. +// The function should return a different message each time it is called. +// +// Note-1: this function assumes a timeout of 5 seconds for each message to be received. +// Note-2: TryConnectionAndEnsureConnected() must be called to connect all nodes before calling this function. +func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { subs := make([]p2p.Subscription, len(nodes)) for i, node := range nodes { - ps, err := node.Subscribe( - topic, - validator.TopicValidator( - unittest.Logger(), - unittest.AllowAllPeerFilter())) + ps, err := node.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) require.NoError(t, err) subs[i] = ps } @@ -571,14 +586,52 @@ func EnsurePubsubMessageExchange(t *testing.T, ctx context.Context, nodes []p2p. require.True(t, ok) for _, node := range nodes { + for i := 0; i < count; i++ { + // creates a unique message to be published by the node + msg := messageFactory() + data := p2pfixtures.MustEncodeEvent(t, msg, channel) + require.NoError(t, node.Publish(ctx, topic, data)) + + // wait for the message to be received by all nodes + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + p2pfixtures.SubsMustReceiveMessage(t, ctx, data, subs) + cancel() + } + } +} + +// EnsurePubsubMessageExchangeFromNode ensures that the given node exchanges the given message on the given channel through pubsub with the other nodes. +// Args: +// - node: the node to exchange messages +// +// - ctx: the context- the test will fail if the context expires. +// - sender: the node that sends the message to the other node. +// - receiver: the node that receives the message from the other node. +// - topic: the topic to exchange messages on. +// - count: the number of messages to exchange from `sender` to `receiver`. +// - messageFactory: a function that creates a unique message to be published by the node. +func EnsurePubsubMessageExchangeFromNode(t *testing.T, ctx context.Context, sender p2p.LibP2PNode, receiver p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { + _, err := sender.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + + toSub, err := receiver.Subscribe(topic, validator.TopicValidator(unittest.Logger(), unittest.AllowAllPeerFilter())) + require.NoError(t, err) + + // let subscriptions propagate + time.Sleep(1 * time.Second) + + channel, ok := channels.ChannelFromTopic(topic) + require.True(t, ok) + + for i := 0; i < count; i++ { // creates a unique message to be published by the node - msg, _ := messageFactory() + msg := messageFactory() data := p2pfixtures.MustEncodeEvent(t, msg, channel) - require.NoError(t, node.Publish(ctx, topic, data)) + require.NoError(t, sender.Publish(ctx, topic, data)) // wait for the message to be received by all nodes ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - p2pfixtures.SubsMustReceiveMessage(t, ctx, data, subs) + p2pfixtures.SubsMustReceiveMessage(t, ctx, data, []p2p.Subscription{toSub}) cancel() } } @@ -605,9 +658,14 @@ func EnsureNotConnectedBetweenGroups(t *testing.T, ctx context.Context, groupA [ } // EnsureNoPubsubMessageExchange ensures that the no pubsub message is exchanged "from" the given nodes "to" the given nodes. -func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p.LibP2PNode, to []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { - _, topic := messageFactory() - +// Args: +// - from: the nodes that send messages to the other group but their message must not be received by the other group. +// +// - to: the nodes that are the target of the messages sent by the other group ("from") but must not receive any message from them. +// - topic: the topic to exchange messages on. +// - count: the number of messages to exchange from each node. +// - messageFactory: a function that creates a unique message to be published by the node. +func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p.LibP2PNode, to []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { subs := make([]p2p.Subscription, len(to)) tv := validator.TopicValidator( unittest.Logger(), @@ -627,27 +685,47 @@ func EnsureNoPubsubMessageExchange(t *testing.T, ctx context.Context, from []p2p // let subscriptions propagate time.Sleep(1 * time.Second) + wg := &sync.WaitGroup{} for _, node := range from { - // creates a unique message to be published by the node. - msg, _ := messageFactory() - channel, ok := channels.ChannelFromTopic(topic) - require.True(t, ok) - data := p2pfixtures.MustEncodeEvent(t, msg, channel) - - // ensure the message is NOT received by any of the nodes. - require.NoError(t, node.Publish(ctx, topic, data)) - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - p2pfixtures.SubsMustNeverReceiveAnyMessage(t, ctx, subs) - cancel() + node := node // capture range variable + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + // creates a unique message to be published by the node. + msg := messageFactory() + channel, ok := channels.ChannelFromTopic(topic) + require.True(t, ok) + data := p2pfixtures.MustEncodeEvent(t, msg, channel) + + // ensure the message is NOT received by any of the nodes. + require.NoError(t, node.Publish(ctx, topic, data)) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + p2pfixtures.SubsMustNeverReceiveAnyMessage(t, ctx, subs) + cancel() + wg.Done() + }() + } } + + // we wait for 5 seconds at most for the messages to be exchanged, hence we wait for a total of 6 seconds here to ensure + // that the goroutines are done in a timely manner. + unittest.RequireReturnsBefore(t, wg.Wait, 6*time.Second, "timed out waiting for messages to be exchanged") } // EnsureNoPubsubExchangeBetweenGroups ensures that no pubsub message is exchanged between the given groups of nodes. -func EnsureNoPubsubExchangeBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode, messageFactory func() (interface{}, channels.Topic)) { +// Args: +// - t: *testing.T instance +// - ctx: context.Context instance +// - groupA: first group of nodes- no message should be exchanged from any node of this group to the other group. +// - groupB: second group of nodes- no message should be exchanged from any node of this group to the other group. +// - topic: pubsub topic- no message should be exchanged on this topic. +// - count: number of messages to be exchanged- no message should be exchanged. +// - messageFactory: function to create a unique message to be published by the node. +func EnsureNoPubsubExchangeBetweenGroups(t *testing.T, ctx context.Context, groupA []p2p.LibP2PNode, groupB []p2p.LibP2PNode, topic channels.Topic, count int, messageFactory func() interface{}) { // ensure no message exchange from group A to group B - EnsureNoPubsubMessageExchange(t, ctx, groupA, groupB, messageFactory) + EnsureNoPubsubMessageExchange(t, ctx, groupA, groupB, topic, count, messageFactory) // ensure no message exchange from group B to group A - EnsureNoPubsubMessageExchange(t, ctx, groupB, groupA, messageFactory) + EnsureNoPubsubMessageExchange(t, ctx, groupB, groupA, topic, count, messageFactory) } // PeerIdSliceFixture returns a slice of random peer IDs for testing. diff --git a/network/p2p/tracer/gossipSubScoreTracer_test.go b/network/p2p/tracer/gossipSubScoreTracer_test.go index a759cc2b46f..2a3ea623eb0 100644 --- a/network/p2p/tracer/gossipSubScoreTracer_test.go +++ b/network/p2p/tracer/gossipSubScoreTracer_test.go @@ -83,9 +83,7 @@ func TestGossipSubScoreTracer(t *testing.T) { }), p2ptest.WithLogger(logger), p2ptest.WithPeerScoreTracerInterval(1*time.Second), // set the peer score log interval to 1 second for sake of testing. - p2ptest.WithPeerScoringEnabled(idProvider), // enable peer scoring for sake of testing. - // 4. Sets some fixed scores for the nodes for the sake of testing based on their roles. - p2ptest.WithPeerScoreParamsOption(&p2p.PeerScoringConfig{ + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ AppSpecificScoreParams: func(pid peer.ID) float64 { id, ok := idProvider.ByPeerID(pid) require.True(t, ok) diff --git a/utils/logging/consts.go b/utils/logging/consts.go index 31cfef3078a..46f48a3c937 100644 --- a/utils/logging/consts.go +++ b/utils/logging/consts.go @@ -1,5 +1,11 @@ package logging -// KeySuspicious is a logging label that is used to flag the log event as suspicious behavior -// This is used to add an easily searchable label to the log event -const KeySuspicious = "suspicious" +const ( + // KeySuspicious is a logging label that is used to flag the log event as suspicious behavior + // This is used to add an easily searchable label to the log event + KeySuspicious = "suspicious" + + // KeyNetworkingSecurity is a logging label that is used to flag the log event as a networking security issue. + // This is used to add an easily searchable label to the log events. + KeyNetworkingSecurity = "networking-security" +) From 2249991a168bd9fb70438f7f555f75b98b67cfeb Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Tue, 18 Jul 2023 17:04:33 -0700 Subject: [PATCH 155/169] [Access] Update REST metrics to use route name for all types --- .../node_builder/access_node_builder.go | 7 ++ engine/access/rest/middleware/metrics.go | 11 +- engine/access/rest/router.go | 2 +- module/metrics/access.go | 9 +- module/metrics/labels.go | 4 + module/metrics/namespaces.go | 2 + module/metrics/rest_api.go | 107 +++++++++--------- 7 files changed, 75 insertions(+), 67 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 5edd2629ee2..6138e70560b 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -37,6 +37,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" + "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/access/state_stream" @@ -216,6 +217,7 @@ type FlowAccessNodeBuilder struct { CollectionsToMarkExecuted *stdmap.Times BlocksToMarkExecuted *stdmap.Times TransactionMetrics *metrics.TransactionCollector + RESTMetrics *metrics.RestCollector AccessMetrics module.AccessMetrics PingMetrics module.PingMetrics Committee hotstuff.DynamicCommittee @@ -964,10 +966,15 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { ) return nil }). + Module("rest metrics", func(node *cmd.NodeConfig) error { + builder.RESTMetrics = metrics.NewRestCollector(rest.URLToRoute, node.MetricsRegisterer) + return nil + }). Module("access metrics", func(node *cmd.NodeConfig) error { builder.AccessMetrics = metrics.NewAccessCollector( metrics.WithTransactionMetrics(builder.TransactionMetrics), metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), + metrics.WithRestMetrics(builder.RESTMetrics), ) return nil }). diff --git a/engine/access/rest/middleware/metrics.go b/engine/access/rest/middleware/metrics.go index 25f82bf4277..54dd5dd2c6a 100644 --- a/engine/access/rest/middleware/metrics.go +++ b/engine/access/rest/middleware/metrics.go @@ -11,19 +11,12 @@ import ( "github.com/onflow/flow-go/module" ) -func MetricsMiddleware(restCollector module.RestMetrics, urlToRoute func(string) (string, error)) mux.MiddlewareFunc { +func MetricsMiddleware(restCollector module.RestMetrics) mux.MiddlewareFunc { metricsMiddleware := middleware.New(middleware.Config{Recorder: restCollector}) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - //urlToRoute transforms specific URL to generic url pattern - routeName, err := urlToRoute(req.URL.Path) - if err != nil { - // In case of an error, an empty route name filled with "unknown" - routeName = "unknown" - } - // This is a custom metric being called on every http request - restCollector.AddTotalRequests(req.Context(), req.Method, routeName) + restCollector.AddTotalRequests(req.Context(), req.Method, req.URL.Path) // Modify the writer respWriter := &responseWriter{w, http.StatusOK} diff --git a/engine/access/rest/router.go b/engine/access/rest/router.go index f51f1c65f3e..a6bfaa501c5 100644 --- a/engine/access/rest/router.go +++ b/engine/access/rest/router.go @@ -24,7 +24,7 @@ func newRouter(backend access.API, logger zerolog.Logger, chain flow.Chain, rest v1SubRouter.Use(middleware.LoggingMiddleware(logger)) v1SubRouter.Use(middleware.QueryExpandable()) v1SubRouter.Use(middleware.QuerySelect()) - v1SubRouter.Use(middleware.MetricsMiddleware(restCollector, URLToRoute)) + v1SubRouter.Use(middleware.MetricsMiddleware(restCollector)) linkGenerator := models.NewLinkGeneratorImpl(v1SubRouter) diff --git a/module/metrics/access.go b/module/metrics/access.go index e1021c93a42..1116f87f433 100644 --- a/module/metrics/access.go +++ b/module/metrics/access.go @@ -3,7 +3,6 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" - metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/counters" @@ -23,6 +22,12 @@ func WithBackendScriptsMetrics(m module.BackendScriptsMetrics) AccessCollectorOp } } +func WithRestMetrics(m module.RestMetrics) AccessCollectorOpts { + return func(ac *AccessCollector) { + ac.RestMetrics = m + } +} + type AccessCollector struct { module.RestMetrics module.TransactionMetrics @@ -101,8 +106,6 @@ func NewAccessCollector(opts ...AccessCollectorOpts) *AccessCollector { Help: "gauge to track the maximum block height of execution receipts received", }), maxReceiptHeightValue: counters.NewMonotonousCounter(0), - - RestMetrics: NewRestCollector(metricsProm.Config{Prefix: "access_rest_api"}), } for _, opt := range opts { diff --git a/module/metrics/labels.go b/module/metrics/labels.go index 9febc9ab391..4efb72b1152 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -20,6 +20,10 @@ const ( LabelSuccess = "success" LabelCtrlMsgType = "control_message" LabelMisbehavior = "misbehavior" + LabelHandler = "handler" + LabelStatusCode = "code" + LabelMethod = "method" + LabelService = "service" ) const ( diff --git a/module/metrics/namespaces.go b/module/metrics/namespaces.go index 6fd0f2db82f..f89f2a530ae 100644 --- a/module/metrics/namespaces.go +++ b/module/metrics/namespaces.go @@ -15,6 +15,7 @@ const ( namespaceExecutionDataSync = "execution_data_sync" namespaceChainsync = "chainsync" namespaceFollowerEngine = "follower" + namespaceRestAPI = "access_rest_api" ) // Network subsystems represent the various layers of networking. @@ -43,6 +44,7 @@ const ( subsystemTransactionTiming = "transaction_timing" subsystemTransactionSubmission = "transaction_submission" subsystemConnectionPool = "connection_pool" + subsystemHTTP = "http" ) // Observer subsystem diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go index 0ab08d0a3ca..aa7978dbe56 100644 --- a/module/metrics/rest_api.go +++ b/module/metrics/rest_api.go @@ -5,11 +5,9 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + httpmetrics "github.com/slok/go-http-metrics/metrics" "github.com/onflow/flow-go/module" - - httpmetrics "github.com/slok/go-http-metrics/metrics" - metricsProm "github.com/slok/go-http-metrics/metrics/prometheus" ) type RestCollector struct { @@ -17,74 +15,50 @@ type RestCollector struct { httpResponseSizeHistogram *prometheus.HistogramVec httpRequestsInflight *prometheus.GaugeVec httpRequestsTotal *prometheus.GaugeVec + + // urlToRouteMapper is a callback that converts a URL to a route name + urlToRouteMapper func(string) (string, error) } var _ module.RestMetrics = (*RestCollector)(nil) // NewRestCollector returns a new metrics RestCollector that implements the RestCollector // using Prometheus as the backend. -func NewRestCollector(cfg metricsProm.Config) module.RestMetrics { - if len(cfg.DurationBuckets) == 0 { - cfg.DurationBuckets = prometheus.DefBuckets - } - - if len(cfg.SizeBuckets) == 0 { - cfg.SizeBuckets = prometheus.ExponentialBuckets(100, 10, 8) - } - - if cfg.Registry == nil { - cfg.Registry = prometheus.DefaultRegisterer - } - - if cfg.HandlerIDLabel == "" { - cfg.HandlerIDLabel = "handler" - } - - if cfg.StatusCodeLabel == "" { - cfg.StatusCodeLabel = "code" - } - - if cfg.MethodLabel == "" { - cfg.MethodLabel = "method" - } - - if cfg.ServiceLabel == "" { - cfg.ServiceLabel = "service" - } - +func NewRestCollector(urlToRouteMapper func(string) (string, error), registerer prometheus.Registerer) *RestCollector { r := &RestCollector{ + urlToRouteMapper: urlToRouteMapper, httpRequestDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "request_duration_seconds", Help: "The latency of the HTTP requests.", - Buckets: cfg.DurationBuckets, - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + Buckets: prometheus.DefBuckets, + }, []string{LabelService, LabelHandler, LabelMethod, LabelStatusCode}), httpResponseSizeHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "response_size_bytes", Help: "The size of the HTTP responses.", - Buckets: cfg.SizeBuckets, - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel, cfg.MethodLabel, cfg.StatusCodeLabel}), + Buckets: prometheus.ExponentialBuckets(100, 10, 8), + }, []string{LabelService, LabelHandler, LabelMethod, LabelStatusCode}), httpRequestsInflight: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "requests_inflight", Help: "The number of inflight requests being handled at the same time.", - }, []string{cfg.ServiceLabel, cfg.HandlerIDLabel}), + }, []string{LabelService, LabelHandler}), httpRequestsTotal: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: cfg.Prefix, - Subsystem: "http", + Namespace: namespaceRestAPI, + Subsystem: subsystemHTTP, Name: "requests_total", Help: "The number of requests handled over time.", - }, []string{cfg.MethodLabel, cfg.HandlerIDLabel}), + }, []string{LabelMethod, LabelHandler}), } - cfg.Registry.MustRegister( + registerer.MustRegister( r.httpRequestDurHistogram, r.httpResponseSizeHistogram, r.httpRequestsInflight, @@ -94,20 +68,45 @@ func NewRestCollector(cfg metricsProm.Config) module.RestMetrics { return r } -// These methods are called automatically by go-http-metrics/middleware +// ObserveHTTPRequestDuration records the duration of the REST request. +// This method is called automatically by go-http-metrics/middleware func (r *RestCollector) ObserveHTTPRequestDuration(_ context.Context, p httpmetrics.HTTPReqProperties, duration time.Duration) { - r.httpRequestDurHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(duration.Seconds()) + handler := r.mapURLToRoute(p.ID) + r.httpRequestDurHistogram.WithLabelValues(p.Service, handler, p.Method, p.Code).Observe(duration.Seconds()) } +// ObserveHTTPResponseSize records the response size of the REST request. +// This method is called automatically by go-http-metrics/middleware func (r *RestCollector) ObserveHTTPResponseSize(_ context.Context, p httpmetrics.HTTPReqProperties, sizeBytes int64) { - r.httpResponseSizeHistogram.WithLabelValues(p.Service, p.ID, p.Method, p.Code).Observe(float64(sizeBytes)) + handler := r.mapURLToRoute(p.ID) + r.httpResponseSizeHistogram.WithLabelValues(p.Service, handler, p.Method, p.Code).Observe(float64(sizeBytes)) } +// AddInflightRequests increments and decrements the number of inflight request being processed. +// This method is called automatically by go-http-metrics/middleware func (r *RestCollector) AddInflightRequests(_ context.Context, p httpmetrics.HTTPProperties, quantity int) { - r.httpRequestsInflight.WithLabelValues(p.Service, p.ID).Add(float64(quantity)) + handler := r.mapURLToRoute(p.ID) + r.httpRequestsInflight.WithLabelValues(p.Service, handler).Add(float64(quantity)) } -// New custom method to track all requests made for every REST API request -func (r *RestCollector) AddTotalRequests(_ context.Context, method string, routeName string) { - r.httpRequestsTotal.WithLabelValues(method, routeName).Inc() +// AddTotalRequests records all REST requests +// This is a custom method called by the REST handler +func (r *RestCollector) AddTotalRequests(_ context.Context, method, path string) { + handler := r.mapURLToRoute(path) + r.httpRequestsTotal.WithLabelValues(method, handler).Inc() +} + +// mapURLToRoute uses the urlToRouteMapper callback to convert a URL to a route name +// This normalizes the URL, removing dynamic information converting it to a static string +func (r *RestCollector) mapURLToRoute(url string) string { + if r.urlToRouteMapper == nil { + return "unknown" + } + + route, err := r.urlToRouteMapper(url) + if err != nil { + return "unknown" + } + + return route } From a2e8fb1b9ecbd099a656da24cc562bd1d5f25d8a Mon Sep 17 00:00:00 2001 From: "Yahya Hassanzadeh, Ph.D" Date: Tue, 18 Jul 2023 19:40:43 -0700 Subject: [PATCH 156/169] [Networking] Handling iHave broken promises (and part-2 of overpromising) spams (#4566) * adds scoring parameters for mesh message delivery * refactors config-based approach to override-based approach * generates mocks * adds a documentation * extends godoc * adds test skeleton * fixes test * implements under-performing test * adds test for under-delivery in two topics * refactors test fixture * wip * err fix * applies refactoring * fixes test * parallelize the fixture * adds logs for overriding parameters * adds logging for network type to builder * fmt * renames a fixture function * adds warn logging for overriding score parameters * revises godocs * adds readme * fixes scoring test * fixing lint * lint fix * improves test quality * enables behavior penalty * adds size method to protected map * refactors gossipsub parameters * implements spammer constructor with inspector * implements broken promises test (below threshold) * adds ihave spam tests * adds godoc * extends godocs * extends comments * extends readme * revises parameters * updates tests * updates a godoc * updates godocs * fixes a godoc * adds t.Parallel * adds t.Parallel to all other tests * fmt --- insecure/corruptlibp2p/gossipsub_spammer.go | 54 ++- .../test/gossipsub/scoring/ihave_spam_test.go | 347 ++++++++++++++++++ .../test/gossipsub/scoring/scoring_test.go | 9 + network/p2p/scoring/README.md | 71 +++- network/p2p/scoring/score_option.go | 111 +++++- utils/unittest/protected_map.go | 7 + 6 files changed, 577 insertions(+), 22 deletions(-) create mode 100644 insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go diff --git a/insecure/corruptlibp2p/gossipsub_spammer.go b/insecure/corruptlibp2p/gossipsub_spammer.go index 2ec81a89e53..d38e1dfa12b 100644 --- a/insecure/corruptlibp2p/gossipsub_spammer.go +++ b/insecure/corruptlibp2p/gossipsub_spammer.go @@ -26,9 +26,33 @@ type GossipSubRouterSpammer struct { SpammerId flow.Identity } -// NewGossipSubRouterSpammer is the main method tests call for spamming attacks. -func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, opts ...p2ptest.NodeFixtureParameterOption) *GossipSubRouterSpammer { - spammerNode, spammerId, router := createSpammerNode(t, sporkId, role, provider, opts...) +// NewGossipSubRouterSpammer creates a new GossipSubRouterSpammer. +// Args: +// - t: the test object. +// - sporkId: the spork node's ID. +// - role: the role of the spork node. +// - provider: the identity provider. +// Returns: +// - the GossipSubRouterSpammer. +func NewGossipSubRouterSpammer(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider) *GossipSubRouterSpammer { + return NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, provider, func(id peer.ID, rpc *corrupt.RPC) error { + return nil // no-op + }) +} + +// NewGossipSubRouterSpammerWithRpcInspector creates a new GossipSubRouterSpammer with a custom RPC inspector. +// The RPC inspector is called before each incoming RPC is processed by the router. +// If the inspector returns an error, the RPC is dropped. +// Args: +// - t: the test object. +// - sporkId: the spork node's ID. +// - role: the role of the spork node. +// - provider: the identity provider. +// - inspector: the RPC inspector. +// Returns: +// - the GossipSubRouterSpammer. +func NewGossipSubRouterSpammerWithRpcInspector(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, inspector func(id peer.ID, rpc *corrupt.RPC) error) *GossipSubRouterSpammer { + spammerNode, spammerId, router := newSpammerNodeWithRpcInspector(t, sporkId, role, provider, inspector) return &GossipSubRouterSpammer{ router: router, SpammerNode: spammerNode, @@ -65,17 +89,31 @@ func (s *GossipSubRouterSpammer) Start(t *testing.T) { s.router.set(s.router.Get()) } -func createSpammerNode(t *testing.T, sporkId flow.Identifier, role flow.Role, provider module.IdentityProvider, opts ...p2ptest.NodeFixtureParameterOption) (p2p.LibP2PNode, flow.Identity, *atomicRouter) { +// newSpammerNodeWithRpcInspector creates a new spammer node, which is capable of sending spam control and actual messages to other nodes. +// It also creates a new atomic router that allows us to set the router to a new instance of the corrupt router. +// Args: +// - sporkId: the spork id of the spammer node. +// - role: the role of the spammer node. +// - provider: the identity provider of the spammer node. +// - inspector: the inspector function that is called when a message is received by the spammer node. +// Returns: +// - p2p.LibP2PNode: the spammer node. +// - flow.Identity: the identity of the spammer node. +// - *atomicRouter: the atomic router that allows us to set the router to a new instance of the corrupt router. +func newSpammerNodeWithRpcInspector( + t *testing.T, + sporkId flow.Identifier, + role flow.Role, + provider module.IdentityProvider, + inspector func(id peer.ID, rpc *corrupt.RPC) error) (p2p.LibP2PNode, flow.Identity, *atomicRouter) { router := newAtomicRouter() + var opts []p2ptest.NodeFixtureParameterOption opts = append(opts, p2ptest.WithRole(role), internal.WithCorruptGossipSub(CorruptGossipSubFactory(func(r *corrupt.GossipSubRouter) { require.NotNil(t, r) router.set(r) }), - CorruptGossipSubConfigFactoryWithInspector(func(id peer.ID, rpc *corrupt.RPC) error { - // here we can inspect the incoming RPC message to the spammer node - return nil - }))) + CorruptGossipSubConfigFactoryWithInspector(inspector))) spammerNode, spammerId := p2ptest.NodeFixture( t, sporkId, diff --git a/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go new file mode 100644 index 00000000000..3342b3ac4dc --- /dev/null +++ b/insecure/integration/functional/test/gossipsub/scoring/ihave_spam_test.go @@ -0,0 +1,347 @@ +package scoring + +import ( + "context" + "fmt" + "testing" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + corrupt "github.com/yhassanzadeh13/go-libp2p-pubsub" + + "github.com/onflow/flow-go/insecure/corruptlibp2p" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/mock" + "github.com/onflow/flow-go/network/channels" + "github.com/onflow/flow-go/network/p2p" + "github.com/onflow/flow-go/network/p2p/scoring" + p2ptest "github.com/onflow/flow-go/network/p2p/test" + "github.com/onflow/flow-go/utils/unittest" +) + +// TestGossipSubIHaveBrokenPromises_Below_Threshold tests that as long as the spammer stays below the ihave spam thresholds, it is not caught and +// penalized by the victim node. +// The thresholds are: +// Maximum messages that include iHave per heartbeat is: 10 (gossipsub parameter). +// Threshold for broken promises of iHave per heartbeat is: 10 (Flow-specific) parameter. It means that GossipSub samples one iHave id out of the +// entire RPC and if that iHave id is not eventually delivered within 3 seconds (gossipsub parameter), then the promise is considered broken. We set +// this threshold to 10 meaning that the first 10 broken promises are ignored. This is to allow for some network churn. +// Also, per hearbeat (i.e., decay interval), the spammer is allowed to send at most 5000 ihave messages (gossip sub parameter) on aggregate, and +// excess messages are dropped (without being counted as broken promises). +func TestGossipSubIHaveBrokenPromises_Below_Threshold(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + receivedIWants := unittest.NewProtectedMap[string, struct{}]() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, idProvider, func(id peer.ID, rpc *corrupt.RPC) error { + // override rpc inspector of the spammer node to keep track of the iwants it has received. + if rpc.RPC.Control == nil || rpc.RPC.Control.Iwant == nil { + return nil + } + for _, iwant := range rpc.RPC.Control.Iwant { + for _, msgId := range iwant.MessageIDs { + receivedIWants.Add(msgId, struct{}{}) + } + } + return nil + }) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + // we disable invalid message delivery parameters, as the way we implement spammer, when it spams ihave messages, it does not sign them. Hence, without decaying the invalid message deliveries, + // the node would be penalized for invalid message delivery way sooner than it can mount an ihave broken-promises spam attack. + blockTopicOverrideParams.InvalidMessageDeliveriesWeight = 0.0 + blockTopicOverrideParams.InvalidMessageDeliveriesDecay = 0.0 + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // creates 10 RPCs each with 10 iHave messages, each iHave message has 50 message ids, hence overall, we have 5000 iHave message ids. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises (one per RPC for a total of 10). + initialBehavioralPenalty := float64(0) // keeps track of the initial behavioral penalty of the spammer node for decay testing. + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if behavioralPenalty < 9 { // ideally it must be 10 (one per RPC), but we give it a buffer of 1 to account for decays and floating point errors. + return false + } + + initialBehavioralPenalty = behavioralPenalty + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // since spammer is not yet considered to be penalized, its score must be greater than the gossipsub health thresholds. + require.Greaterf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be greater than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be greater than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be greater than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // eventually, after a heartbeat the spammer behavioral counter must be decayed + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + if behavioralPenalty >= initialBehavioralPenalty { // after a heartbeat the spammer behavioral counter must be decayed. + return false + } + + return true + }, 2*time.Second, 100*time.Millisecond, "sanity check failed, the spammer behavioral counter must be decayed after a heartbeat") + + // since spammer stays below the threshold, it should be able to exchange messages with the victim node over pubsub. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// TestGossipSubIHaveBrokenPromises_Above_Threshold tests that a continuous stream of spam iHave broken promises will +// eventually cause the spammer node to be graylisted (i.e., no incoming RPCs from the spammer node will be accepted, and +// no outgoing RPCs to the spammer node will be sent). +// The test performs 3 rounds of attacks: each round with 10 RPCs, each RPC with 10 iHave messages, each iHave message with 50 message ids, hence overall, we have 5000 iHave message ids. +// Note that based on GossipSub parameters 5000 iHave is the most one can send within one decay interval. +// First round of attack makes spammers broken promises still below the threshold of 10 RPCs (broken promises are counted per RPC), hence no degradation of the spammers score. +// Second round of attack makes spammers broken promises above the threshold of 10 RPCs, hence a degradation of the spammers score. +// Third round of attack makes spammers broken promises to around 20 RPCs above the threshold, which causes the graylisting of the spammer node. +func TestGossipSubIHaveBrokenPromises_Above_Threshold(t *testing.T) { + t.Parallel() + + role := flow.RoleConsensus + sporkId := unittest.IdentifierFixture() + blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) + + receivedIWants := unittest.NewProtectedMap[string, struct{}]() + idProvider := mock.NewIdentityProvider(t) + spammer := corruptlibp2p.NewGossipSubRouterSpammerWithRpcInspector(t, sporkId, role, idProvider, func(id peer.ID, rpc *corrupt.RPC) error { + // override rpc inspector of the spammer node to keep track of the iwants it has received. + if rpc.RPC.Control == nil || rpc.RPC.Control.Iwant == nil { + return nil + } + for _, iwant := range rpc.RPC.Control.Iwant { + for _, msgId := range iwant.MessageIDs { + receivedIWants.Add(msgId, struct{}{}) + } + } + return nil + }) + + ctx, cancel := context.WithCancel(context.Background()) + signalerCtx := irrecoverable.NewMockSignalerContext(t, ctx) + // we override some of the default scoring parameters in order to speed up the test in a time-efficient manner. + blockTopicOverrideParams := scoring.DefaultTopicScoreParams() + blockTopicOverrideParams.MeshMessageDeliveriesActivation = 1 * time.Second // we start observing the mesh message deliveries after 1 second of the node startup. + // we disable invalid message delivery parameters, as the way we implement spammer, when it spams ihave messages, it does not sign them. Hence, without decaying the invalid message deliveries, + // the node would be penalized for invalid message delivery way sooner than it can mount an ihave broken-promises spam attack. + blockTopicOverrideParams.InvalidMessageDeliveriesWeight = 0.0 + blockTopicOverrideParams.InvalidMessageDeliveriesDecay = 0.0 + victimNode, victimIdentity := p2ptest.NodeFixture( + t, + sporkId, + t.Name(), + idProvider, + p2ptest.WithRole(role), + p2ptest.WithPeerScoreTracerInterval(1*time.Second), + p2ptest.EnablePeerScoringWithOverride(&p2p.PeerScoringConfigOverride{ + TopicScoreParams: map[channels.Topic]*pubsub.TopicScoreParams{ + blockTopic: blockTopicOverrideParams, + }, + DecayInterval: 1 * time.Second, // we override the decay interval to 1 second so that the score is updated within 1 second intervals. + }), + ) + + idProvider.On("ByPeerID", victimNode.Host().ID()).Return(&victimIdentity, true).Maybe() + idProvider.On("ByPeerID", spammer.SpammerNode.Host().ID()).Return(&spammer.SpammerId, true).Maybe() + ids := flow.IdentityList{&spammer.SpammerId, &victimIdentity} + nodes := []p2p.LibP2PNode{spammer.SpammerNode, victimNode} + + p2ptest.StartNodes(t, signalerCtx, nodes, 100*time.Millisecond) + defer p2ptest.StopNodes(t, nodes, cancel, 2*time.Second) + + p2ptest.LetNodesDiscoverEachOther(t, ctx, nodes, ids) + p2ptest.TryConnectionAndEnsureConnected(t, ctx, nodes) + + // checks end-to-end message delivery works on GossipSub + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + initScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "score for spammer node must be present") + + // FIRST ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the second round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + // ideally it must be 10 (one per RPC), but we give it a buffer of 1 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 9 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + scoreAfterFirstRound, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "score for spammer node must be present") + // spammer score after first round must not be decreased severely, we account for 10% drop due to under-performing + // (on sending fresh new messages since that is not part of the test). + require.Greater(t, scoreAfterFirstRound, 0.9*initScore) + + // SECOND ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the second round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + + // ideally we should have 20 (10 from the first round, 10 from the second round), but we give it a buffer of 2 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 18 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok := victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // with the second round of the attack, the spammer is about 10 broken promises above the threshold (total ~20 broken promises, but the first 10 are not counted). + // we expect the score to be dropped to initScore - 10 * 10 * 0.01 * scoring.MaxAppSpecificReward, however, instead of 10, we consider 8 about the threshold, to account for decays. + require.LessOrEqual(t, spammerScore, initScore-8*8*0.01*scoring.MaxAppSpecificReward, "sanity check failed, the score of the spammer node must be less than the initial score minus 8 * 8 * 0.01 * scoring.MaxAppSpecificReward: %f, actual: %f", initScore-10*10*10-2*scoring.MaxAppSpecificReward, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be greater than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be greater than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Greaterf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be greater than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // since the spammer score is above the gossip, graylist and publish thresholds, it should be still able to exchange messages with victim. + p2ptest.EnsurePubsubMessageExchange(t, ctx, nodes, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) + + // THIRD ROUND OF ATTACK: spammer sends 10 RPCs to the victim node, each containing 500 iHave messages, we expect spammer to be graylisted. + spamIHaveBrokenPromise(t, spammer, blockTopic.String(), receivedIWants, victimNode) + + // wait till victim counts the spam iHaves as broken promises for the third round of attack (one per RPC for a total of 10). + require.Eventually(t, func() bool { + behavioralPenalty, ok := victimNode.PeerScoreExposer().GetBehaviourPenalty(spammer.SpammerNode.Host().ID()) + if !ok { + return false + } + // ideally we should have 30 (10 from the first round, 10 from the second round, 10 from the third round), but we give it a buffer of 3 to account for decays and floating point errors. + // note that we intentionally override the decay speed to be 60-times faster in this test. + if behavioralPenalty < 27 { + return false + } + + return true + // Note: we have to wait at least 3 seconds for an iHave to be considered as broken promise (gossipsub parameters), we set it to 10 + // seconds to be on the safe side. + }, 10*time.Second, 100*time.Millisecond) + + spammerScore, ok = victimNode.PeerScoreExposer().GetScore(spammer.SpammerNode.Host().ID()) + require.True(t, ok, "sanity check failed, we should have a score for the spammer node") + // with the third round of the attack, the spammer is about 20 broken promises above the threshold (total ~30 broken promises), hence its overall score must be below the gossip, publish, and graylist thresholds, meaning that + // victim will not exchange messages with it anymore, and also that it will be graylisted meaning all incoming and outgoing RPCs to and from the spammer will be dropped by the victim. + require.Lessf(t, spammerScore, scoring.DefaultGossipThreshold, "sanity check failed, the score of the spammer node must be less than gossip threshold: %f, actual: %f", scoring.DefaultGossipThreshold, spammerScore) + require.Lessf(t, spammerScore, scoring.DefaultPublishThreshold, "sanity check failed, the score of the spammer node must be less than publish threshold: %f, actual: %f", scoring.DefaultPublishThreshold, spammerScore) + require.Lessf(t, spammerScore, scoring.DefaultGraylistThreshold, "sanity check failed, the score of the spammer node must be less than graylist threshold: %f, actual: %f", scoring.DefaultGraylistThreshold, spammerScore) + + // since the spammer score is below the gossip, graylist and publish thresholds, it should not be able to exchange messages with victim anymore. + p2ptest.EnsureNoPubsubExchangeBetweenGroups(t, ctx, []p2p.LibP2PNode{spammer.SpammerNode}, []p2p.LibP2PNode{victimNode}, blockTopic, 1, func() interface{} { + return unittest.ProposalFixture() + }) +} + +// spamIHaveBrokenPromises is a test utility function that is exclusive for the TestGossipSubIHaveBrokenPromises tests. +// It creates and sends 10 RPCs each with 10 iHave messages, each iHave message has 50 message ids, hence overall, we have 5000 iHave message ids. +// It then sends those iHave spams to the victim node and waits till the victim node receives them. +// Args: +// - t: the test instance. +// - spammer: the spammer node. +// - topic: the topic to spam. +// - receivedIWants: a map to keep track of the iWants received by the victim node (exclusive to TestGossipSubIHaveBrokenPromises). +// - victimNode: the victim node. +func spamIHaveBrokenPromise(t *testing.T, spammer *corruptlibp2p.GossipSubRouterSpammer, topic string, receivedIWants *unittest.ProtectedMap[string, struct{}], victimNode p2p.LibP2PNode) { + spamMsgs := spammer.GenerateCtlMessages(10, corruptlibp2p.WithIHave(10, 50, topic)) + var sentIHaves []string + for _, msg := range spamMsgs { + for _, iHave := range msg.Ihave { + for _, msgId := range iHave.MessageIDs { + require.NotContains(t, sentIHaves, msgId) + sentIHaves = append(sentIHaves, msgId) + } + } + } + require.Len(t, sentIHaves, 5000, "sanity check failed, we should have 5000 iHave message ids, actual: %d", len(sentIHaves)) + + // spams the victim node with 1000 spam iHave messages, since iHave messages are for junk message ids, there will be no + // reply from spammer to victim over the iWants. Hence, the victim must count this towards 10 broken promises. + // This sums up to 10 broken promises (1 per RPC). + spammer.SpamControlMessage(t, victimNode, spamMsgs, p2ptest.PubsubMessageFixture(t, p2ptest.WithTopic(topic))) + + // wait till all the spam iHaves are responded with iWants. + require.Eventually(t, func() bool { + for _, msgId := range sentIHaves { + if _, ok := receivedIWants.Get(msgId); !ok { + return false + } + } + + return true + }, 5*time.Second, 100*time.Millisecond, fmt.Sprintf("sanity check failed, we should have received all the iWants for the spam iHaves, expected: %d, actual: %d", len(sentIHaves), receivedIWants.Size())) +} diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index 2fc1135b5bb..d8baf4be735 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -27,6 +27,8 @@ import ( // a spammer peer, the victim will eventually penalize the spammer and stop receiving messages from them. // Note: the term integration is used here because it requires integrating all components of the libp2p stack. func TestGossipSubInvalidMessageDelivery_Integration(t *testing.T) { + t.Parallel() + tt := []struct { name string spamMsgFactory func(spammerId peer.ID, victimId peer.ID, topic channels.Topic) *pubsub_pb.Message @@ -94,6 +96,7 @@ func TestGossipSubInvalidMessageDelivery_Integration(t *testing.T) { // - t: the test instance. // - spamMsgFactory: a function that creates unique invalid messages to spam the victim with. func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory func(peer.ID, peer.ID, channels.Topic) *pubsub_pb.Message) { + role := flow.RoleConsensus sporkId := unittest.IdentifierFixture() blockTopic := channels.TopicFromChannel(channels.PushBlocks, sporkId) @@ -177,6 +180,8 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun // TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic tests that when a peer is under-performing in a topic mesh, its score is (slightly) penalized. func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus sporkId := unittest.IdentifierFixture() @@ -277,6 +282,8 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_SingleTopic(t *testing.T) { // TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics tests that when a peer is under-performing in two topics, it is penalized in both topics. func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus sporkId := unittest.IdentifierFixture() @@ -388,6 +395,8 @@ func TestGossipSubMeshDeliveryScoring_UnderDelivery_TwoTopics(t *testing.T) { // TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted tests that replayed messages will not be counted towards the mesh message deliveries. func TestGossipSubMeshDeliveryScoring_Replay_Will_Not_Counted(t *testing.T) { + t.Parallel() + role := flow.RoleConsensus sporkId := unittest.IdentifierFixture() diff --git a/network/p2p/scoring/README.md b/network/p2p/scoring/README.md index a04e93fa741..38a758db439 100644 --- a/network/p2p/scoring/README.md +++ b/network/p2p/scoring/README.md @@ -159,7 +159,6 @@ This decay process ensures that a peer cannot rest on its past deliveries; it mu It helps maintain a lively and dynamic network environment, incentivizing constant active participation from all peers. ### Scenario 6: Replay Attack -## Example Scenario: Preventing Replay Attacks The `defaultMeshMessageDeliveriesWindow` and `defaultMeshMessageDeliveriesActivation` parameters play a crucial role in preventing replay attacks in the GossipSub protocol. Let's illustrate this with an example. Consider a scenario where we have three peers: `Peer A`, `Peer B`, and `Peer C`. All three peers are active participants in `Topic X`. **Initial State**: At Time = 0: `Peer A` generates and broadcasts a new message `M` in `Topic X`. `Peer B` and `Peer C` receive this message from `Peer A` and update their message caches accordingly. @@ -172,6 +171,75 @@ Therefore, the message `M` from `Peer B` will not count towards `Peer B`'s messa This effectively discouraging replay attacks of messages older than the `defaultMeshMessageDeliveriesWindow`. This mechanism, combined with other parameters, helps maintain the security and efficiency of the network by discouraging harmful behaviors such as message replay attacks. +## Mitigating iHave Broken Promises Attacks in GossipSub Protocol +### What is an iHave Broken Promise Attack? +In the GossipSub protocol, peers gossip information about new messages to a subset of random peers (out of their local mesh) in the form of an "iHave" message which basically tells the receiving peer what messages the sender has. +The receiving peer then replies with an "iWant" message, requesting for the messages it doesn't have. Note that for the peers in local mesh the actual new messages are sent instead of an "iHave" message (i.e., eager push). However, +iHave-iWant protocol is part of a complementary mechanism to ensure that the information is disseminated to the entire network in a timely manner (i.e., lazy pull). + +An "iHave Broken Promise" attack occurs when a peer advertises many "iHave" for a message but doesn't respond to the "iWant" requests for those messages. +This not only hinders the effective dissemination of information but can also strain the network with redundant requests. Hence, we stratify it as a spam behavior mounting a DoS attack on the network. + +### Detecting iHave Broken Promise Attacks +Detecting iHave Broken Promise Attacks is done by the GossipSub itself. On each incoming RPC from a remote node, the local GossipSub node checks if the RPC contains an iHave message. It then samples one (and only one) iHave message +randomly out of the entire set of iHave messages piggybacked on the incoming RPC. If the sampled iHave message is not literally addressed with the actual message, the local GossipSub node considers this as an iHave broken promise and +increases the behavior penalty counter for that remote node. Hence, incrementing the behavior penalty counter for a remote peer is done per RPC containing at least one iHave broken promise and not per iHave message. +Note that the behavior penalty counter also keeps track of GRAFT flood attacks that are done by a remote peer when it advertises many GRAFTs while it is on a PRUNE backoff by the local node. Mitigating iHave broken promise attacks also +mitigates GRAFT flood attacks. + +### Configuring GossipSub Parameters +In order to mitigate the iHave broken promises attacks, GossipSub expects the application layer (i.e., Flow protocol) to properly configure the relevant scoring parameters, notably: + +- `BehaviourPenaltyThreshold` is set to `defaultBehaviourPenaltyThreshold`, i.e., `10`. +- `BehaviourPenaltyWeight` is set to `defaultBehaviourPenaltyWeight`, i.e., `0.01` * `MaxAppSpecificPenalty` +- `BehaviourPenaltyDecay` is set to `defaultBehaviourPenaltyDecay`, i.e., `0.99`. + +#### 1. `defaultBehaviourPenaltyThreshold` +This parameter sets the threshold for when the behavior of a peer is considered bad. Misbehavior is defined as advertising an iHave without responding to the iWants (iHave broken promises), and attempting on GRAFT when the peer is considered for a PRUNE backoff. +If a remote peer sends an RPC that advertises at least one iHave for a message but doesn't respond to the iWant requests for that message within the next `3 seconds`, the peer misbehavior counter is incremented by `1`. This threshold is set to `10`, meaning that we at most tolerate 10 such RPCs containing iHave broken promises. After this, the peer is penalized for every excess RPC containing iHave broken promises. The counter decays by (`0.99`) every decay interval (defaultDecayInterval) i.e., every minute. + +#### 2. `defaultBehaviourPenaltyWeight` +This is the weight applied as a penalty when a peer's misbehavior goes beyond the `defaultBehaviourPenaltyThreshold`. +The penalty is applied to the square of the difference between the misbehavior counter and the threshold, i.e., -|w| * (misbehavior counter - threshold)^2, where `|w|` is the absolute value of the `defaultBehaviourPenaltyWeight`. +Note that `defaultBehaviourPenaltyWeight` is a negative value, meaning that the penalty is applied in the opposite direction of the misbehavior counter. For sake of illustration, we use the notion of `-|w|` to denote that a negative penalty is applied. +We set `defaultBehaviourPenaltyWeight` to `0.01 * MaxAppSpecificPenalty`, meaning a peer misbehaving `10` times more than the threshold (i.e., `10 + 10`) will lose its entire `MaxAppSpecificReward`, which is a reward given to all staked nodes in Flow blockchain. +This also means that a peer misbehaving `sqrt(2) * 10` times more than the threshold will cause the peer score to be dropped below the `MaxAppSpecificPenalty`, which is also below the `GraylistThreshold`, and the peer will be graylisted (i.e., all incoming and outgoing GossipSub RPCs from and to that peer will be rejected). +This means the peer is temporarily disconnected from the network, preventing it from causing further harm. + +#### 3. defaultBehaviourPenaltyDecay +This is the decay interval for the misbehavior counter of a peer. This counter is decayed by the `defaultBehaviourPenaltyDecay` parameter (e.g., `0.99`) per decay interval, which is currently every 1 minute. +This parameter helps to gradually reduce the effect of past misbehaviors and provides a chance for penalized nodes to rejoin the network. A very slow decay rate can help identify and isolate persistent offenders, while also allowing potentially honest nodes that had transient issues to regain their standing in the network. +The duration a peer remains graylisted is governed by the choice of `defaultBehaviourPenaltyWeight` and the decay parameters. +Based on the given configuration, a peer which has misbehaved on `sqrt(2) * 10` RPCs more than the threshold will get graylisted (disconnected at GossipSub level). +With the decay interval set to 1 minute and decay value of 0.99, a graylisted peer due to broken promises would be expected to be reconnected in about 50 minutes. +This is calculated by solving for `x` in the equation `(0.99)^x * (sqrt(2) * 10)^2 * MaxAppSpecificPenalty > GraylistThreshold`. +Simplifying, we find `x` to be approximately `527` decay intervals, or roughly `527` minutes. +This is the estimated time it would take for a severely misbehaving peer to have its penalty decayed enough to exceed the `GraylistThreshold` and thus be reconnected to the network. + +### Example Scenarios +**Scenario 1: Misbehaving Below Threshold** +In this scenario, consider peer `B` that has recently joined the network and is taking part in GossipSub. +This peer advertises to peer `A` many `iHave` messages over an RPC. But when other peer `A` requests these message with `iWant`s it fails to deliver the message within 3 seconds. +This action constitutes an _iHave broken promise_ for a single RPC and peer `A` increases the local behavior penalty counter of peer `B` by 1. +If the peer `B` commits this misbehavior infrequently, such that the total number of these RPCs does not exceed the `defaultBehaviourPenaltyThreshold` (set to 10 in our configuration), +the misbehavior counter for this peer will increment by 1 for each RPC and decays by `1%` evey decay interval (1 minute), but no additional penalty will be applied. +The misbehavior counter decays by a factor of `defaultBehaviourPenaltyDecay` (0.99) every minute, allowing the peer to recover from these minor infractions without significant disruption. + +**Scenario 2: Misbehaving Above Threshold But Below Graylisting** +Now consider that peer `B` frequently sends RPCs advertising many `iHaves` to peer `A` but fails to deliver the promised messages. +If the number of these misbehaviors exceeds our threshold (10 in our configuration), the peer `B` is now penalized by the local GossipSub mechanism of peer `A`. +The amount of the penalty is determined by the `defaultBehaviourPenaltyWeight` (set to 0.01 * MaxAppSpecificPenalty) applied to the square of the difference between the misbehavior counter and the threshold. +This penalty will progressively affect the peer's score, deteriorating its reputation in the local GossipSub scoring system of node `A`, but does not yet result in disconnection or graylisting. +The peer has a chance to amend its behavior before crossing into graylisting territory through stop misbehaving and letting the score to decay. +When peer `B` has a deteriorated score at node `A`, it will be less likely to be selected by node `A` as its local mesh peer (i.e., to directly receive new messages from node `A`), and is deprived of the opportunity to receive new messages earlier through node `A`. + +**Scenario 3: Graylisting** +Now assume that peer `B` peer has been continually misbehaving, with RPCs including iHave broken promises exceeding `sqrt(2) * 10` the threshold. +At this point, the peer's score drops below the `GraylistThreshold` due to the `defaultBehaviorPenaltyWeight` applied to the excess misbehavior. +The peer is then graylisted by peer `A`, i.e., peer `A` rejects all incoming RPCs to and from peer `B` at GossipSub level. +In our configuration, peer `B` will stay disconnected for at least `527` decay intervals or approximately `527` minutes. +This gives a strong disincentive for the peer to continue this behavior and also gives it time to recover and eventually be reconnected to the network. + ## Customization The scoring mechanism can be easily customized to suit the needs of the Flow network. This includes changing the scoring parameters, thresholds, and the scoring function itself. You can customize the scoring parameters and thresholds by using the various setter methods provided in the `ScoreOptionConfig` object. Additionally, you can provide a custom app-specific scoring function through the `SetAppSpecificScoreFunction` method. @@ -183,7 +251,6 @@ Example of setting custom app-specific scoring function: config.SetAppSpecificScoreFunction(customAppSpecificScoreFunction) ``` - ## Peer Scoring System Integration The peer scoring system is integrated with the GossipSub protocol through the `ScoreOption` configuration option. This option is passed to the GossipSub at the time of initialization. diff --git a/network/p2p/scoring/score_option.go b/network/p2p/scoring/score_option.go index cb7c5e3a4d4..0ae676005cb 100644 --- a/network/p2p/scoring/score_option.go +++ b/network/p2p/scoring/score_option.go @@ -18,10 +18,22 @@ import ( ) const ( + // DefaultAppSpecificScoreWeight is the default weight for app-specific scores. It is used to scale the app-specific + // scores to the same range as the other scores. At the current version, we don't distinguish between the app-specific + // scores and the other scores, so we set it to 1. DefaultAppSpecificScoreWeight = 1 - MaxAppSpecificPenalty = float64(-100) - MinAppSpecificPenalty = -1 - MaxAppSpecificReward = float64(100) + + // MaxAppSpecificReward is the default reward for well-behaving staked peers. If a peer does not have + // any misbehavior record, e.g., invalid subscription, invalid message, etc., it will be rewarded with this score. + MaxAppSpecificReward = float64(100) + + // MaxAppSpecificPenalty is the maximum penalty for sever offenses that we apply to a remote node score. The score + // mechanism of GossipSub in Flow is designed in a way that all other infractions are penalized with a fraction of + // this value. We have also set the other parameters such as DefaultGraylistThreshold, DefaultGossipThreshold and DefaultPublishThreshold to + // be a bit higher than this, i.e., MaxAppSpecificPenalty + 1. This ensures that a node with a score of MaxAppSpecificPenalty + // will be graylisted (i.e., all incoming and outgoing RPCs are rejected) and will not be able to publish or gossip any messages. + MaxAppSpecificPenalty = -1 * MaxAppSpecificReward + MinAppSpecificPenalty = -1 // DefaultStakedIdentityReward is the default reward for staking peers. It is applied to the peer's score when // the peer does not have any misbehavior record, e.g., invalid subscription, invalid message, etc. @@ -44,7 +56,7 @@ const ( // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all gossips // to and from peers with penalty -100 are ignored. - DefaultGossipThreshold = -99 + DefaultGossipThreshold = MaxAppSpecificPenalty + 1 // DefaultPublishThreshold when a peer's penalty drops below this threshold, // self-published messages are not propagated towards this peer. @@ -55,7 +67,7 @@ const ( // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all penalized peers are deprived of // receiving any published messages. - DefaultPublishThreshold = -99 + DefaultPublishThreshold = MaxAppSpecificPenalty + 1 // DefaultGraylistThreshold when a peer's penalty drops below this threshold, the peer is graylisted, i.e., // incoming RPCs from the peer are ignored. @@ -65,7 +77,7 @@ const ( // // How we use it: // As current max penalty is -100, we set the threshold to -99 so that all penalized peers are graylisted. - DefaultGraylistThreshold = -99 + DefaultGraylistThreshold = MaxAppSpecificPenalty + 1 // DefaultAcceptPXThreshold when a peer sends us PX information with a prune, we only accept it and connect to the supplied // peers if the originating peer's penalty exceeds this threshold. @@ -75,7 +87,7 @@ const ( // How we use it: // As current max reward is 100, we set the threshold to 99 so that we only receive supplied peers from // well-behaved peers. - DefaultAcceptPXThreshold = 99 + DefaultAcceptPXThreshold = MaxAppSpecificReward - 1 // DefaultOpportunisticGraftThreshold when the median peer penalty in the mesh drops below this value, // the peer may select more peers with penalty above the median to opportunistically graft on the mesh. @@ -210,6 +222,73 @@ const ( // number of actual message deliveries of a peer in a topic mesh. This is to account for // the time that it takes for a peer to start up and receive messages from other peers in the topic mesh. defaultMeshMessageDeliveriesActivation = 2 * defaultDecayInterval + + // defaultBehaviorPenaltyThreshold is the threshold when the behavior of a peer is considered as bad by GossipSub. + // Currently, the misbehavior is defined as advertising an iHave without responding to the iWants (iHave broken promises), as well as attempting + // on GRAFT when the peer is considered for a PRUNE backoff, i.e., the local peer does not allow the peer to join the local topic mesh + // for a while, and the remote peer keep attempting on GRAFT (aka GRAFT flood). + // When the misbehavior counter of a peer goes beyond this threshold, the peer is penalized by defaultBehaviorPenaltyWeight (see below) for the excess misbehavior. + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // + // We set it to 10, meaning that we at most tolerate 10 of such RPCs containing iHave broken promises. After that, the peer is penalized for every + // excess RPC containing iHave broken promises. + // The counter is also decayed by (0.99) every decay interval (defaultDecayInterval) i.e., every minute. + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system). + defaultBehaviourPenaltyThreshold = 10 + + // defaultBehaviorPenaltyWeight is the weight for applying penalty when a peer misbehavior goes beyond the threshold. + // Misbehavior of a peer at gossipsub layer is defined as advertising an iHave without responding to the iWants (broken promises), as well as attempting + // on GRAFT when the peer is considered for a PRUNE backoff, i.e., the local peer does not allow the peer to join the local topic mesh + // This is detected by the GossipSub scoring system, and the peer is penalized by defaultBehaviorPenaltyWeight. + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // + // The penalty is applied to the square of the difference between the misbehavior counter and the threshold, i.e., -|w| * (misbehavior counter - threshold)^2. + // We set it to 0.01 * MaxAppSpecificPenalty, which means that misbehaving 10 times more than the threshold (i.e., 10 + 10) will cause the peer to lose + // its entire AppSpecificReward that is awarded by our app-specific scoring function to all staked (i.e., authorized) nodes by default. + // Moreover, as the MaxAppSpecificPenalty is -MaxAppSpecificReward, misbehaving sqrt(2) * 10 times more than the threshold will cause the peer score + // to be dropped below the MaxAppSpecificPenalty, which is also below the GraylistThreshold, and the peer will be graylisted (i.e., disconnected). + // + // The math is as follows: -|w| * (misbehavior - threshold)^2 = 0.01 * MaxAppSpecificPenalty * (misbehavior - threshold)^2 < 2 * MaxAppSpecificPenalty + // if misbehavior > threshold + sqrt(2) * 10. + // As shown above, with this choice of defaultBehaviorPenaltyWeight, misbehaving sqrt(2) * 10 times more than the threshold will cause the peer score + // to be dropped below the MaxAppSpecificPenalty, which is also below the GraylistThreshold, and the peer will be graylisted (i.e., disconnected). This weight + // is chosen in a way that with almost a few misbehaviors more than the threshold, the peer will be graylisted. The rationale relies on the fact that + // the misbehavior counter is incremented by 1 for each RPC containing one or more broken promises. Hence, it is per RPC, and not per broken promise. + // Having sqrt(2) * 10 broken promises RPC is a blatant misbehavior, and the peer should be graylisted. With decay interval of 1 minute, and decay value of + // 0.99 we expect a graylisted node due to borken promises to get back in about 527 minutes, i.e., (0.99)^x * (sqrt(2) * 10)^2 * MaxAppSpecificPenalty > GraylistThreshold + // where x is the number of decay intervals that the peer is graylisted. As MaxAppSpecificPenalty and GraylistThresholds are close, we can simplify the inequality + // to (0.99)^x * (sqrt(2) * 10)^2 > 1 --> (0.99)^x * 200 > 1 --> (0.99)^x > 1/200 --> x > log(1/200) / log(0.99) --> x > 527.17 decay intervals, i.e., 527 minutes. + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system that are reported by the engines). + defaultBehaviourPenaltyWeight = 0.01 * MaxAppSpecificPenalty + + // defaultBehaviorPenaltyDecay is the decay interval for the misbehavior counter of a peer. The misbehavior counter is + // incremented by GossipSub for iHave broken promises or the GRAFT flooding attacks (i.e., each GRAFT received from a remote peer while that peer is on a PRUNE backoff). + // + // An iHave broken promise means that a peer advertises an iHave for a message, but does not respond to the iWant requests for that message. + // For iHave broken promises, the gossipsub scoring works as follows: + // It samples ONLY A SINGLE iHave out of the entire RPC. + // If that iHave is not followed by an actual message within the next 3 seconds, the peer misbehavior counter is incremented by 1. + // This means that regardless of how many iHave broken promises an RPC contains, the misbehavior counter is incremented by 1. + // That is why we decay the misbehavior counter very slow, as this counter indicates a severe misbehavior. + // + // The misbehavior counter is decayed per decay interval (i.e., defaultDecayInterval = 1 minute) by GossipSub. + // We set it to 0.99, which means that the misbehavior counter is decayed by 1% per decay interval. + // With the generous threshold that we set (i.e., defaultBehaviourPenaltyThreshold = 10), we take the peers going beyond the threshold as persistent misbehaviors, + // We expect honest peers never to go beyond the threshold, and if they do, we expect them to go back below the threshold quickly. + // + // Note that misbehaviors are counted by GossipSub across all topics (and is different from the Application Layer Misbehaviors that we count through + // the ALSP system that is based on the engines report). + defaultBehaviourPenaltyDecay = 0.99 ) // ScoreOption is a functional option for configuring the peer scoring system. @@ -333,7 +412,7 @@ func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { s.logger. Warn(). Str(logging.KeyNetworkingSecurity, "true"). - Msg("app specific score function is overridden") + Msg("app specific score function is overridden, should never happen in production") } if cfg.decayInterval > 0 { @@ -343,7 +422,7 @@ func NewScoreOption(cfg *ScoreOptionConfig) *ScoreOption { Warn(). Str(logging.KeyNetworkingSecurity, "true"). Dur("decay_interval_ms", cfg.decayInterval). - Msg("decay interval is overridden") + Msg("decay interval is overridden, should never happen in production") } // registers the score registry as the consumer of the invalid control message notifications @@ -411,14 +490,15 @@ func (s *ScoreOption) TopicScoreParams(topic *pubsub.Topic) *pubsub.TopicScorePa } func defaultPeerScoreParams() *pubsub.PeerScoreParams { + // DO NOT CHANGE THE DEFAULT VALUES, THEY ARE TUNED FOR THE BEST SECURITY PRACTICES. return &pubsub.PeerScoreParams{ Topics: make(map[string]*pubsub.TopicScoreParams), // we don't set all the parameters, so we skip the atomic validation. // atomic validation fails initialization if any parameter is not set. SkipAtomicValidation: true, - // DecayInterval is the interval over which we decay the effect of past behavior. So that - // a good or bad behavior will not have a permanent effect on the penalty. It is also interval - // that GossipSub refreshes the scores of all peers. + // DecayInterval is the interval over which we decay the effect of past behavior, so that + // a good or bad behavior will not have a permanent effect on the penalty. It is also the interval + // that GossipSub uses to refresh the scores of all peers. DecayInterval: defaultDecayInterval, // DecayToZero defines the maximum value below which a peer scoring counter is reset to zero. // This is to prevent the counter from decaying to a very small value. @@ -427,11 +507,18 @@ func defaultPeerScoreParams() *pubsub.PeerScoreParams { DecayToZero: defaultDecayToZero, // AppSpecificWeight is the weight of the application specific penalty. AppSpecificWeight: DefaultAppSpecificScoreWeight, + // BehaviourPenaltyThreshold is the threshold above which a peer is penalized for GossipSub-level misbehaviors. + BehaviourPenaltyThreshold: defaultBehaviourPenaltyThreshold, + // BehaviourPenaltyWeight is the weight of the GossipSub-level penalty. + BehaviourPenaltyWeight: defaultBehaviourPenaltyWeight, + // BehaviourPenaltyDecay is the decay of the GossipSub-level penalty (applied every decay interval). + BehaviourPenaltyDecay: defaultBehaviourPenaltyDecay, } } // DefaultTopicScoreParams returns the default score params for topics. func DefaultTopicScoreParams() *pubsub.TopicScoreParams { + // DO NOT CHANGE THE DEFAULT VALUES, THEY ARE TUNED FOR THE BEST SECURITY PRACTICES. p := &pubsub.TopicScoreParams{ TopicWeight: defaultTopicWeight, SkipAtomicValidation: defaultTopicSkipAtomicValidation, diff --git a/utils/unittest/protected_map.go b/utils/unittest/protected_map.go index f0b4a65ad92..a2af2f5f513 100644 --- a/utils/unittest/protected_map.go +++ b/utils/unittest/protected_map.go @@ -57,3 +57,10 @@ func (p *ProtectedMap[K, V]) ForEach(fn func(k K, v V) error) error { } return nil } + +// Size returns the size of the map. +func (p *ProtectedMap[K, V]) Size() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.m) +} From f2bc373a52912c7a936bee6e78e08246fab32add Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 18 Jul 2023 21:37:00 -0600 Subject: [PATCH 157/169] use new interface EntropyProvider in FVM instead of protocol.Snapshot --- .../computation/computer/computer.go | 16 +++++++- engine/execution/testutil/fixtures.go | 18 ++++++--- engine/verification/fetcher/engine.go | 1 - fvm/context.go | 10 ++--- fvm/environment/env.go | 3 +- fvm/environment/facade_env.go | 2 +- fvm/environment/mock/entropy_provider.go | 37 +++++++++++++++++++ fvm/environment/random_generator.go | 36 ++++++++++-------- fvm/environment/random_generator_test.go | 10 ++--- fvm/fvm_blockcontext_test.go | 4 +- model/verification/verifiableChunkData.go | 2 +- module/chunks/chunkVerifier.go | 16 +++++++- 12 files changed, 111 insertions(+), 44 deletions(-) create mode 100644 fvm/environment/mock/entropy_provider.go diff --git a/engine/execution/computation/computer/computer.go b/engine/execution/computation/computer/computer.go index ad155dc097c..4c3749f9e8f 100644 --- a/engine/execution/computation/computer/computer.go +++ b/engine/execution/computation/computer/computer.go @@ -209,7 +209,13 @@ func (e *blockComputer) queueTransactionRequests( collectionCtx := fvm.NewContextFromParent( e.vmCtx, fvm.WithBlockHeader(blockHeader), - fvm.WithProtocolSnapshot(e.protocolState.AtBlockID(blockId)), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(e.protocolState.AtBlockID(blockId)), ) for idx, collection := range rawCollections { @@ -244,7 +250,13 @@ func (e *blockComputer) queueTransactionRequests( systemCtx := fvm.NewContextFromParent( e.systemChunkCtx, fvm.WithBlockHeader(blockHeader), - fvm.WithProtocolSnapshot(e.protocolState.AtBlockID(blockId)), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(e.protocolState.AtBlockID(blockId)), ) systemCollectionLogger := systemCtx.Logger.With(). Str("block_id", blockIdStr). diff --git a/engine/execution/testutil/fixtures.go b/engine/execution/testutil/fixtures.go index f85ff2f4f42..3113f2df9af 100644 --- a/engine/execution/testutil/fixtures.go +++ b/engine/execution/testutil/fixtures.go @@ -18,6 +18,8 @@ import ( "github.com/onflow/flow-go/engine/execution" "github.com/onflow/flow-go/engine/execution/utils" "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/environment" + envMock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/ledger/common/pathfinder" @@ -630,16 +632,16 @@ func ComputationResultFixture(t *testing.T) *execution.ComputationResult { } } -// ProtocolSnapshotWithSourceFixture returns a protocol state snapshot mock that only +// EntropyProviderFixture returns an entropy provider mock that // supports RandomSource(). // If input is nil, a random source fixture is generated. -func ProtocolSnapshotWithSourceFixture(source []byte) protocol.Snapshot { +func EntropyProviderFixture(source []byte) environment.EntropyProvider { if source == nil { source = unittest.SignatureFixture() } - snapshot := protocolMock.Snapshot{} - snapshot.On("RandomSource").Return(source, nil) - return &snapshot + provider := envMock.EntropyProvider{} + provider.On("RandomSource").Return(source, nil) + return &provider } // ProtocolStateWithSourceFixture returns a protocol state mock that only @@ -647,7 +649,11 @@ func ProtocolSnapshotWithSourceFixture(source []byte) protocol.Snapshot { // The snapshot mock only supports RandomSource(). // If input is nil, a random source fixture is generated. func ProtocolStateWithSourceFixture(source []byte) protocol.State { - snapshot := ProtocolSnapshotWithSourceFixture(source) + if source == nil { + source = unittest.SignatureFixture() + } + snapshot := &protocolMock.Snapshot{} + snapshot.On("RandomSource").Return(source, nil) state := protocolMock.State{} state.On("AtBlockID", mock.Anything).Return(snapshot) return &state diff --git a/engine/verification/fetcher/engine.go b/engine/verification/fetcher/engine.go index 9373001918d..fd53417b720 100644 --- a/engine/verification/fetcher/engine.go +++ b/engine/verification/fetcher/engine.go @@ -527,7 +527,6 @@ func (e *Engine) pushToVerifier(chunk *flow.Chunk, return fmt.Errorf("could not get block: %w", err) } snapshot := e.state.AtBlockID(header.ID()) - vchunk, err := e.makeVerifiableChunkData(chunk, header, snapshot, result, chunkDataPack) if err != nil { return fmt.Errorf("could not verify chunk: %w", err) diff --git a/fvm/context.go b/fvm/context.go index 349f25af8d8..250955d2082 100644 --- a/fvm/context.go +++ b/fvm/context.go @@ -13,7 +13,6 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" - "github.com/onflow/flow-go/state/protocol" ) const ( @@ -161,13 +160,12 @@ func WithEventCollectionSizeLimit(limit uint64) Option { } } -// WithProtocolSnapshot sets the protocol state for a virtual machine context. +// WithEntropyProvider sets the entropy provider of a virtual machine context. // -// The VM uses the protocol state to provide protocol information to the Cadence runtime, -// including the source of the pseudorandom number generator. -func WithProtocolSnapshot(snapshot protocol.Snapshot) Option { +// The VM uses the input to provide entropy to the Cadence runtime randomness functions. +func WithEntropyProvider(source environment.EntropyProvider) Option { return func(ctx Context) Context { - ctx.Snapshot = snapshot + ctx.EntropyProvider = source return ctx } } diff --git a/fvm/environment/env.go b/fvm/environment/env.go index 21808c40e65..518b49a737a 100644 --- a/fvm/environment/env.go +++ b/fvm/environment/env.go @@ -10,7 +10,6 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/state/protocol" ) // Environment implements the accounts business logic and exposes cadence @@ -99,7 +98,7 @@ type EnvironmentParams struct { BlockInfoParams TransactionInfoParams - protocol.Snapshot + EntropyProvider ContractUpdaterParams } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 7c73dca0854..76ac5205725 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -228,7 +228,7 @@ func NewTransactionEnvironment( env.RandomGenerator = NewRandomGenerator( tracer, - params.Snapshot, + params.EntropyProvider, params.TxId, ) diff --git a/fvm/environment/mock/entropy_provider.go b/fvm/environment/mock/entropy_provider.go new file mode 100644 index 00000000000..d35570da859 --- /dev/null +++ b/fvm/environment/mock/entropy_provider.go @@ -0,0 +1,37 @@ +// Code generated by mockery v2.21.4. DO NOT EDIT. + +package mock + +import mock "github.com/stretchr/testify/mock" + +// EntropyProvider is an autogenerated mock type for the EntropyProvider type +type EntropyProvider struct { + mock.Mock +} + +// RandomSource provides a mock function with given fields: +func (_m *EntropyProvider) RandomSource() ([]byte, error) { + ret := _m.Called() + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func() ([]byte, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []byte); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + diff --git a/fvm/environment/random_generator.go b/fvm/environment/random_generator.go index 030d7381c52..63562ff06bf 100644 --- a/fvm/environment/random_generator.go +++ b/fvm/environment/random_generator.go @@ -10,13 +10,22 @@ import ( "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module/trace" - "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/prg" ) +// EntropyProvider represents an entropy (source of randomness) provider +type EntropyProvider interface { + // RandomSource provides a source of entropy that can be + // expanded into randoms (using a pseudo-random generator). + // The returned slice should have at least 128 bits of entropy. + // The function doesn't error in normal operations, any + // error should be treated as an exception. + RandomSource() ([]byte, error) +} + type RandomGenerator interface { // UnsafeRandom returns a random uint64 - // Todo: rename to Random() once Cadence interface is updated + // The name follows Cadence interface UnsafeRandom() (uint64, error) } @@ -25,13 +34,11 @@ var _ RandomGenerator = (*randomGenerator)(nil) // randomGenerator implements RandomGenerator and is used // for the transactions execution environment type randomGenerator struct { - tracer tracing.TracerSpan - - stateSnapshot protocol.Snapshot + tracer tracing.TracerSpan + entropySource EntropyProvider txId flow.Identifier - - prg random.Rand - isPRGCreated bool + prg random.Rand + isPRGCreated bool } type ParseRestrictedRandomGenerator struct { @@ -61,12 +68,12 @@ func (gen ParseRestrictedRandomGenerator) UnsafeRandom() ( func NewRandomGenerator( tracer tracing.TracerSpan, - stateSnapshot protocol.Snapshot, + entropySource EntropyProvider, txId flow.Identifier, ) RandomGenerator { gen := &randomGenerator{ tracer: tracer, - stateSnapshot: stateSnapshot, + entropySource: entropySource, txId: txId, isPRGCreated: false, // PRG is not created } @@ -77,12 +84,9 @@ func NewRandomGenerator( func (gen *randomGenerator) createPRG() (random.Rand, error) { // Use the protocol state source of randomness [SoR] for the current block's // execution - source, err := gen.stateSnapshot.RandomSource() - // expected errors of RandomSource() are: - // - storage.ErrNotFound if the QC is unknown. - // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown - // at this stage, snapshot reference block should be known and the QC should also be known, - // so no error is expected in normal operations + source, err := gen.entropySource.RandomSource() + // `RandomSource` does not error in normal operations. + // Any error should be treated as an exception. if err != nil { return nil, fmt.Errorf("reading random source from state failed: %w", err) } diff --git a/fvm/environment/random_generator_test.go b/fvm/environment/random_generator_test.go index 60f413b7a36..539aa99423f 100644 --- a/fvm/environment/random_generator_test.go +++ b/fvm/environment/random_generator_test.go @@ -8,22 +8,22 @@ import ( "github.com/stretchr/testify/require" "github.com/onflow/flow-go/crypto/random" - "github.com/onflow/flow-go/engine/execution/testutil" "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) func TestRandomGenerator(t *testing.T) { - // protocol snapshot mock - snapshot := testutil.ProtocolSnapshotWithSourceFixture(nil) + entropyProvider := &mock.EntropyProvider{} + entropyProvider.On("RandomSource").Return(unittest.RandomBytes(48), nil) getRandoms := func(txId flow.Identifier, N int) []uint64 { // seed the RG with the same block header urg := environment.NewRandomGenerator( tracing.NewTracerSpan(), - snapshot, + entropyProvider, txId) numbers := make([]uint64, N) for i := 0; i < N; i++ { @@ -41,7 +41,7 @@ func TestRandomGenerator(t *testing.T) { txId := unittest.TransactionFixture().ID() urg := environment.NewRandomGenerator( tracing.NewTracerSpan(), - snapshot, + entropyProvider, txId) // make sure n is a power of 2 so that there is no bias in the last class diff --git a/fvm/fvm_blockcontext_test.go b/fvm/fvm_blockcontext_test.go index d3ef4cb4100..e4148fcc5c7 100644 --- a/fvm/fvm_blockcontext_test.go +++ b/fvm/fvm_blockcontext_test.go @@ -1671,12 +1671,12 @@ func TestBlockContext_Random(t *testing.T) { chain, vm := createChainAndVm(flow.Mainnet) header := &flow.Header{Height: 42} - snapshot := testutil.ProtocolSnapshotWithSourceFixture(nil) + source := testutil.EntropyProviderFixture(nil) ctx := fvm.NewContext( fvm.WithChain(chain), fvm.WithBlockHeader(header), - fvm.WithProtocolSnapshot(snapshot), + fvm.WithEntropyProvider(source), fvm.WithCadenceLogging(true), ) diff --git a/model/verification/verifiableChunkData.go b/model/verification/verifiableChunkData.go index 293240a55c3..2f6f1e22579 100644 --- a/model/verification/verifiableChunkData.go +++ b/model/verification/verifiableChunkData.go @@ -11,7 +11,7 @@ type VerifiableChunkData struct { IsSystemChunk bool // indicates whether this is a system chunk Chunk *flow.Chunk // the chunk to be verified Header *flow.Header // BlockHeader that contains this chunk - Snapshot protocol.Snapshot // protocol state snapshot + Snapshot protocol.Snapshot // state snapshot at the chunk's block Result *flow.ExecutionResult // execution result of this block ChunkDataPack *flow.ChunkDataPack // chunk data package needed to verify this chunk EndState flow.StateCommitment // state commitment at the end of this chunk diff --git a/module/chunks/chunkVerifier.go b/module/chunks/chunkVerifier.go index 1d3561ac092..fb1b81fbf98 100644 --- a/module/chunks/chunkVerifier.go +++ b/module/chunks/chunkVerifier.go @@ -58,7 +58,13 @@ func (fcv *ChunkVerifier) Verify( ctx = fvm.NewContextFromParent( fcv.systemChunkCtx, fvm.WithBlockHeader(vc.Header), - fvm.WithProtocolSnapshot(vc.Snapshot), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(vc.Snapshot), ) txBody, err := blueprints.SystemChunkTransaction(fcv.vmCtx.Chain) @@ -73,7 +79,13 @@ func (fcv *ChunkVerifier) Verify( ctx = fvm.NewContextFromParent( fcv.vmCtx, fvm.WithBlockHeader(vc.Header), - fvm.WithProtocolSnapshot(vc.Snapshot), + // `protocol.Snapshot` implements `EntropyProvider` interface + // Note that `Snapshot` possible errors for RandomSource() are: + // - storage.ErrNotFound if the QC is unknown. + // - state.ErrUnknownSnapshotReference if the snapshot reference block is unknown + // However, at this stage, snapshot reference block should be known and the QC should also be known, + // so no error is expected in normal operations, as required by `EntropyProvider`. + fvm.WithEntropyProvider(vc.Snapshot), ) transactions = make( From 29deae0115e32ccf3efaa6f79b2c38c66264702c Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 18 Jul 2023 22:01:17 -0600 Subject: [PATCH 158/169] update entropy provider mock file --- fvm/environment/mock/entropy_provider.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fvm/environment/mock/entropy_provider.go b/fvm/environment/mock/entropy_provider.go index d35570da859..cf3f19fb306 100644 --- a/fvm/environment/mock/entropy_provider.go +++ b/fvm/environment/mock/entropy_provider.go @@ -35,3 +35,17 @@ func (_m *EntropyProvider) RandomSource() ([]byte, error) { return r0, r1 } +type mockConstructorTestingTNewEntropyProvider interface { + mock.TestingT + Cleanup(func()) +} + +// NewEntropyProvider creates a new instance of EntropyProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEntropyProvider(t mockConstructorTestingTNewEntropyProvider) *EntropyProvider { + mock := &EntropyProvider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From c6624d2b86e2eb18160797ff8a92635a08af83be Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Wed, 19 Jul 2023 11:49:00 +0300 Subject: [PATCH 159/169] ScriptExecutor should always populate env values even when the script errors --- fvm/script.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fvm/script.go b/fvm/script.go index 10bd5d68717..4b457a66e21 100644 --- a/fvm/script.go +++ b/fvm/script.go @@ -198,11 +198,13 @@ func (executor *scriptExecutor) executeScript() error { Source: executor.proc.Script, Arguments: executor.proc.Arguments, }, - common.ScriptLocation(executor.proc.ID)) + common.ScriptLocation(executor.proc.ID), + ) + populateErr := executor.output.PopulateEnvironmentValues(executor.env) if err != nil { return err } executor.output.Value = value - return executor.output.PopulateEnvironmentValues(executor.env) + return populateErr } From 574fb22f64c8dd2e60dc957bb3f8883f89195833 Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Wed, 19 Jul 2023 18:19:46 +0300 Subject: [PATCH 160/169] Append both errors in executeScript() with multierror --- fvm/script.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fvm/script.go b/fvm/script.go index 4b457a66e21..977ba7c0a42 100644 --- a/fvm/script.go +++ b/fvm/script.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/hashicorp/go-multierror" "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" @@ -206,5 +207,5 @@ func (executor *scriptExecutor) executeScript() error { } executor.output.Value = value - return populateErr + return multierror.Append(err, populateErr) } From 1987e6041e8046f8c41f637dd4bffec93797292c Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Wed, 19 Jul 2023 10:33:37 -0700 Subject: [PATCH 161/169] require url mapper in constructor --- cmd/access/node_builder/access_node_builder.go | 6 +++++- module/metrics/rest_api.go | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 6138e70560b..5e60ef5bdf5 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -967,7 +967,11 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("rest metrics", func(node *cmd.NodeConfig) error { - builder.RESTMetrics = metrics.NewRestCollector(rest.URLToRoute, node.MetricsRegisterer) + m, err := metrics.NewRestCollector(rest.URLToRoute, node.MetricsRegisterer) + if err != nil { + return err + } + builder.RESTMetrics = m return nil }). Module("access metrics", func(node *cmd.NodeConfig) error { diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go index aa7978dbe56..70c4bab2154 100644 --- a/module/metrics/rest_api.go +++ b/module/metrics/rest_api.go @@ -2,6 +2,7 @@ package metrics import ( "context" + "fmt" "time" "github.com/prometheus/client_golang/prometheus" @@ -24,7 +25,11 @@ var _ module.RestMetrics = (*RestCollector)(nil) // NewRestCollector returns a new metrics RestCollector that implements the RestCollector // using Prometheus as the backend. -func NewRestCollector(urlToRouteMapper func(string) (string, error), registerer prometheus.Registerer) *RestCollector { +func NewRestCollector(urlToRouteMapper func(string) (string, error), registerer prometheus.Registerer) (*RestCollector, error) { + if urlToRouteMapper == nil { + return nil, fmt.Errorf("urlToRouteMapper cannot be nil") + } + r := &RestCollector{ urlToRouteMapper: urlToRouteMapper, httpRequestDurHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ @@ -99,10 +104,6 @@ func (r *RestCollector) AddTotalRequests(_ context.Context, method, path string) // mapURLToRoute uses the urlToRouteMapper callback to convert a URL to a route name // This normalizes the URL, removing dynamic information converting it to a static string func (r *RestCollector) mapURLToRoute(url string) string { - if r.urlToRouteMapper == nil { - return "unknown" - } - route, err := r.urlToRouteMapper(url) if err != nil { return "unknown" From a7634ff51ae22150c75d7e25263ef1a3ae4e69e5 Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Wed, 19 Jul 2023 11:43:16 -0700 Subject: [PATCH 162/169] fix error return --- cmd/access/node_builder/access_node_builder.go | 6 +++--- module/metrics/rest_api.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 5e60ef5bdf5..c1b59dd5681 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -217,7 +217,7 @@ type FlowAccessNodeBuilder struct { CollectionsToMarkExecuted *stdmap.Times BlocksToMarkExecuted *stdmap.Times TransactionMetrics *metrics.TransactionCollector - RESTMetrics *metrics.RestCollector + RestMetrics *metrics.RestCollector AccessMetrics module.AccessMetrics PingMetrics module.PingMetrics Committee hotstuff.DynamicCommittee @@ -971,14 +971,14 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { if err != nil { return err } - builder.RESTMetrics = m + builder.RestMetrics = m return nil }). Module("access metrics", func(node *cmd.NodeConfig) error { builder.AccessMetrics = metrics.NewAccessCollector( metrics.WithTransactionMetrics(builder.TransactionMetrics), metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), - metrics.WithRestMetrics(builder.RESTMetrics), + metrics.WithRestMetrics(builder.RestMetrics), ) return nil }). diff --git a/module/metrics/rest_api.go b/module/metrics/rest_api.go index 70c4bab2154..e9132f243c6 100644 --- a/module/metrics/rest_api.go +++ b/module/metrics/rest_api.go @@ -70,7 +70,7 @@ func NewRestCollector(urlToRouteMapper func(string) (string, error), registerer r.httpRequestsTotal, ) - return r + return r, nil } // ObserveHTTPRequestDuration records the duration of the REST request. From 1aa6e3dc006a989c623a429439a73a857e0ac140 Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Thu, 20 Jul 2023 10:56:38 +0300 Subject: [PATCH 163/169] Add test cases for asserting the population of ProcedureOutput from script execution --- fvm/fvm_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ fvm/script.go | 4 +-- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index 587df638ee9..e93c19c575a 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -2023,6 +2023,87 @@ func TestScriptAccountKeyMutationsFailure(t *testing.T) { ) } +func TestScriptExecutionLimit(t *testing.T) { + + t.Parallel() + + script := fvm.Script([]byte(` + pub fun main() { + var s: Int256 = 1024102410241024 + var i: Int256 = 0 + var a: Int256 = 7 + var b: Int256 = 5 + var c: Int256 = 2 + + while i < 150000 { + s = s * a + s = s / b + s = s / c + i = i + 1 + } + } + `)) + + bootstrapProcedureOptions := []fvm.BootstrapProcedureOption{ + fvm.WithTransactionFee(fvm.DefaultTransactionFees), + fvm.WithExecutionMemoryLimit(math.MaxUint32), + fvm.WithExecutionEffortWeights(map[common.ComputationKind]uint64{ + common.ComputationKindStatement: 1569, + common.ComputationKindLoop: 1569, + common.ComputationKindFunctionInvocation: 1569, + environment.ComputationKindGetValue: 808, + environment.ComputationKindCreateAccount: 2837670, + environment.ComputationKindSetValue: 765, + }), + fvm.WithExecutionMemoryWeights(meter.DefaultMemoryWeights), + fvm.WithMinimumStorageReservation(fvm.DefaultMinimumStorageReservation), + fvm.WithAccountCreationFee(fvm.DefaultAccountCreationFee), + fvm.WithStorageMBPerFLOW(fvm.DefaultStorageMBPerFLOW), + } + + t.Run("Exceeding computation limit", + newVMTest().withBootstrapProcedureOptions( + bootstrapProcedureOptions..., + ).withContextOptions( + fvm.WithTransactionFeesEnabled(true), + fvm.WithAccountStorageLimit(true), + fvm.WithComputationLimit(10000), + ).run( + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + scriptCtx := fvm.NewContextFromParent(ctx) + + _, output, err := vm.Run(scriptCtx, script, snapshotTree) + require.NoError(t, err) + require.Error(t, output.Err) + require.True(t, errors.IsComputationLimitExceededError(output.Err)) + require.ErrorContains(t, output.Err, "computation exceeds limit (10000)") + require.GreaterOrEqual(t, output.ComputationUsed, uint64(10000)) + require.GreaterOrEqual(t, output.MemoryEstimate, uint64(548020260)) + }, + ), + ) + + t.Run("Sufficient computation limit", + newVMTest().withBootstrapProcedureOptions( + bootstrapProcedureOptions..., + ).withContextOptions( + fvm.WithTransactionFeesEnabled(true), + fvm.WithAccountStorageLimit(true), + fvm.WithComputationLimit(20000), + ).run( + func(t *testing.T, vm fvm.VM, chain flow.Chain, ctx fvm.Context, snapshotTree snapshot.SnapshotTree) { + scriptCtx := fvm.NewContextFromParent(ctx) + + _, output, err := vm.Run(scriptCtx, script, snapshotTree) + require.NoError(t, err) + require.NoError(t, output.Err) + require.GreaterOrEqual(t, output.ComputationUsed, uint64(17955)) + require.GreaterOrEqual(t, output.MemoryEstimate, uint64(984017413)) + }, + ), + ) +} + func TestInteractionLimit(t *testing.T) { type testCase struct { name string diff --git a/fvm/script.go b/fvm/script.go index 977ba7c0a42..c979fb309f5 100644 --- a/fvm/script.go +++ b/fvm/script.go @@ -203,9 +203,9 @@ func (executor *scriptExecutor) executeScript() error { ) populateErr := executor.output.PopulateEnvironmentValues(executor.env) if err != nil { - return err + return multierror.Append(err, populateErr) } executor.output.Value = value - return multierror.Append(err, populateErr) + return populateErr } From c0b443602e69fa52d7e350a4da06fe128c42b152 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 20 Jul 2023 13:33:55 +0300 Subject: [PATCH 164/169] Updated according to comments --- engine/access/integration_unsecure_grpc_server_test.go | 4 +++- module/grpcserver/server.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/engine/access/integration_unsecure_grpc_server_test.go b/engine/access/integration_unsecure_grpc_server_test.go index 78725d73f7e..e5fa9dbe4c9 100644 --- a/engine/access/integration_unsecure_grpc_server_test.go +++ b/engine/access/integration_unsecure_grpc_server_test.go @@ -248,6 +248,8 @@ func (suite *SameGRPCPortTestSuite) SetupTest() { unittest.AssertClosesBefore(suite.T(), suite.stateStreamEng.Ready(), 2*time.Second) } +// TestEnginesOnTheSameGrpcPort verifies if both AccessAPI and ExecutionDataAPI client successfully connect and continue +// to work when configured on the same port func (suite *SameGRPCPortTestSuite) TestEnginesOnTheSameGrpcPort() { ctx := context.Background() @@ -285,7 +287,7 @@ func TestSameGRPCTestSuite(t *testing.T) { suite.Run(t, new(SameGRPCPortTestSuite)) } -// unsecureGRPCClient creates an unsecure GRPC client +// unsecureAccessAPIClient creates an unsecure grpc AccessAPI client func (suite *SameGRPCPortTestSuite) unsecureAccessAPIClient(conn *grpc.ClientConn) accessproto.AccessAPIClient { client := accessproto.NewAccessAPIClient(conn) return client diff --git a/module/grpcserver/server.go b/module/grpcserver/server.go index 425c37a042a..309cb9315f2 100644 --- a/module/grpcserver/server.go +++ b/module/grpcserver/server.go @@ -12,8 +12,8 @@ import ( "github.com/onflow/flow-go/module/irrecoverable" ) -// GrpcServer defines a grpc server that starts once and uses in different Engines. -// It makes it easy to configure the node to use the same port for both APIs. +// GrpcServer wraps `grpc.Server` and allows to manage it using `component.Component` interface. It can be injected +// into different engines making it possible to use single grpc server for multiple services which live in different modules. type GrpcServer struct { component.Component log zerolog.Logger @@ -25,6 +25,8 @@ type GrpcServer struct { grpcAddress net.Addr } +var _ component.Component = (*GrpcServer)(nil) + // NewGrpcServer returns a new grpc server. func NewGrpcServer(log zerolog.Logger, grpcListenAddr string, From ea7748f94d81618ce9037f7ec324accd1ac1bfee Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 20 Jul 2023 16:13:18 +0300 Subject: [PATCH 165/169] Added part of updates according to comments --- access/handler.go | 38 +++---------------- .../rest/apiproxy/rest_proxy_handler.go | 3 +- engine/access/rest/routes/scripts_test.go | 3 -- .../access/rest/routes/transactions_test.go | 5 +-- engine/access/rpc/backend/backend.go | 3 +- engine/common/rpc/convert/events.go | 34 ++++++++++++++++- engine/common/rpc/convert/events_test.go | 3 +- integration/tests/access/observer_test.go | 2 +- utils/unittest/fixtures.go | 2 +- 9 files changed, 46 insertions(+), 47 deletions(-) diff --git a/access/handler.go b/access/handler.go index e5b4a18374c..11e47dd3521 100644 --- a/access/handler.go +++ b/access/handler.go @@ -3,17 +3,17 @@ package access import ( "context" - "github.com/onflow/flow/protobuf/go/flow/access" - "github.com/onflow/flow/protobuf/go/flow/entities" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/onflow/flow-go/consensus/hotstuff" "github.com/onflow/flow-go/consensus/hotstuff/signature" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" + + "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" ) type Handler struct { @@ -516,7 +516,7 @@ func (h *Handler) GetEventsForHeightRange( return nil, err } - resultEvents, err := BlockEventsToMessages(results) + resultEvents, err := convert.BlockEventsToMessages(results) if err != nil { return nil, err } @@ -548,7 +548,7 @@ func (h *Handler) GetEventsForBlockIDs( return nil, err } - resultEvents, err := BlockEventsToMessages(results) + resultEvents, err := convert.BlockEventsToMessages(results) if err != nil { return nil, err } @@ -680,34 +680,6 @@ func executionResultToMessages(er *flow.ExecutionResult, metadata *entities.Meta }, nil } -func BlockEventsToMessages(blocks []flow.BlockEvents) ([]*access.EventsResponse_Result, error) { - results := make([]*access.EventsResponse_Result, len(blocks)) - - for i, block := range blocks { - event, err := blockEventsToMessage(block) - if err != nil { - return nil, err - } - results[i] = event - } - - return results, nil -} - -func blockEventsToMessage(block flow.BlockEvents) (*access.EventsResponse_Result, error) { - eventMessages := make([]*entities.Event, len(block.Events)) - for i, event := range block.Events { - eventMessages[i] = convert.EventToMessage(event) - } - timestamp := timestamppb.New(block.BlockTimestamp) - return &access.EventsResponse_Result{ - BlockId: block.BlockID[:], - BlockHeight: block.BlockHeight, - BlockTimestamp: timestamp, - Events: eventMessages, - }, nil -} - // WithBlockSignerDecoder configures the Handler to decode signer indices // via the provided hotstuff.BlockSignerDecoder func WithBlockSignerDecoder(signerIndicesDecoder hotstuff.BlockSignerDecoder) func(*Handler) { diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index 8275348959d..b89dc46b771 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -2,6 +2,7 @@ package apiproxy import ( "context" + "fmt" "time" "google.golang.org/grpc/status" @@ -44,7 +45,7 @@ func NewRestProxyHandler( timeout, maxMsgSize) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create REST forwarder: %w", err) } restProxyHandler := &RestProxyHandler{ diff --git a/engine/access/rest/routes/scripts_test.go b/engine/access/rest/routes/scripts_test.go index a3f2a64663c..8a6a63cc819 100644 --- a/engine/access/rest/routes/scripts_test.go +++ b/engine/access/rest/routes/scripts_test.go @@ -48,7 +48,6 @@ func TestScripts(t *testing.T) { t.Run("get by Latest height", func(t *testing.T) { backend := &mock.API{} - backend.Mock. On("ExecuteScriptAtLatestBlock", mocks.Anything, validCode, [][]byte{validArgs}). Return([]byte("hello world"), nil) @@ -92,7 +91,6 @@ func TestScripts(t *testing.T) { t.Run("get error", func(t *testing.T) { backend := &mock.API{} - backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, uint64(1337), validCode, [][]byte{validArgs}). Return(nil, status.Error(codes.Internal, "internal server error")) @@ -109,7 +107,6 @@ func TestScripts(t *testing.T) { t.Run("get invalid", func(t *testing.T) { backend := &mock.API{} - backend.Mock. On("ExecuteScriptAtBlockHeight", mocks.Anything, mocks.Anything, mocks.Anything, mocks.Anything). Return(nil, nil) diff --git a/engine/access/rest/routes/transactions_test.go b/engine/access/rest/routes/transactions_test.go index a23e2053cb7..3b02c4d5de5 100644 --- a/engine/access/rest/routes/transactions_test.go +++ b/engine/access/rest/routes/transactions_test.go @@ -72,7 +72,6 @@ func createTransactionReq(body interface{}) *http.Request { func TestGetTransactions(t *testing.T) { t.Run("get by ID without results", func(t *testing.T) { backend := &mock.API{} - tx := unittest.TransactionFixture() req := getTransactionReq(tx.ID().String(), false, "", "") @@ -348,7 +347,7 @@ func TestCreateTransaction(t *testing.T) { tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} tx.Arguments = [][]uint8{} - req := createTransactionReq(unittest.ValidCreateBody(tx)) + req := createTransactionReq(unittest.CreateSendTxHttpPayload(tx)) backend.Mock. On("SendTransaction", mocks.Anything, &tx). @@ -415,7 +414,7 @@ func TestCreateTransaction(t *testing.T) { for _, test := range tests { tx := unittest.TransactionBodyFixture() tx.PayloadSignatures = []flow.TransactionSignature{unittest.TransactionSignatureFixture()} - testTx := unittest.ValidCreateBody(tx) + testTx := unittest.CreateSendTxHttpPayload(tx) testTx[test.inputField] = test.inputValue req := createTransactionReq(testTx) diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 590dff099a8..fbc98f5319a 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -213,7 +213,8 @@ func New( return b } -// NewCache constructs cache and its size. +// NewCache constructs cache for storing connections to other nodes. +// No errors are expected during normal operations. func NewCache( log zerolog.Logger, accessMetrics module.AccessMetrics, diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index e80631ae4a3..c25f405355b 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -4,13 +4,15 @@ import ( "encoding/json" "fmt" + "google.golang.org/protobuf/types/known/timestamppb" + "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow-go/model/flow" accessproto "github.com/onflow/flow/protobuf/go/flow/access" + "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" - - "github.com/onflow/flow-go/model/flow" ) // EventToMessage converts a flow.Event to a protobuf message @@ -193,3 +195,31 @@ func MessageToBlockEvents(blockEvents *accessproto.EventsResponse_Result) flow.B Events: MessagesToEvents(blockEvents.Events), } } + +func BlockEventsToMessages(blocks []flow.BlockEvents) ([]*accessproto.EventsResponse_Result, error) { + results := make([]*accessproto.EventsResponse_Result, len(blocks)) + + for i, block := range blocks { + event, err := BlockEventsToMessage(block) + if err != nil { + return nil, err + } + results[i] = event + } + + return results, nil +} + +func BlockEventsToMessage(block flow.BlockEvents) (*accessproto.EventsResponse_Result, error) { + eventMessages := make([]*entities.Event, len(block.Events)) + for i, event := range block.Events { + eventMessages[i] = EventToMessage(event) + } + timestamp := timestamppb.New(block.BlockTimestamp) + return &accessproto.EventsResponse_Result{ + BlockId: block.BlockID[:], + BlockHeight: block.BlockHeight, + BlockTimestamp: timestamp, + Events: eventMessages, + }, nil +} diff --git a/engine/common/rpc/convert/events_test.go b/engine/common/rpc/convert/events_test.go index 6483dac72ee..879db710f8b 100644 --- a/engine/common/rpc/convert/events_test.go +++ b/engine/common/rpc/convert/events_test.go @@ -11,7 +11,6 @@ import ( jsoncdc "github.com/onflow/cadence/encoding/json" execproto "github.com/onflow/flow/protobuf/go/flow/execution" - "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -207,7 +206,7 @@ func TestConvertMessagesToBlockEvents(t *testing.T) { blockEvents[i] = unittest.BlockEventsFixture(header, 2) } - msg, err := access.BlockEventsToMessages(blockEvents) + msg, err := convert.BlockEventsToMessages(blockEvents) require.NoError(t, err) converted := convert.MessagesToBlockEvents(msg) diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 1cc222cc990..55733820d92 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -492,7 +492,7 @@ func createTx(net *testnet.FlowNetwork) *bytes.Buffer { tx.PayloadSignatures = []flow.TransactionSignature{signature} tx.EnvelopeSignatures = []flow.TransactionSignature{signature} - jsonBody, _ := json.Marshal(unittest.ValidCreateBody(*tx)) + jsonBody, _ := json.Marshal(unittest.CreateSendTxHttpPayload(*tx)) return bytes.NewBuffer(jsonBody) } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 1da31deb093..e2b30bcefad 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -2455,7 +2455,7 @@ func ChunkExecutionDataFixture(t *testing.T, minSize int, opts ...func(*executio } } -func ValidCreateBody(tx flow.TransactionBody) map[string]interface{} { +func CreateSendTxHttpPayload(tx flow.TransactionBody) map[string]interface{} { tx.Arguments = [][]uint8{} // fix how fixture creates nil values auth := make([]string, len(tx.Authorizers)) for i, a := range tx.Authorizers { From 06c67f205090b40ee97c5135b0896e6d897dbedc Mon Sep 17 00:00:00 2001 From: Janez Podhostnik Date: Thu, 20 Jul 2023 17:19:44 +0200 Subject: [PATCH 166/169] Upgrade Emulator --- insecure/go.mod | 22 +++++++--------- insecure/go.sum | 36 +++++++++++++------------- integration/go.mod | 5 ++-- integration/go.sum | 6 ++--- integration/localnet/client/Dockerfile | 12 ++++----- 5 files changed, 38 insertions(+), 43 deletions(-) diff --git a/insecure/go.mod b/insecure/go.mod index 4673eca91ef..02cbccb4c28 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -9,22 +9,22 @@ require ( github.com/libp2p/go-libp2p v0.28.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/onflow/flow-go v0.29.8 + github.com/onflow/flow-go v0.31.1-0.20230718164039-e3411eff1e9d github.com/onflow/flow-go/crypto v0.24.9 github.com/rs/zerolog v1.29.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/yhassanzadeh13/go-libp2p-pubsub v0.6.11-flow-expose-msg.0.20230703223453-544e2fe28a26 go.uber.org/atomic v1.11.0 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.1 google.golang.org/protobuf v1.30.0 ) require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect cloud.google.com/go/storage v1.28.1 // indirect github.com/aws/aws-sdk-go-v2 v1.17.7 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.19 // indirect @@ -59,7 +59,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -74,7 +74,6 @@ require ( github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gammazero/deque v0.1.0 // indirect github.com/gammazero/workerpool v1.1.2 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -103,14 +102,14 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2 v2.0.0-rc.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-20200501113911-9a95f0fdbfea // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-block-format v0.1.2 // indirect @@ -221,7 +220,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.15.0 // indirect github.com/stretchr/objx v0.5.0 // indirect @@ -254,7 +253,7 @@ require ( golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/term v0.9.0 // indirect @@ -265,10 +264,9 @@ require ( gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index ba524fee752..e4153a1db4f 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -37,14 +37,14 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/kms v1.0.0/go.mod h1:nhUehi+w7zht2XrUfvTRNpxrfayBHqP4lu2NSywui/0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/profiler v0.3.0 h1:R6y/xAeifaUXxd2x6w+jIwKxoKl8Cv5HJvcvASTPWJo= @@ -272,8 +272,9 @@ github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDm github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -344,7 +345,6 @@ github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2K github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g= github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -561,8 +561,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -608,8 +609,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= @@ -1444,8 +1445,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1778,8 +1779,8 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2124,8 +2125,8 @@ google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2161,8 +2162,8 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0 h1:TLkBREm4nIsEcexnCjgQd5GQWaHcqMzwQV0TX9pq8S0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= @@ -2215,7 +2216,6 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/integration/go.mod b/integration/go.mod index eaee57337b3..f42b3a76f1c 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -20,8 +20,8 @@ require ( github.com/onflow/cadence v0.39.14 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 - github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7 - github.com/onflow/flow-go v0.31.1-0.20230704154018-87a84e9d36c2 + github.com/onflow/flow-emulator v0.53.0 + github.com/onflow/flow-go v0.31.1-0.20230718164039-e3411eff1e9d github.com/onflow/flow-go-sdk v0.41.9 github.com/onflow/flow-go/crypto v0.24.9 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 @@ -229,7 +229,6 @@ require ( github.com/onflow/atree v0.6.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v0.7.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.1.0 // indirect - github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e // indirect github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d // indirect github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead // indirect github.com/onflow/sdks v0.5.0 // indirect diff --git a/integration/go.sum b/integration/go.sum index 820fdc51d4b..9649fd57d60 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1360,8 +1360,8 @@ github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-5 github.com/onflow/flow-core-contracts/lib/go/contracts v1.2.4-0.20230703193002-53362441b57d/go.mod h1:xAiV/7TKhw863r6iO3CS5RnQ4F+pBY1TxD272BsILlo= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3 h1:X25A1dNajNUtE+KoV76wQ6BR6qI7G65vuuRXxDDqX7E= github.com/onflow/flow-core-contracts/lib/go/templates v1.2.3/go.mod h1:dqAUVWwg+NlOhsuBHex7bEWmsUjsiExzhe/+t4xNH6A= -github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7 h1:UFcuL4WO1h41vTL6MVBNA6JSeCDrxgHXv2R7617ukeQ= -github.com/onflow/flow-emulator v0.51.2-0.20230704183611-ecad54e231b7/go.mod h1:lwMNonHdLvfTF+YIU3yjz9huy/KSjqu+5exZ/TT7Hvc= +github.com/onflow/flow-emulator v0.53.0 h1:VIMljBL77VnO+CeeJX1N5GVmF245XwZrFGv63dLPQGk= +github.com/onflow/flow-emulator v0.53.0/go.mod h1:o7O+b3fQYs26vJ+4SeMY/T9kA1rT09tFxQccTFyM5b4= github.com/onflow/flow-ft/lib/go/contracts v0.7.0 h1:XEKE6qJUw3luhsYmIOteXP53gtxNxrwTohgxJXCYqBE= github.com/onflow/flow-ft/lib/go/contracts v0.7.0/go.mod h1:kTMFIySzEJJeupk+7EmXs0EJ6CBWY/MV9fv9iYQk+RU= github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJoIv4FaEMg74= @@ -1375,8 +1375,6 @@ github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9j github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391 h1:6uKg0gpLKpTZKMihrsFR0Gkq++1hykzfR1tQCKuOfw4= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230602212908-08fc6536d391/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e h1:RHaXPHvWCy3VM62+HTyu6DYq5T8rrK1gxxqogKuJ4S4= -github.com/onflow/fusd/lib/go/contracts v0.0.0-20211021081023-ae9de8fb2c7e/go.mod h1:CRX9eXtc9zHaRVTW1Xh4Cf5pZgKkQuu1NuSEVyHXr/0= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/nft-storefront/lib/go/contracts v0.0.0-20221222181731-14b90207cead h1:2j1Unqs76Z1b95Gu4C3Y28hzNUHBix7wL490e61SMSw= diff --git a/integration/localnet/client/Dockerfile b/integration/localnet/client/Dockerfile index ac1fbb8d8e7..4908e287624 100644 --- a/integration/localnet/client/Dockerfile +++ b/integration/localnet/client/Dockerfile @@ -1,13 +1,13 @@ -FROM golang:1.17 +FROM golang:1.20 COPY flow-localnet.json /go WORKDIR /go -RUN curl -L https://github.com/onflow/flow-cli/archive/refs/tags/v0.36.2.tar.gz | tar -xzv -RUN cd flow-cli-0.36.2 && go mod download -RUN cd flow-cli-0.36.2 && make -RUN /go/flow-cli-0.36.2/cmd/flow/flow version -RUN cp /go/flow-cli-0.36.2/cmd/flow/flow /go/flow +RUN curl -L https://github.com/onflow/flow-cli/archive/refs/tags/v1.3.3.tar.gz | tar -xzv +RUN cd flow-cli-1.3.3 && go mod download +RUN cd flow-cli-1.3.3 && make +RUN /go/flow-cli-1.3.3/cmd/flow/flow version +RUN cp /go/flow-cli-1.3.3/cmd/flow/flow /go/flow CMD /go/flow -f /go/flow-localnet.json -n observer blocks get latest From c1347be245eb6960db19ad46782441db61fb9fc3 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Thu, 20 Jul 2023 22:21:40 +0300 Subject: [PATCH 167/169] Linted --- engine/common/rpc/convert/events.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/common/rpc/convert/events.go b/engine/common/rpc/convert/events.go index c25f405355b..58ccb0ed9a1 100644 --- a/engine/common/rpc/convert/events.go +++ b/engine/common/rpc/convert/events.go @@ -8,9 +8,10 @@ import ( "github.com/onflow/cadence/encoding/ccf" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow-go/model/flow" - accessproto "github.com/onflow/flow/protobuf/go/flow/access" + accessproto "github.com/onflow/flow/protobuf/go/flow/access" "github.com/onflow/flow/protobuf/go/flow/entities" execproto "github.com/onflow/flow/protobuf/go/flow/execution" ) From 7faa9e73a655a3caafdecc379007b20ce9afa806 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 21 Jul 2023 00:52:56 +0300 Subject: [PATCH 168/169] Fixed last commit --- cmd/access/node_builder/access_node_builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index bb609cd5815..2a576945ac2 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -37,7 +37,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/ingestion" pingeng "github.com/onflow/flow-go/engine/access/ping" - "github.com/onflow/flow-go/engine/access/rest" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/access/state_stream" @@ -969,7 +969,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). Module("rest metrics", func(node *cmd.NodeConfig) error { - m, err := metrics.NewRestCollector(rest.URLToRoute, node.MetricsRegisterer) + m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) if err != nil { return err } From 26b435e151c76984cab675624f4717433f176a59 Mon Sep 17 00:00:00 2001 From: UlyanaAndrukhiv Date: Fri, 21 Jul 2023 14:42:35 +0300 Subject: [PATCH 169/169] Updated according to comments, added rest metrics to observer node --- cmd/observer/node_builder/observer_builder.go | 20 +++++- .../rest/apiproxy/rest_proxy_handler.go | 3 +- go.sum | 4 +- integration/tests/access/observer_test.go | 65 +++++++++++-------- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 8309f12dd28..b2d179aa942 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -29,6 +29,7 @@ import ( "github.com/onflow/flow-go/crypto" "github.com/onflow/flow-go/engine/access/apiproxy" restapiproxy "github.com/onflow/flow-go/engine/access/rest/apiproxy" + "github.com/onflow/flow-go/engine/access/rest/routes" "github.com/onflow/flow-go/engine/access/rpc" "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/follower" @@ -166,6 +167,9 @@ type ObserverServiceBuilder struct { // Public network peerID peer.ID + + RestMetrics *metrics.RestCollector + AccessMetrics module.AccessMetrics } // deriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters. @@ -849,8 +853,22 @@ func (builder *ObserverServiceBuilder) enqueueConnectWithStakedAN() { } func (builder *ObserverServiceBuilder) enqueueRPCServer() { + builder.Module("rest metrics", func(node *cmd.NodeConfig) error { + m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) + if err != nil { + return err + } + builder.RestMetrics = m + return nil + }) + builder.Module("access metrics", func(node *cmd.NodeConfig) error { + builder.AccessMetrics = metrics.NewAccessCollector( + metrics.WithRestMetrics(builder.RestMetrics), + ) + return nil + }) builder.Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { - accessMetrics := metrics.NewAccessCollector() + accessMetrics := builder.AccessMetrics config := builder.rpcConf backendConfig := config.BackendConfig diff --git a/engine/access/rest/apiproxy/rest_proxy_handler.go b/engine/access/rest/apiproxy/rest_proxy_handler.go index b89dc46b771..01e7b56724d 100644 --- a/engine/access/rest/apiproxy/rest_proxy_handler.go +++ b/engine/access/rest/apiproxy/rest_proxy_handler.go @@ -10,7 +10,6 @@ import ( "github.com/rs/zerolog" "github.com/onflow/flow-go/access" - "github.com/onflow/flow-go/engine/access/rest/models" "github.com/onflow/flow-go/engine/common/grpc/forwarder" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" @@ -187,7 +186,7 @@ func (r *RestProxyHandler) GetAccountAtBlockHeight(ctx context.Context, address r.log("upstream", "GetAccountAtBlockHeight", err) if err != nil { - return nil, models.NewNotFoundError("not found account at block height", err) + return nil, err } return convert.MessageToAccount(accountResponse.Account) diff --git a/go.sum b/go.sum index 14df972d74f..97ca856b74d 100644 --- a/go.sum +++ b/go.sum @@ -1254,13 +1254,13 @@ github.com/onflow/flow-go-sdk v0.24.0/go.mod h1:IoptMLPyFXWvyd9yYA6/4EmSeeozl6nJ github.com/onflow/flow-go-sdk v0.41.9 h1:cyplhhhc0RnfOAan2t7I/7C9g1hVGDDLUhWj6ZHAkk4= github.com/onflow/flow-go-sdk v0.41.9/go.mod h1:e9Q5TITCy7g08lkdQJxP8fAKBnBoC5FjALvUKr36j4I= github.com/onflow/flow-go/crypto v0.21.3/go.mod h1:vI6V4CY3R6c4JKBxdcRiR/AnjBfL8OSD97bJc60cLuQ= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= -github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/flow-go/crypto v0.24.9 h1:0EQp+kSZYJepMIiSypfJVe7tzsPcb6UXOdOtsTCDhBs= github.com/onflow/flow-go/crypto v0.24.9/go.mod h1:fqCzkIBBMRRkciVrvW21rECKq1oD7Q6u+bCI78lfNX0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0 h1:rhUDeD27jhLwOqQKI/23008CYfnqXErrJvc4EFRP2a0= github.com/onflow/flow-nft/lib/go/contracts v1.1.0/go.mod h1:YsvzYng4htDgRB9sa9jxdwoTuuhjK8WYWXTyLkIigZY= github.com/onflow/flow/protobuf/go/flow v0.2.2/go.mod h1:gQxYqCfkI8lpnKsmIjwtN2mV/N2PIwc1I+RUK4HPIc8= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce h1:YQKijiQaq8SF1ayNqp3VVcwbBGXSnuHNHq4GQmVGybE= +github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20230628215638-83439d22e0ce/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d h1:QcOAeEyF3iAUHv21LQ12sdcsr0yFrJGoGLyCAzYYtvI= github.com/onflow/go-bitswap v0.0.0-20230703214630-6d3db958c73d/go.mod h1:GCPpiyRoHncdqPj++zPr9ZOYBX4hpJ0pYZRYqSE8VKk= github.com/onflow/sdks v0.5.0 h1:2HCRibwqDaQ1c9oUApnkZtEAhWiNY2GTpRD5+ftdkN8= diff --git a/integration/tests/access/observer_test.go b/integration/tests/access/observer_test.go index 55733820d92..25bfeab2f3a 100644 --- a/integration/tests/access/observer_test.go +++ b/integration/tests/access/observer_test.go @@ -5,9 +5,7 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" - "strings" "testing" "google.golang.org/grpc" @@ -20,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/onflow/flow-go/engine/access/rest/util" "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" @@ -77,14 +76,14 @@ func (s *ObserverSuite) SetupTest() { // access node with unstaked nodes supported testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.InfoLevel), testnet.WithAdditionalFlag("--supports-observer=true")), - // need one dummy execution node (unused ghost) - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), + // need one dummy execution node + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), // need one dummy verification node (unused ghost) testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), - // need one controllable collection node (unused ghost) - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), + // need one controllable collection node + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel)), // need three consensus nodes (unused ghost) testnet.NewNodeConfig(flow.RoleConsensus, testnet.WithLogLevel(zerolog.FatalLevel), testnet.AsGhost()), @@ -184,23 +183,20 @@ func (s *ObserverSuite) TestObserverRest() { observerAddr := s.net.ContainerByName("observer_1").Addr(testnet.RESTPort) httpClient := http.DefaultClient - makeHttpCall := func(method string, url string, body io.Reader) (*http.Response, error) { + makeHttpCall := func(method string, url string, body interface{}) (*http.Response, error) { switch method { case http.MethodGet: return httpClient.Get(url) case http.MethodPost: - if body == nil { - return httpClient.Post(url, "application/json", strings.NewReader("{}")) - } else { - return httpClient.Post(url, "application/json", body) - } + jsonBody, _ := json.Marshal(body) + return httpClient.Post(url, "application/json", bytes.NewBuffer(jsonBody)) } panic("not supported") } - makeObserverCall := func(method string, path string, body io.Reader) (*http.Response, error) { + makeObserverCall := func(method string, path string, body interface{}) (*http.Response, error) { return makeHttpCall(method, "http://"+observerAddr+"/v1"+path, body) } - makeAccessCall := func(method string, path string, body io.Reader) (*http.Response, error) { + makeAccessCall := func(method string, path string, body interface{}) (*http.Response, error) { return makeHttpCall(method, "http://"+accessAddr+"/v1"+path, body) } @@ -218,6 +214,10 @@ func (s *ObserverSuite) TestObserverRest() { assert.NoError(t, observerErr) assert.Equal(t, accessResp.Status, observerResp.Status) assert.Equal(t, accessResp.StatusCode, observerResp.StatusCode) + assert.Contains(t, [...]int{ + http.StatusNotFound, + http.StatusOK, + }, observerResp.StatusCode) }) } }) @@ -236,7 +236,6 @@ func (s *ObserverSuite) TestObserverRest() { observerResp, observerErr := makeObserverCall(endpoint.method, endpoint.path, endpoint.body) require.NoError(t, observerErr) assert.Contains(t, [...]int{ - http.StatusInternalServerError, http.StatusServiceUnavailable}, observerResp.StatusCode) }) } @@ -392,12 +391,12 @@ type RestEndpointTest struct { name string method string path string - body io.Reader + body interface{} } func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { transactionId := unittest.IdentifierFixture().String() - account, _ := unittest.AccountFixture() + account := flow.Localnet.Chain().ServiceAddress().String() block := unittest.BlockFixture() executionResult := unittest.ExecutionResultFixture() collection := unittest.CollectionFixture(2) @@ -454,16 +453,17 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { name: "executeScript", method: http.MethodPost, path: "/scripts", + body: createScript(), }, { name: "getAccount", method: http.MethodGet, - path: "/accounts/" + account.Address.HexWithPrefix() + "?block_height=1", + path: "/accounts/" + account + "?block_height=1", }, { name: "getEvents", method: http.MethodGet, - path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", eventType, 1, 3), + path: fmt.Sprintf("/events?type=%s&start_height=%d&end_height=%d", eventType, 0, 3), }, { name: "getNetworkParameters", @@ -478,10 +478,15 @@ func (s *ObserverSuite) getRestEndpoints() []RestEndpointTest { } } -func createTx(net *testnet.FlowNetwork) *bytes.Buffer { +func createTx(net *testnet.FlowNetwork) interface{} { flowAddr := flow.Localnet.Chain().ServiceAddress() - signature := unittest.TransactionSignatureFixture() - signature.Address = flowAddr + payloadSignature := unittest.TransactionSignatureFixture() + envelopeSignature := unittest.TransactionSignatureFixture() + + payloadSignature.Address = flowAddr + + envelopeSignature.Address = flowAddr + envelopeSignature.KeyIndex = 2 tx := flow.NewTransactionBody(). AddAuthorizer(flowAddr). @@ -489,10 +494,18 @@ func createTx(net *testnet.FlowNetwork) *bytes.Buffer { SetScript(unittest.NoopTxScript()). SetReferenceBlockID(net.Root().ID()). SetProposalKey(flowAddr, 1, 0) - tx.PayloadSignatures = []flow.TransactionSignature{signature} - tx.EnvelopeSignatures = []flow.TransactionSignature{signature} + tx.PayloadSignatures = []flow.TransactionSignature{payloadSignature} + tx.EnvelopeSignatures = []flow.TransactionSignature{envelopeSignature} - jsonBody, _ := json.Marshal(unittest.CreateSendTxHttpPayload(*tx)) + return unittest.CreateSendTxHttpPayload(*tx) +} - return bytes.NewBuffer(jsonBody) +func createScript() interface{} { + validCode := []byte(`pub fun main(foo: String): String { return foo }`) + validArgs := []byte(`{ "type": "String", "value": "hello world" }`) + body := map[string]interface{}{ + "script": util.ToBase64(validCode), + "arguments": []string{util.ToBase64(validArgs)}, + } + return body }