diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 265f0f4b4..b2527ca39 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,447 +1,77 @@ name: CI + on: - pull_request: push: branches: - - main - develop - - rc/next + - main + tags-ignore: + - "**" + paths-ignore: + - "**/CHANGELOG.md" + pull_request: workflow_dispatch: jobs: - pre_run: - name: Cancel previous runs + build-publish: runs-on: ubuntu-latest steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@ad6cb1b847ffb509a69b745b6ee2f1d14dfe14b8 + - uses: actions/checkout@v2 with: - access_token: ${{ github.token }} + token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + fetch-depth: 0 persist-credentials: false - get_release_info: - name: Get Release Info - runs-on: ubuntu-latest - needs: pre_run - outputs: - tag: ${{ steps.new_release_tag.outputs.TAG }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Get latest release - if: startsWith(github.ref, 'refs/heads/main') - id: release - uses: pozetroninc/github-action-get-latest-release@master - with: - repository: ${{ github.repository }} - excludes: prerelease, draft - - - name: Determine if release build - if: startsWith(github.ref, 'refs/heads/main') - id: new_release_tag - env: - LATEST_RELEASE: ${{ steps.release.outputs.release }} - run: | - CARGO_VERSION=v$(grep "version" components/chainhook-cli/Cargo.toml | head -n 1 | cut -d\" -f2) - if [[ "${CARGO_VERSION}" != "${LATEST_RELEASE}" ]]; then - echo "::set-output name=TAG::${CARGO_VERSION}" - echo "::warning::Will create release for version: ${CARGO_VERSION}" - else - echo "::warning::Will not create a release" - fi - - audit: - name: Audit and format - runs-on: ubuntu-latest - needs: pre_run - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - components: rustfmt - override: true - - - name: Set Cargo file permissions - run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - - name: Cache cargo - uses: actions/cache@v2 - with: - path: ~/.cargo/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install dependencies - run: cargo install cargo-audit - - - name: Run audit - run: cargo audit --ignore RUSTSEC-2021-0076 --ignore RUSTSEC-2021-0119 --ignore RUSTSEC-2022-0028 --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2021-0124 --ignore RUSTSEC-2022-0040 --ignore RUSTSEC-2023-0018 - - - name: Run rustfmt - run: cargo fmt --all -- --check - - # test_coverage_cargo: - # name: Generate test coverage - # runs-on: ubuntu-latest - # needs: pre_run - # strategy: - # fail-fast: false - # matrix: - # include: - # - name: clarinet - # working-directory: components/clarinet-cli - # - name: clarity-repl - # working-directory: components/clarity-repl - # - name: chainhook-cli - # working-directory: components/chainhook-event-observer - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - # with: - # submodules: recursive - - # - name: Install Rust toolchain stable - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # profile: minimal - # override: true - - # - name: Cache cargo - # uses: actions/cache@v2 - # id: cache-cargo - # with: - # path: | - # ~/.cargo/bin/ - # ~/.cargo/registry/index/ - # ~/.cargo/registry/cache/ - # ~/.cargo/git/db/ - # target/ - # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - # - name: Install dependencies - # if: steps.cache-cargo.outputs.cache-hit != 'true' - # run: RUSTC_BOOTSTRAP=1 cargo install grcov - - # - name: Install Rust toolchain - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # profile: minimal - # components: llvm-tools-preview - # override: true - - # - name: Unit Tests - # env: - # RUSTFLAGS: "-C instrument-coverage" - # LLVM_PROFILE_FILE: "${{ matrix.name }}-%p-%m.profraw" - # run: cargo build --package=clarinet-cli --locked && cargo test --package=clarinet-cli - - # - name: Generate coverage - # run: grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info - - # # Run functional tests here in addition to the other jobs so we can fail fast - # # Since these tests are reached much earlier in the pipeline - # - name: Functional Tests - # if: matrix.name == 'clarinet' - # run: | - # for testdir in $(ls components/clarinet-cli/examples); do - # ./target/debug/clarinet test --manifest-path components/clarinet-cli/examples/${testdir}/Clarinet.toml - # done - - # - name: Upload coverage report - # uses: codecov/codecov-action@v1 - # with: - # flags: unittests - # name: ${{ matrix.name }} - # verbose: true - - dist_chainhook: - name: Build Chainhook Distributions - runs-on: ${{ matrix.os }} - needs: pre_run - # Related upstream issue: - # https://github.com/nagisa/rust_libloading/issues/61#issuecomment-607941377 - # - # env: - # CC: deny_c - - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - platform: linux - target: x86_64-unknown-linux-gnu - architecture: x64 - libc: glibc - - os: windows-latest - platform: windows - target: x86_64-pc-windows-msvc - architecture: x64 - - os: macos-latest - platform: darwin - target: x86_64-apple-darwin - architecture: x64 - - os: macos-latest - platform: darwin - target: aarch64-apple-darwin - architecture: arm64 - - steps: - - name: Configure git to use LF (Windows) - if: matrix.os == 'windows-latest' - run: | - git config --global core.autocrlf false - git config --global core.eol lf - - - name: Checkout repository - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - target: ${{ matrix.target }} - profile: minimal - components: llvm-tools-preview - override: true - - - name: Install wix (Windows) - if: matrix.os == 'windows-latest' - run: cargo install cargo-wix - - - if: matrix.os != 'windows-latest' - run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - - - name: Cache cargo - uses: actions/cache@v2 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/${{ matrix.target }}/release/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - # Set environment variables required from cross compiling from macos-x86_64 to macos-arm64 - - name: Configure macos-arm64 cross compile config - if: matrix.target == 'aarch64-apple-darwin' - run: | - echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV - echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV - - - name: Configure artifact names (libc) - if: ${{ matrix.libc }} - shell: bash - run: | - echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV - echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV - - - name: Configure artifact names (not libc) - if: ${{ ! matrix.libc }} - shell: bash - run: | - echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}" >> $GITHUB_ENV - echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-unknown" >> $GITHUB_ENV - - - name: Build - Cargo - if: matrix.target != 'x86_64-unknown-linux-musl' - run: cargo build --release --features=telemetry --locked --target ${{ matrix.target }} - - - name: Code sign bin (Windows) - if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest' - run: | - $certificate_file_name = "${env:TEMP}\certificate.pfx" - - $bytes_cert = [Convert]::FromBase64String('${{ secrets.WINDOWS_CODE_SIGNING_CERTIFICATE }}') - [IO.File]::WriteAllBytes(${certificate_file_name}, ${bytes_cert}) - - $signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe" - $bin_path = (Resolve-Path -Path "target/${{ matrix.target }}/release/chainhook.exe").Path - & ${signtool_path} sign ` - /d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." ` - /du "https://github.com/hirosystems/chainhook" ` - /tr http://timestamp.digicert.com ` - /td sha256 ` - /fd sha256 ` - -f "${certificate_file_name}" ` - -p "${{ secrets.WINDOWS_CODE_SIGNING_PASSWORD }}" ` - "${bin_path}" - - - name: Build Installer (Windows) - if: matrix.os == 'windows-latest' - run: cargo wix -v --no-build --nocapture -p chainhook-cli - - - name: Code sign installer (Windows) - if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest' + - name: Cargo test run: | - $certificate_file_name = "${env:TEMP}\certificate.pfx" - - $bytes_cert = [Convert]::FromBase64String('${{ secrets.WINDOWS_CODE_SIGNING_CERTIFICATE }}') - [IO.File]::WriteAllBytes(${certificate_file_name}, ${bytes_cert}) - - $signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe" - $msi_path = (Resolve-Path -Path "target/wix/*.msi").Path - & ${signtool_path} sign ` - /d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." ` - /du "https://github.com/hirosystems/chainhook" ` - /tr http://timestamp.digicert.com ` - /td sha256 ` - /fd sha256 ` - -f "${certificate_file_name}" ` - -p "${{ secrets.WINDOWS_CODE_SIGNING_PASSWORD }}" ` - "${msi_path}" - - # Don't compress for Windows because winget can't yet unzip files - - name: Compress cargo artifact (Linux) - if: matrix.os != 'windows-latest' - run: tar -C target/${{ matrix.target }}/release -zcvf chainhook-${{ env.SHORT_TARGET_NAME }}.tar.gz chainhook - - - name: Rename cargo artifact (Windows) - if: matrix.os == 'windows-latest' - shell: bash - run: mv target/wix/*.msi chainhook-${{ env.SHORT_TARGET_NAME }}.msi - - # Separate uploads to prevent paths from being preserved - - name: Upload cargo artifacts (Linux) - if: matrix.os != 'windows-latest' - uses: actions/upload-artifact@v2 - with: - name: chainhook-${{ env.SHORT_TARGET_NAME }} - path: chainhook-${{ env.SHORT_TARGET_NAME }}.tar.gz - - - name: Upload cargo artifact (Windows) - if: matrix.os == 'windows-latest' - uses: actions/upload-artifact@v2 + rustup update + cargo check + RUST_BACKTRACE=1 cargo test --all + + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v3 + id: semantic + # Only run on non-PR events or only PRs that aren't from forks + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }} + SEMANTIC_RELEASE_PACKAGE: ${{ github.event.repository.name }} with: - name: chainhook-${{ env.SHORT_TARGET_NAME }} - path: chainhook-${{ env.SHORT_TARGET_NAME }}.msi + semantic_version: 19 + extra_plugins: | + @semantic-release/changelog@6.0.3 + @semantic-release/git@10.0.1 + conventional-changelog-conventionalcommits@6.1.0 - - name: Unit Tests - Cargo - # can't easily run mac-arm64 tests in GH without native runners for that arch - if: matrix.target != 'aarch64-apple-darwin' - run: RUST_BACKTRACE=1 cargo test --release --locked --target ${{ matrix.target }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - # - name: Functional Tests (Linux) - # # can't easily run mac-arm64 tests in GH without native runners for that arch - # if: matrix.os != 'windows-latest' && matrix.target != 'aarch64-apple-darwin' - # run: | - # for testdir in $(ls components/chainhook-cli/examples); do - # ./target/${{ matrix.target }}/release/clarinet test --manifest-path components/clarinet-cli/examples/${testdir}/Clarinet.toml - # done - - # - name: Functional Tests (Windows) - # if: matrix.os == 'windows-latest' - # run: | - # foreach($testdir in Get-ChildItem components/clarinet-cli/examples) { - # ./target/${{ matrix.target }}/release/clarinet test --manifest-path ${testdir}/Clarinet.toml - # } - - docker_images: - name: Create ${{ matrix.name }} Docker Image - runs-on: ubuntu-latest - needs: - - get_release_info - - dist_chainhook_node - outputs: - version: ${{ steps.docker_meta.outputs.version }} - strategy: - fail-fast: false - matrix: - include: - - name: Chainhook Node - description: A Stacks event observer. - image: ${{ github.repository_owner }}/chainhook-cli - artifact: chainhook-cli-linux-x64-glibc - dockerfile: dockerfiles/components/chainhook-cli.dockerfile - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Generate Docker tags/labels - id: docker_meta + - name: Docker Meta + id: meta uses: docker/metadata-action@v4 with: - images: ${{ matrix.image }} + images: | + blockstack/${{ github.event.repository.name }} + hirosystems/${{ github.event.repository.name }} tags: | type=ref,event=branch type=ref,event=pr - type=semver,pattern={{version}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }} - type=semver,pattern={{major}}.{{minor}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }} - labels: | - org.opencontainers.image.title=${{ matrix.name }} - org.opencontainers.image.description=${{ matrix.description }} + type=semver,pattern={{version}},value=${{ steps.semantic.outputs.new_release_version }},enable=${{ steps.semantic.outputs.new_release_version != '' }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.semantic.outputs.new_release_version }},enable=${{ steps.semantic.outputs.new_release_version != '' }} + type=raw,value=latest,enable={{is_default_branch}} - - name: Login to Dockerhub + - name: Login to DockerHub uses: docker/login-action@v2 - if: github.event_name != 'pull_request' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Download pre-built dist - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.artifact }} - - - name: Untar pre-built dist - run: tar zxvf *.tar.gz - - - name: Create Image - uses: docker/build-push-action@v3 + - name: Build/Tag/Push Image + uses: docker/build-push-action@v2 with: context: . - file: ${{ matrix.dockerfile }} - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - - release: - name: Release - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/heads/main') && needs.get_release_info.outputs.tag != '' - needs: - - get_release_info - - audit - # - test_coverage_cargo - - docker_images - permissions: - actions: write - contents: write - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Download pre-built dists - uses: actions/download-artifact@v3 - - - name: Tag and Release - uses: ncipollo/release-action@v1 - with: - artifacts: "**/*.tar.gz,**/*.msi" - tag: ${{ needs.get_release_info.outputs.tag }} - commit: ${{ env.GITHUB_SHA }} - - - name: Trigger pkg-version-bump workflow - uses: peter-evans/repository-dispatch@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - event-type: released - client-payload: '{"tag": "${{ needs.get_release_info.outputs.tag }}"}' + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + file: ./dockerfiles/components/chainhook.dockerfile + # Only push if (there's a new release on main branch, or if building a non-main branch) and (Only run on non-PR events or only PRs that aren't from forks) + push: ${{ (github.ref != 'refs/heads/master' || steps.semantic.outputs.new_release_version != '') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} diff --git a/.gitignore b/.gitignore index 9aa2f587e..9f6f049ca 100644 --- a/.gitignore +++ b/.gitignore @@ -12,18 +12,11 @@ npm-debug.log* **/.requirements **/.cache **/.build -components/stacks-devnet-js/dist -components/stacks-devnet-js/build components/chainhook-types-js/dist *.tar.gz *.zip *.rdb - -components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/bin -components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/tmp -components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/vendor -components/chainhook-cli/cache -components/chainhook-cli/index.redb +*.redb cache/ components/chainhook-cli/src/service/tests/fixtures/tmp \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 8f87fc1ad..764fe81b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,15 +8,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" -dependencies = [ - "const-random", -] - [[package]] name = "ahash" version = "0.7.6" @@ -114,19 +105,6 @@ dependencies = [ "syn 1.0.105", ] -[[package]] -name = "asynchronous-codec" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182" -dependencies = [ - "bytes", - "futures-sink", - "futures-util", - "memchr", - "pin-project-lite", -] - [[package]] name = "atomic" version = "0.5.1" @@ -403,10 +381,14 @@ dependencies = [ ] [[package]] -name = "cfg-if" -version = "0.1.10" +name = "cfg-expr" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "b40ccee03b5175c18cde8f37e7d2a33bcef6f8ec8f7cc0d81090d1bb380949c9" +dependencies = [ + "smallvec 1.10.0", + "target-lexicon", +] [[package]] name = "cfg-if" @@ -427,7 +409,7 @@ dependencies = [ "clarity-repl", "clarity-vm", "criterion", - "crossbeam-channel 0.5.8", + "crossbeam-channel", "csv", "ctrlc", "flate2", @@ -451,22 +433,22 @@ dependencies = [ "test-case", "threadpool", "tokio", - "toml", - "uuid 1.3.0", + "toml 0.5.9", + "uuid", ] [[package]] name = "chainhook-sdk" -version = "0.8.2" +version = "0.9.0" dependencies = [ "base58 0.2.0", "base64", "bitcoincore-rpc", "bitcoincore-rpc-json", - "chainhook-types 1.0.11", + "chainhook-types 1.0.12", "clarinet-utils", - "crossbeam-channel 0.5.8", - "dashmap 5.4.0", + "crossbeam-channel", + "dashmap", "futures", "fxhash", "hex", @@ -486,7 +468,7 @@ dependencies = [ "test-case", "threadpool", "tokio", - "zeromq", + "zmq", ] [[package]] @@ -505,7 +487,7 @@ dependencies = [ [[package]] name = "chainhook-types" -version = "1.0.11" +version = "1.0.12" dependencies = [ "hex", "schemars 0.8.12", @@ -562,7 +544,7 @@ dependencies = [ "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.2", "once_cell", "strsim", "termcolor", @@ -625,7 +607,7 @@ dependencies = [ "serde", "serde_derive", "tiny-hderive", - "toml", + "toml 0.5.9", "url", ] @@ -673,7 +655,7 @@ dependencies = [ "sha2 0.10.6", "sha3 0.9.1", "tokio", - "tokio-util 0.7.2", + "tokio-util", ] [[package]] @@ -740,28 +722,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "const-random" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" -dependencies = [ - "const-random-macro", - "proc-macro-hack", -] - -[[package]] -name = "const-random-macro" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" -dependencies = [ - "getrandom 0.2.8", - "once_cell", - "proc-macro-hack", - "tiny-keccak", -] - [[package]] name = "const_fn" version = "0.4.9" @@ -810,7 +770,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -851,26 +811,16 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.7.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-channel 0.4.4", - "crossbeam-deque 0.7.4", - "crossbeam-epoch 0.8.2", + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", "crossbeam-queue", - "crossbeam-utils 0.7.2", -] - -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -879,19 +829,8 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.14", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "cfg-if", + "crossbeam-utils", ] [[package]] @@ -900,24 +839,9 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch 0.9.13", - "crossbeam-utils 0.8.14", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -927,32 +851,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ "autocfg", - "cfg-if 1.0.0", - "crossbeam-utils 0.8.14", + "cfg-if", + "crossbeam-utils", "memoffset 0.7.1", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "cfg-if", + "crossbeam-utils", ] [[package]] @@ -961,7 +873,7 @@ version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1124,28 +1036,17 @@ dependencies = [ "syn 1.0.105", ] -[[package]] -name = "dashmap" -version = "3.11.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" -dependencies = [ - "ahash 0.3.8", - "cfg-if 0.1.10", - "num_cpus", -] - [[package]] name = "dashmap" version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core 0.9.5", + "parking_lot_core", ] [[package]] @@ -1221,13 +1122,24 @@ dependencies = [ "subtle 2.4.1", ] +[[package]] +name = "dircpy" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8466f8d28ca6da4c9dfbbef6ad4bff6f2fdd5e412d821025b0d3f0a9d74a8c1e" +dependencies = [ + "jwalk", + "log", + "walkdir", +] + [[package]] name = "dirs-next" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -1285,7 +1197,7 @@ version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1295,15 +1207,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] -name = "enum-primitive-derive" -version = "0.2.2" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c375b9c5eadb68d0a6efee2999fef292f45854c3444c86f09d8ab086ba942b0e" -dependencies = [ - "num-traits", - "quote", - "syn 1.0.105", -] +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -1380,7 +1287,7 @@ version = "3.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "rustix 0.36.5", "windows-sys 0.42.0", ] @@ -1394,7 +1301,7 @@ dependencies = [ "atomic", "pear", "serde", - "toml", + "toml 0.5.9", "uncased", "version_check", ] @@ -1405,7 +1312,7 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall", "windows-sys 0.42.0", @@ -1600,7 +1507,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1611,7 +1518,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -1636,10 +1543,10 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.2", "slab", "tokio", - "tokio-util 0.7.2", + "tokio-util", "tracing", ] @@ -1655,7 +1562,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.7.6", + "ahash", ] [[package]] @@ -1664,6 +1571,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hashlink" version = "0.7.0" @@ -1914,6 +1827,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1926,7 +1849,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2012,6 +1935,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56" +dependencies = [ + "crossbeam", + "rayon", +] + [[package]] name = "keccak" version = "0.1.3" @@ -2045,7 +1978,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "winapi", ] @@ -2187,7 +2120,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2196,7 +2129,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "generator", "scoped-tls", "serde", @@ -2236,15 +2169,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.5" @@ -2318,7 +2242,7 @@ dependencies = [ "mime", "spin 0.9.4", "tokio", - "tokio-util 0.7.2", + "tokio-util", "version_check", ] @@ -2366,7 +2290,7 @@ checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ "bitflags 1.3.2", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "memoffset 0.6.5", ] @@ -2379,7 +2303,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "libc", ] @@ -2479,7 +2403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -2529,17 +2453,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -2547,21 +2460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.5", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if 1.0.0", - "instant", - "libc", - "redox_syscall", - "smallvec 1.10.0", - "winapi", + "parking_lot_core", ] [[package]] @@ -2570,7 +2469,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall", "smallvec 1.10.0", @@ -2923,9 +2822,9 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ - "crossbeam-channel 0.5.8", - "crossbeam-deque 0.8.2", - "crossbeam-utils 0.8.14", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", "num_cpus", ] @@ -3070,7 +2969,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls", - "tokio-util 0.7.2", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -3118,13 +3017,13 @@ dependencies = [ "either", "figment", "futures", - "indexmap", + "indexmap 1.9.2", "is-terminal", "log", "memchr", "multer", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "rand 0.8.5", "ref-cast", @@ -3137,7 +3036,7 @@ dependencies = [ "time 0.3.17", "tokio", "tokio-stream", - "tokio-util 0.7.2", + "tokio-util", "ubyte", "version_check", "yansi", @@ -3151,7 +3050,7 @@ checksum = "7093353f14228c744982e409259fb54878ba9563d08214f2d880d59ff2fc508b" dependencies = [ "devise", "glob", - "indexmap", + "indexmap 1.9.2", "proc-macro2", "quote", "rocket_http", @@ -3170,7 +3069,7 @@ dependencies = [ "futures", "http", "hyper", - "indexmap", + "indexmap 1.9.2", "log", "memchr", "pear", @@ -3229,7 +3128,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2288c66aeafe3b2ed227c981f364f9968fa952ef0b30e84ada4486e7ee24d00a" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "proc-macro2", "quote", "rustc_version 0.4.0", @@ -3358,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "clipboard-win", "dirs-next", "fd-lock", @@ -3418,11 +3317,11 @@ version = "0.8.12" source = "git+https://github.com/hirosystems/schemars.git?branch=feat-chainhook-fixes#15fdd4711700114d57c090aad62516593bd4ca6d" dependencies = [ "dyn-clone", - "indexmap", + "indexmap 1.9.2", "schemars_derive 0.8.12", "serde", "serde_json", - "uuid 1.3.0", + "uuid", ] [[package]] @@ -3631,6 +3530,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_stacker" version = "0.1.6" @@ -3659,11 +3567,11 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ - "dashmap 5.4.0", + "dashmap", "futures", "lazy_static", "log", - "parking_lot 0.12.1", + "parking_lot", "serial_test_derive", ] @@ -3712,7 +3620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.9.0", "opaque-debug 0.3.0", @@ -3724,7 +3632,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.6", ] @@ -3805,7 +3713,7 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" dependencies = [ - "crossbeam-channel 0.5.8", + "crossbeam-channel", "slog", "take_mut", "thread_local", @@ -3913,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "psm", "winapi", @@ -4099,6 +4007,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-deps" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30c2de8a4d8f4b823d634affc9cd2a74ec98c53a756f317e529a48046cbf71f3" +dependencies = [ + "cfg-expr", + "heck 0.4.0", + "pkg-config", + "toml 0.7.6", + "version-compare", +] + [[package]] name = "take_mut" version = "0.2.2" @@ -4116,13 +4037,19 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-lexicon" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" + [[package]] name = "tempfile" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "libc", "redox_syscall", @@ -4165,7 +4092,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d10394d5d1e27794f772b6fc854c7e91a2dc26e2cbf807ad523370c2a59c0cee" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "proc-macro-error", "proc-macro2", "quote", @@ -4329,15 +4256,6 @@ dependencies = [ "sha2 0.8.2", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -4375,7 +4293,7 @@ dependencies = [ "memchr", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -4426,21 +4344,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.2" @@ -4461,10 +4364,44 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ - "indexmap", + "indexmap 1.9.2", "serde", ] +[[package]] +name = "toml" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -4477,7 +4414,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4633,15 +4570,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.8", -] - [[package]] name = "uuid" version = "1.3.0" @@ -4664,6 +4592,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" + [[package]] name = "version_check" version = "0.9.4" @@ -4715,7 +4649,7 @@ version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -4740,7 +4674,7 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -5044,6 +4978,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" @@ -5075,27 +5018,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" [[package]] -name = "zeromq" -version = "0.3.3" +name = "zeromq-src" +version = "0.2.6+4.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "667ece59294ccaf617fcf2e5decc9114a06427c1f68990028b9f12d322686bdc" +checksum = "fc120b771270365d5ed0dfb4baf1005f2243ae1ae83703265cb3504070f4160b" dependencies = [ - "async-trait", - "asynchronous-codec", - "bytes", - "crossbeam", - "dashmap 3.11.10", - "enum-primitive-derive", - "futures", - "futures-util", - "lazy_static", - "log", - "num-traits", - "parking_lot 0.11.2", - "rand 0.7.3", - "regex", - "thiserror", - "tokio", - "tokio-util 0.6.10", - "uuid 0.8.2", + "cc", + "dircpy", +] + +[[package]] +name = "zmq" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd3091dd571fb84a9b3e5e5c6a807d186c411c812c8618786c3c30e5349234e7" +dependencies = [ + "bitflags 1.3.2", + "libc", + "zmq-sys", +] + +[[package]] +name = "zmq-sys" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8351dc72494b4d7f5652a681c33634063bbad58046c1689e75270908fdc864" +dependencies = [ + "libc", + "system-deps", + "zeromq-src", ] diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml index 77fb2c55d..596c9ec98 100644 --- a/components/chainhook-cli/Cargo.toml +++ b/components/chainhook-cli/Cargo.toml @@ -15,7 +15,7 @@ redis = "0.21.5" serde-redis = "0.12.0" hex = "0.4.3" rand = "0.8.5" -chainhook-sdk = { version = "0.8.2", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" } +chainhook-sdk = { version = "0.9.0", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" } clarinet-files = "1.0.1" hiro-system-kit = "0.1.0" # clarinet-files = { path = "../../../clarinet/components/clarinet-files" } diff --git a/components/chainhook-cli/src/cli/mod.rs b/components/chainhook-cli/src/cli/mod.rs index ef52d5a47..a07a7aebc 100644 --- a/components/chainhook-cli/src/cli/mod.rs +++ b/components/chainhook-cli/src/cli/mod.rs @@ -273,14 +273,14 @@ pub fn main() { let opts: Opts = match Opts::try_parse() { Ok(opts) => opts, Err(e) => { - println!("{}", e); + error!(ctx.expect_logger(), "{e}"); process::exit(1); } }; - match hiro_system_kit::nestable_block_on(handle_command(opts, ctx)) { + match hiro_system_kit::nestable_block_on(handle_command(opts, ctx.clone())) { Err(e) => { - println!("{e}"); + error!(ctx.expect_logger(), "{e}"); process::exit(1); } Ok(_) => {} diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs index 039ff97fb..28eecfa3d 100644 --- a/components/chainhook-cli/src/config/mod.rs +++ b/components/chainhook-cli/src/config/mod.rs @@ -109,7 +109,6 @@ impl Config { pub fn get_event_observer_config(&self) -> EventObserverConfig { EventObserverConfig { bitcoin_rpc_proxy_enabled: true, - event_handlers: vec![], chainhook_config: None, ingestion_port: DEFAULT_INGESTION_PORT, bitcoind_rpc_username: self.network.bitcoind_rpc_username.clone(), @@ -120,6 +119,7 @@ impl Config { cache_path: self.storage.working_dir.clone(), bitcoin_network: self.network.bitcoin_network.clone(), stacks_network: self.network.stacks_network.clone(), + data_handler_tx: None, } } diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml index 923946362..96dd054ed 100644 --- a/components/chainhook-sdk/Cargo.toml +++ b/components/chainhook-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "chainhook-sdk" -version = "0.8.2" +version = "0.9.0" description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin" license = "GPL-3.0" edition = "2021" @@ -18,7 +18,7 @@ hiro-system-kit = "0.1.0" # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" } # clarinet-utils = { version = "1", path = "../../../clarinet/components/clarinet-utils" } # hiro-system-kit = { version = "0.1.0", path = "../../../clarinet/components/hiro-system-kit" } -chainhook-types = { version = "1.0.11", path = "../chainhook-types-rs" } +chainhook-types = { version = "1.0.12", path = "../chainhook-types-rs" } rocket = { version = "=0.5.0-rc.3", features = ["json"] } bitcoincore-rpc = "0.16.0" bitcoincore-rpc-json = "0.16.0" @@ -37,7 +37,7 @@ hyper = { version = "0.14.24", features = ["http1", "client"] } hex = "0.4.3" threadpool = "1.8.1" rand = "0.8.5" -zeromq = { version = "0.3.3", default-features = false, features = ["tokio-runtime", "tcp-transport"], optional = true } +zmq = { version = "0.10.0", optional = true } dashmap = "5.4.0" fxhash = "0.2.1" lazy_static = "1.4.0" @@ -51,5 +51,5 @@ test-case = "3.1.0" [features] default = ["log"] -zeromq = ["dep:zeromq"] +zeromq = ["zmq"] log = ["hiro-system-kit/log"] diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs index 3b6a3be7a..f6ce4a596 100644 --- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs @@ -24,12 +24,12 @@ pub struct BitcoinTriggerChainhook<'a> { pub rollback: Vec<(Vec<&'a BitcoinTransactionData>, &'a BitcoinBlockData)>, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize)] pub struct BitcoinApplyTransactionPayload { pub block: BitcoinBlockData, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize)] pub struct BitcoinRollbackTransactionPayload { pub block: BitcoinBlockData, } diff --git a/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs b/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs deleted file mode 100644 index 2c0c8e130..000000000 --- a/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs +++ /dev/null @@ -1,366 +0,0 @@ -use crate::{ - indexer::{ChainSegment, ChainSegmentIncompatibility}, - utils::Context, -}; -use chainhook_types::{ - BitcoinBlockData, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData, - BitcoinChainUpdatedWithReorgData, BlockIdentifier, -}; -use hiro_system_kit::slog; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; - -pub struct BitcoinBlockPool { - canonical_fork_id: usize, - orphans: BTreeSet, - block_store: HashMap, - forks: BTreeMap, -} - -impl BitcoinBlockPool { - pub fn new() -> BitcoinBlockPool { - let mut forks = BTreeMap::new(); - forks.insert(0, ChainSegment::new()); - BitcoinBlockPool { - canonical_fork_id: 0, - block_store: HashMap::new(), - orphans: BTreeSet::new(), - forks, - } - } - - pub fn process_block( - &mut self, - block: BitcoinBlockData, - ctx: &Context, - ) -> Result, String> { - ctx.try_log(|logger| { - slog::info!( - logger, - "Start processing Bitcoin {}", - block.block_identifier - ) - }); - - // Keep block data in memory - let existing_entry = self - .block_store - .insert(block.block_identifier.clone(), block.clone()); - if existing_entry.is_some() { - ctx.try_log(|logger| { - slog::warn!( - logger, - "Bitcoin {} has already been processed", - block.block_identifier - ) - }); - return Ok(None); - } - - for (i, fork) in self.forks.iter() { - ctx.try_log(|logger| slog::info!(logger, "Active fork {}: {}", i, fork)); - } - // Retrieve previous canonical fork - let previous_canonical_fork_id = self.canonical_fork_id; - let previous_canonical_fork = match self.forks.get(&previous_canonical_fork_id) { - Some(fork) => fork.clone(), - None => { - ctx.try_log(|logger| { - slog::error!(logger, "unable to retrieve previous bitcoin fork") - }); - return Ok(None); - } - }; - - let mut fork_updated = None; - for (_, fork) in self.forks.iter_mut() { - let (block_appended, mut new_fork) = fork.try_append_block(&block, ctx); - if block_appended { - if let Some(new_fork) = new_fork.take() { - let fork_id = self.forks.len(); - self.forks.insert(fork_id, new_fork); - fork_updated = self.forks.get_mut(&fork_id); - } else { - fork_updated = Some(fork); - } - // A block can only be added to one segment - break; - } - } - - let fork_updated = match fork_updated.take() { - Some(fork) => { - ctx.try_log(|logger| { - slog::debug!( - logger, - "Bitcoin {} successfully appended to {}", - block.block_identifier, - fork - ) - }); - fork - } - None => { - ctx.try_log(|logger| { - slog::debug!( - logger, - "Unable to process Bitcoin {} - inboxed for later", - block.block_identifier - ) - }); - self.orphans.insert(block.block_identifier.clone()); - return Ok(None); - } - }; - - // Process former orphans - let orphans = self.orphans.clone(); - let mut orphans_to_untrack = HashSet::new(); - - let mut at_least_one_orphan_appended = true; - // As long as we are successful appending blocks that were previously unprocessable, - // Keep looping on this backlog - let mut applied = HashSet::new(); - let mut forks_created = vec![]; - while at_least_one_orphan_appended { - at_least_one_orphan_appended = false; - for orphan_block_identifier in orphans.iter() { - if applied.contains(orphan_block_identifier) { - continue; - } - let block = match self.block_store.get(orphan_block_identifier) { - Some(block) => block.clone(), - None => continue, - }; - - let (orphan_appended, mut new_fork) = fork_updated.try_append_block(&block, ctx); - if orphan_appended { - applied.insert(orphan_block_identifier); - orphans_to_untrack.insert(orphan_block_identifier); - if let Some(new_fork) = new_fork.take() { - forks_created.push(new_fork); - } - } - at_least_one_orphan_appended = at_least_one_orphan_appended || orphan_appended; - } - } - - // Update orphans - for orphan in orphans_to_untrack.into_iter() { - ctx.try_log(|logger| slog::info!(logger, "Dequeuing orphan {}", orphan)); - self.orphans.remove(orphan); - } - - // Select canonical fork - let mut canonical_fork_id = 0; - let mut highest_height = 0; - for (fork_id, fork) in self.forks.iter() { - ctx.try_log(|logger| slog::info!(logger, "Active fork: {} - {}", fork_id, fork)); - if fork.get_length() >= highest_height { - highest_height = fork.get_length(); - canonical_fork_id = *fork_id; - } - } - ctx.try_log(|logger| { - slog::info!( - logger, - "Active fork selected as canonical: {}", - canonical_fork_id - ) - }); - - self.canonical_fork_id = canonical_fork_id; - // Generate chain event from the previous and current canonical forks - let canonical_fork = self.forks.get(&canonical_fork_id).unwrap().clone(); - if canonical_fork.eq(&previous_canonical_fork) { - ctx.try_log(|logger| slog::info!(logger, "Canonical fork unchanged")); - return Ok(None); - } - - let res = self.generate_block_chain_event(&canonical_fork, &previous_canonical_fork, ctx); - let mut chain_event = match res { - Ok(chain_event) => chain_event, - Err(ChainSegmentIncompatibility::ParentBlockUnknown) => { - self.canonical_fork_id = previous_canonical_fork_id; - return Ok(None); - } - _ => return Ok(None), - }; - - self.collect_and_prune_confirmed_blocks(&mut chain_event, ctx); - - Ok(Some(chain_event)) - } - - pub fn collect_and_prune_confirmed_blocks( - &mut self, - chain_event: &mut BitcoinChainEvent, - ctx: &Context, - ) { - let (tip, confirmed_blocks) = match chain_event { - BitcoinChainEvent::ChainUpdatedWithBlocks(ref mut event) => { - match event.new_blocks.last() { - Some(tip) => (tip.block_identifier.clone(), &mut event.confirmed_blocks), - None => return, - } - } - BitcoinChainEvent::ChainUpdatedWithReorg(ref mut event) => { - match event.blocks_to_apply.last() { - Some(tip) => (tip.block_identifier.clone(), &mut event.confirmed_blocks), - None => return, - } - } - }; - - let mut forks_to_prune = vec![]; - let mut ancestor_identifier = &tip; - - // Retrieve the whole canonical segment present in memory, ascending order - // [1] ... [6] [7] - let canonical_segment = { - let mut segment = vec![]; - while let Some(ancestor) = self.block_store.get(&ancestor_identifier) { - ancestor_identifier = &ancestor.parent_block_identifier; - segment.push(ancestor.block_identifier.clone()); - } - segment - }; - if canonical_segment.len() < 7 { - return; - } - // Any block beyond 6th ancestor is considered as confirmed and can be pruned - let cut_off = &canonical_segment[5]; - - // Prune forks using the confirmed block - let mut blocks_to_prune = vec![]; - for (fork_id, fork) in self.forks.iter_mut() { - let mut res = fork.prune_confirmed_blocks(&cut_off); - blocks_to_prune.append(&mut res); - if fork.block_ids.is_empty() { - forks_to_prune.push(*fork_id); - } - } - - // Prune orphans using the confirmed block - let iter = self.orphans.clone().into_iter(); - for orphan in iter { - if orphan.index < cut_off.index { - self.orphans.remove(&orphan); - blocks_to_prune.push(orphan); - } - } - - for confirmed_block in canonical_segment[6..].into_iter() { - let block = match self.block_store.remove(confirmed_block) { - None => { - ctx.try_log(|logger| { - slog::error!(logger, "unable to retrieve data for {}", confirmed_block) - }); - return; - } - Some(block) => block, - }; - confirmed_blocks.push(block); - } - - // Prune data - for block_to_prune in blocks_to_prune { - self.block_store.remove(&block_to_prune); - } - for fork_id in forks_to_prune { - self.forks.remove(&fork_id); - } - confirmed_blocks.reverse(); - } - - pub fn generate_block_chain_event( - &mut self, - canonical_segment: &ChainSegment, - other_segment: &ChainSegment, - ctx: &Context, - ) -> Result { - if other_segment.is_empty() { - let mut new_blocks = vec![]; - for i in 0..canonical_segment.block_ids.len() { - let block_identifier = - &canonical_segment.block_ids[canonical_segment.block_ids.len() - 1 - i]; - let block = match self.block_store.get(block_identifier) { - Some(block) => block.clone(), - None => { - ctx.try_log(|logger| { - slog::error!( - logger, - "unable to retrive Bitcoin {} from block store", - block_identifier - ) - }); - return Err(ChainSegmentIncompatibility::Unknown); - } - }; - new_blocks.push(block) - } - return Ok(BitcoinChainEvent::ChainUpdatedWithBlocks( - BitcoinChainUpdatedWithBlocksData { - new_blocks, - confirmed_blocks: vec![], - }, - )); - } - if let Ok(divergence) = canonical_segment.try_identify_divergence(other_segment, false, ctx) - { - if divergence.block_ids_to_rollback.is_empty() { - let mut new_blocks = vec![]; - for i in 0..divergence.block_ids_to_apply.len() { - let block_identifier = &divergence.block_ids_to_apply[i]; - let block = match self.block_store.get(block_identifier) { - Some(block) => block.clone(), - None => panic!("unable to retrive block from block store"), - }; - new_blocks.push(block) - } - return Ok(BitcoinChainEvent::ChainUpdatedWithBlocks( - BitcoinChainUpdatedWithBlocksData { - new_blocks, - confirmed_blocks: vec![], - }, - )); - } else { - return Ok(BitcoinChainEvent::ChainUpdatedWithReorg( - BitcoinChainUpdatedWithReorgData { - blocks_to_rollback: divergence - .block_ids_to_rollback - .iter() - .map(|block_id| { - let block = match self.block_store.get(block_id) { - Some(block) => block.clone(), - None => panic!("unable to retrive block from block store"), - }; - block - }) - .collect::>(), - blocks_to_apply: divergence - .block_ids_to_apply - .iter() - .map(|block_id| { - let block = match self.block_store.get(block_id) { - Some(block) => block.clone(), - None => panic!("unable to retrive block from block store"), - }; - block - }) - .collect::>(), - confirmed_blocks: vec![], - }, - )); - } - } - ctx.try_log(|logger| { - slog::debug!( - logger, - "Unable to infer chain event out of {} and {}", - canonical_segment, - other_segment - ) - }); - Err(ChainSegmentIncompatibility::ParentBlockUnknown) - } -} diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs index a6014f4a8..3bdf00718 100644 --- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs +++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs @@ -1,5 +1,3 @@ -mod blocks_pool; - use std::time::Duration; use crate::chainhooks::types::{ @@ -11,7 +9,6 @@ use crate::utils::Context; use bitcoincore_rpc::bitcoin::hashes::Hash; use bitcoincore_rpc::bitcoin::{self, Address, Amount, BlockHash}; use bitcoincore_rpc_json::GetRawTransactionResultVoutScriptPubKey; -pub use blocks_pool::BitcoinBlockPool; use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut}; use chainhook_types::{ BitcoinBlockData, BitcoinBlockMetadata, BitcoinNetwork, BitcoinTransactionData, diff --git a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs index 3f47941c5..ca32bba7c 100644 --- a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs +++ b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs @@ -29,6 +29,15 @@ impl ForkScratchPad { } } + pub fn can_process_header(&self, header: &BlockHeader) -> bool { + if self.headers_store.is_empty() { + return true; + } + + self.headers_store + .contains_key(&header.parent_block_identifier) + } + pub fn process_header( &mut self, header: BlockHeader, diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs index f43f12ebd..5d080a130 100644 --- a/components/chainhook-sdk/src/observer/mod.rs +++ b/components/chainhook-sdk/src/observer/mod.rs @@ -1,4 +1,6 @@ mod http; +#[cfg(feature = "zeromq")] +mod zmq; use crate::chainhooks::bitcoin::{ evaluate_bitcoin_chainhooks_on_chain_event, handle_bitcoin_hook_action, @@ -28,7 +30,6 @@ use chainhook_types::{ }; use hiro_system_kit; use hiro_system_kit::slog; -use reqwest::Client as HttpClient; use rocket::config::{self, Config, LogLevel}; use rocket::data::{Limits, ToByteUnit}; use rocket::serde::Deserialize; @@ -41,9 +42,7 @@ use std::str; use std::str::FromStr; use std::sync::mpsc::{Receiver, Sender}; use std::sync::{Arc, Mutex, RwLock}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -#[cfg(feature = "zeromq")] -use zeromq::{Socket, SocketRecv}; +use std::time::{SystemTime, UNIX_EPOCH}; pub const DEFAULT_INGESTION_PORT: u16 = 20445; @@ -61,64 +60,15 @@ pub enum Event { StacksChainEvent(StacksChainEvent), } -// TODO(lgalabru): Support for GRPC? -#[derive(Deserialize, Debug, Clone)] -pub enum EventHandler { - WebHook(String), -} - -impl EventHandler { - async fn propagate_stacks_event(&self, stacks_event: &StacksChainEvent) { - match self { - EventHandler::WebHook(host) => { - let path = "chain-events/stacks"; - let url = format!("{}/{}", host, path); - let body = rocket::serde::json::serde_json::to_vec(&stacks_event).unwrap_or(vec![]); - let http_client = HttpClient::builder() - .timeout(Duration::from_secs(20)) - .build() - .expect("Unable to build http client"); - let _ = http_client - .post(url) - .header("Content-Type", "application/json") - .body(body) - .send() - .await; - // TODO(lgalabru): handle response errors - } - } - } - - async fn propagate_bitcoin_event(&self, bitcoin_event: &BitcoinChainEvent) { - match self { - EventHandler::WebHook(host) => { - let path = "chain-events/bitcoin"; - let url = format!("{}/{}", host, path); - let body = - rocket::serde::json::serde_json::to_vec(&bitcoin_event).unwrap_or(vec![]); - let http_client = HttpClient::builder() - .timeout(Duration::from_secs(20)) - .build() - .expect("Unable to build http client"); - let _res = http_client - .post(url) - .header("Content-Type", "application/json") - .body(body) - .send() - .await; - // TODO(lgalabru): handle response errors - } - } - } - - async fn notify_bitcoin_transaction_proxied(&self) {} +pub enum DataHandlerEvent { + Process(BitcoinChainhookOccurrencePayload), + Terminate, } #[derive(Debug, Clone)] pub struct EventObserverConfig { pub chainhook_config: Option, pub bitcoin_rpc_proxy_enabled: bool, - pub event_handlers: Vec, pub ingestion_port: u16, pub bitcoind_rpc_username: String, pub bitcoind_rpc_password: String, @@ -128,6 +78,7 @@ pub struct EventObserverConfig { pub cache_path: String, pub bitcoin_network: BitcoinNetwork, pub stacks_network: StacksNetwork, + pub data_handler_tx: Option>, } #[derive(Deserialize, Debug, Clone)] @@ -162,6 +113,23 @@ impl EventObserverConfig { bitcoin_config } + pub fn get_chainhook_store(&self) -> ChainhookStore { + let mut chainhook_store = ChainhookStore::new(); + // If authorization not required, we create a default ChainhookConfig + if let Some(ref chainhook_config) = self.chainhook_config { + let mut chainhook_config = chainhook_config.clone(); + chainhook_store + .predicates + .stacks_chainhooks + .append(&mut chainhook_config.stacks_chainhooks); + chainhook_store + .predicates + .bitcoin_chainhooks + .append(&mut chainhook_config.bitcoin_chainhooks); + } + chainhook_store + } + pub fn get_stacks_node_config(&self) -> &StacksNodeConfig { match self.bitcoin_block_signaling { BitcoinBlockSignaling::Stacks(ref config) => config, @@ -188,7 +156,6 @@ impl EventObserverConfig { let config = EventObserverConfig { bitcoin_rpc_proxy_enabled: false, - event_handlers: vec![], chainhook_config: None, ingestion_port: overrides .and_then(|c| c.ingestion_port) @@ -226,6 +193,7 @@ impl EventObserverConfig { .unwrap_or("cache".to_string()), bitcoin_network, stacks_network, + data_handler_tx: None, }; Ok(config) } @@ -415,12 +383,81 @@ pub struct ObserverMetrics { pub stacks: ChainMetrics, } +#[derive(Debug, Clone)] +pub struct BitcoinBlockDataCached { + pub block: BitcoinBlockData, + pub processed_by_sidecar: bool, +} + +pub struct ObserverSidecar { + pub bitcoin_blocks_mutator: Option<( + crossbeam_channel::Sender<(Vec, Vec)>, + crossbeam_channel::Receiver>, + )>, + pub bitcoin_chain_event_notifier: Option>, +} + +impl ObserverSidecar { + fn perform_bitcoin_sidecar_mutations( + &self, + blocks: Vec, + blocks_ids_to_rollback: Vec, + ctx: &Context, + ) -> Vec { + if let Some(ref block_mutator) = self.bitcoin_blocks_mutator { + ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",)); + let _ = block_mutator + .0 + .send((blocks.clone(), blocks_ids_to_rollback)); + ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",)); + match block_mutator.1.recv() { + Ok(updated_blocks) => { + ctx.try_log(|logger| slog::info!(logger, "Block received from pre-processor",)); + updated_blocks + } + Err(e) => { + ctx.try_log(|logger| { + slog::error!( + logger, + "Unable to receive block from pre-processor {}", + e.to_string() + ) + }); + blocks + } + } + } else { + blocks + } + } + + fn notify_chain_event(&self, chain_event: &BitcoinChainEvent, _ctx: &Context) { + if let Some(ref notifier) = self.bitcoin_chain_event_notifier { + match chain_event { + BitcoinChainEvent::ChainUpdatedWithBlocks(data) => { + for block in data.new_blocks.iter() { + let _ = notifier.send(HandleBlock::ApplyBlock(block.clone())); + } + } + BitcoinChainEvent::ChainUpdatedWithReorg(data) => { + for block in data.blocks_to_rollback.iter() { + let _ = notifier.send(HandleBlock::UndoBlock(block.clone())); + } + for block in data.blocks_to_apply.iter() { + let _ = notifier.send(HandleBlock::ApplyBlock(block.clone())); + } + } + } + } + } +} + pub fn start_event_observer( config: EventObserverConfig, observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, - block_pre_processor: Option<(Sender, Receiver>)>, + observer_sidecar: Option, ctx: Context, ) -> Result<(), Box> { match config.bitcoin_block_signaling { @@ -437,7 +474,7 @@ pub fn start_event_observer( observer_commands_tx_moved, observer_commands_rx, observer_events_tx, - block_pre_processor, + observer_sidecar, context_cloned, ); let _ = hiro_system_kit::nestable_block_on(future); @@ -454,7 +491,7 @@ pub fn start_event_observer( observer_commands_tx_moved, observer_commands_rx, observer_events_tx, - block_pre_processor, + observer_sidecar, context_cloned, ); let _ = hiro_system_kit::nestable_block_on(future); @@ -478,13 +515,13 @@ pub fn start_event_observer( pub async fn start_bitcoin_event_observer( config: EventObserverConfig, - observer_commands_tx: Sender, + _observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, - block_pre_processor: Option<(Sender, Receiver>)>, + observer_sidecar: Option, ctx: Context, ) -> Result<(), Box> { - let chainhook_store = ChainhookStore::new(); + let chainhook_store = config.get_chainhook_store(); let observer_metrics = ObserverMetrics { bitcoin: ChainMetrics { @@ -498,7 +535,16 @@ pub async fn start_bitcoin_event_observer( }; let observer_metrics_rw_lock = Arc::new(RwLock::new(observer_metrics)); - start_zeromq_runloop(&config, observer_commands_tx, &ctx); + #[cfg(feature = "zeromq")] + { + let ctx_moved = ctx.clone(); + let config_moved = config.clone(); + let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || { + let future = + zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved); + let _ = hiro_system_kit::nestable_block_on(future); + }); + } // This loop is used for handling background jobs, emitted by HTTP calls. start_observer_commands_handler( @@ -508,18 +554,18 @@ pub async fn start_bitcoin_event_observer( observer_events_tx, None, observer_metrics_rw_lock.clone(), - block_pre_processor, + observer_sidecar, ctx, ) .await } pub async fn start_stacks_event_observer( - mut config: EventObserverConfig, + config: EventObserverConfig, observer_commands_tx: Sender, observer_commands_rx: Receiver, observer_events_tx: Option>, - block_pre_processor: Option<(Sender, Receiver>)>, + observer_sidecar: Option, ctx: Context, ) -> Result<(), Box> { let indexer_config = IndexerConfig { @@ -547,18 +593,7 @@ pub async fn start_stacks_event_observer( let bitcoin_rpc_proxy_enabled = config.bitcoin_rpc_proxy_enabled; let bitcoin_config = config.get_bitcoin_config(); - let mut chainhook_store = ChainhookStore::new(); - // If authorization not required, we create a default ChainhookConfig - if let Some(ref mut initial_chainhook_config) = config.chainhook_config { - chainhook_store - .predicates - .stacks_chainhooks - .append(&mut initial_chainhook_config.stacks_chainhooks); - chainhook_store - .predicates - .bitcoin_chainhooks - .append(&mut initial_chainhook_config.bitcoin_chainhooks); - } + let chainhook_store = config.get_chainhook_store(); let indexer_rw_lock = Arc::new(RwLock::new(indexer)); @@ -636,7 +671,7 @@ pub async fn start_stacks_event_observer( observer_events_tx, ingestion_shutdown, observer_metrics_rw_lock.clone(), - block_pre_processor, + observer_sidecar, ctx, ) .await @@ -663,117 +698,6 @@ pub fn get_bitcoin_proof( } } -#[allow(unused_variables, unused_imports)] -pub fn start_zeromq_runloop( - config: &EventObserverConfig, - observer_commands_tx: Sender, - ctx: &Context, -) { - #[cfg(feature = "zeromq")] - { - use crate::indexer::fork_scratch_pad::ForkScratchPad; - - if let BitcoinBlockSignaling::ZeroMQ(ref bitcoind_zmq_url) = config.bitcoin_block_signaling - { - let bitcoind_zmq_url = bitcoind_zmq_url.clone(); - let ctx_moved = ctx.clone(); - let bitcoin_config = config.get_bitcoin_config(); - let http_client = build_http_client(); - - hiro_system_kit::thread_named("Bitcoind zmq listener") - .spawn(move || { - ctx_moved.try_log(|logger| { - slog::info!( - logger, - "Waiting for ZMQ connection acknowledgment from bitcoind" - ) - }); - - let _: Result<(), Box> = - hiro_system_kit::nestable_block_on(async move { - let mut socket = zeromq::SubSocket::new(); - - socket - .connect(&bitcoind_zmq_url) - .await - .expect("Failed to connect"); - - socket.subscribe("").await?; - ctx_moved.try_log(|logger| { - slog::info!(logger, "Waiting for ZMQ messages from bitcoind") - }); - - let mut bitcoin_blocks_pool = ForkScratchPad::new(); - - loop { - let message = match socket.recv().await { - Ok(message) => message, - Err(e) => { - ctx_moved.try_log(|logger| { - slog::error!( - logger, - "Unable to receive ZMQ message: {}", - e.to_string() - ) - }); - continue; - } - }; - let block_hash = hex::encode(message.get(1).unwrap().to_vec()); - - let block = match download_and_parse_block_with_retry( - &http_client, - &block_hash, - &bitcoin_config, - &ctx_moved, - ) - .await - { - Ok(block) => block, - Err(e) => { - ctx_moved.try_log(|logger| { - slog::warn!( - logger, - "unable to download_and_parse_block: {}", - e.to_string() - ) - }); - continue; - } - }; - - ctx_moved.try_log(|logger| { - slog::info!( - logger, - "Bitcoin block #{} dispatched for processing", - block.height - ) - }); - - let header = block.get_block_header(); - let _ = observer_commands_tx - .send(ObserverCommand::ProcessBitcoinBlock(block)); - - if let Ok(Some(event)) = - bitcoin_blocks_pool.process_header(header, &ctx_moved) - { - let _ = observer_commands_tx - .send(ObserverCommand::PropagateBitcoinChainEvent(event)); - } - } - }); - }) - .expect("unable to spawn thread"); - } - } -} - -pub fn pre_process_bitcoin_block() {} - -pub fn apply_bitcoin_block() {} - -pub fn rollback_bitcoin_block() {} - pub fn gather_proofs<'a>( trigger: &BitcoinTriggerChainhook<'a>, proofs: &mut HashMap<&'a TransactionIdentifier, String>, @@ -817,8 +741,8 @@ pub fn gather_proofs<'a>( } pub enum HandleBlock { - ApplyBlocks(Vec), - UndoBlocks(Vec), + ApplyBlock(BitcoinBlockData), + UndoBlock(BitcoinBlockData), } pub async fn start_observer_commands_handler( @@ -828,14 +752,17 @@ pub async fn start_observer_commands_handler( observer_events_tx: Option>, ingestion_shutdown: Option, observer_metrics: Arc>, - block_pre_processor: Option<(Sender, Receiver>)>, + observer_sidecar: Option, ctx: Context, ) -> Result<(), Box> { let mut chainhooks_occurrences_tracker: HashMap = HashMap::new(); - let event_handlers = config.event_handlers.clone(); let networks = (&config.bitcoin_network, &config.stacks_network); - let mut bitcoin_block_store: HashMap = HashMap::new(); + let mut bitcoin_block_store: HashMap = HashMap::new(); let http_client = build_http_client(); + let store_update_required = observer_sidecar + .as_ref() + .and_then(|s| s.bitcoin_blocks_mutator.as_ref()) + .is_some(); loop { let command = match observer_commands_rx.recv() { @@ -861,7 +788,7 @@ pub async fn start_observer_commands_handler( } ObserverCommand::ProcessBitcoinBlock(mut block_data) => { let block_hash = block_data.hash.to_string(); - let new_block = loop { + let block = loop { match standardize_bitcoin_block( block_data.clone(), &config.bitcoin_network, @@ -899,8 +826,8 @@ pub async fn start_observer_commands_handler( }; match observer_metrics.write() { Ok(mut metrics) => { - if new_block.block_identifier.index > metrics.bitcoin.tip_height { - metrics.bitcoin.tip_height = new_block.block_identifier.index; + if block.block_identifier.index > metrics.bitcoin.tip_height { + metrics.bitcoin.tip_height = block.block_identifier.index; } metrics.bitcoin.last_block_ingestion_at = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -912,10 +839,22 @@ pub async fn start_observer_commands_handler( slog::warn!(logger, "unable to acquire observer_metrics_rw_lock:{}", e) }), }; - bitcoin_block_store.insert(new_block.block_identifier.clone(), new_block); + bitcoin_block_store.insert( + block.block_identifier.clone(), + BitcoinBlockDataCached { + block, + processed_by_sidecar: false, + }, + ); } ObserverCommand::CacheBitcoinBlock(block) => { - bitcoin_block_store.insert(block.block_identifier.clone(), block); + bitcoin_block_store.insert( + block.block_identifier.clone(), + BitcoinBlockDataCached { + block, + processed_by_sidecar: false, + }, + ); } ObserverCommand::PropagateBitcoinChainEvent(blockchain_event) => { ctx.try_log(|logger| { @@ -926,36 +865,40 @@ pub async fn start_observer_commands_handler( // Update Chain event before propagation let chain_event = match blockchain_event { BlockchainEvent::BlockchainUpdatedWithHeaders(data) => { + let mut blocks_to_mutate = vec![]; let mut new_blocks = vec![]; for header in data.new_headers.iter() { - match bitcoin_block_store.get(&header.block_identifier) { - Some(block) => { - new_blocks.push(block.clone()); - } - None => { - ctx.try_log(|logger| { - slog::error!( - logger, - "Unable to retrieve bitcoin block {}", - header.block_identifier - ) - }); - } - } + if store_update_required { + let Some(block) = bitcoin_block_store.remove(&header.block_identifier) else { + continue; + }; + blocks_to_mutate.push(block); + } else { + let Some(cache) = bitcoin_block_store.get(&header.block_identifier) else { + continue; + }; + new_blocks.push(cache.block.clone()); + }; } - new_blocks = handle_blocks_pre_processing( - &block_pre_processor, - new_blocks, - true, - &ctx, - ); + if let Some(ref sidecar) = observer_sidecar { + let updated_blocks = sidecar.perform_bitcoin_sidecar_mutations( + blocks_to_mutate, + vec![], + &ctx, + ); + for cache in updated_blocks.into_iter() { + bitcoin_block_store + .insert(cache.block.block_identifier.clone(), cache.clone()); + new_blocks.push(cache.block); + } + } for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { - Some(block) => { - confirmed_blocks.push(block); + Some(res) => { + confirmed_blocks.push(res.block); } None => { ctx.try_log(|logger| { @@ -977,28 +920,32 @@ pub async fn start_observer_commands_handler( ) } BlockchainEvent::BlockchainUpdatedWithReorg(data) => { - let mut blocks_to_apply = vec![]; let mut blocks_to_rollback = vec![]; - let blocks_ids_to_rollback = data - .headers_to_rollback - .iter() - .map(|b| b.block_identifier.index.to_string()) - .collect::>(); - let blocks_ids_to_apply = data - .headers_to_apply - .iter() - .map(|b| b.block_identifier.index.to_string()) - .collect::>(); + let mut blocks_to_mutate = vec![]; + let mut blocks_to_apply = vec![]; - ctx.try_log(|logger| { - slog::info!(logger, "Bitcoin reorg detected, will rollback blocks {} and apply blocks {}", blocks_ids_to_rollback.join(", "), blocks_ids_to_apply.join(", ")) - }); + for header in data.headers_to_apply.iter() { + if store_update_required { + let Some(block) = bitcoin_block_store.remove(&header.block_identifier) else { + continue; + }; + blocks_to_mutate.push(block); + } else { + let Some(cache) = bitcoin_block_store.get(&header.block_identifier) else { + continue; + }; + blocks_to_apply.push(cache.block.clone()); + }; + } + + let mut blocks_ids_to_rollback: Vec = vec![]; for header in data.headers_to_rollback.iter() { match bitcoin_block_store.get(&header.block_identifier) { - Some(block) => { - blocks_to_rollback.push(block.clone()); + Some(cache) => { + blocks_ids_to_rollback.push(header.block_identifier.clone()); + blocks_to_rollback.push(cache.block.clone()); } None => { ctx.try_log(|logger| { @@ -1012,41 +959,23 @@ pub async fn start_observer_commands_handler( } } - blocks_to_rollback = handle_blocks_pre_processing( - &block_pre_processor, - blocks_to_rollback, - false, - &ctx, - ); - - for header in data.headers_to_apply.iter() { - match bitcoin_block_store.get_mut(&header.block_identifier) { - Some(block) => { - blocks_to_apply.push(block.clone()); - } - None => { - ctx.try_log(|logger| { - slog::error!( - logger, - "Unable to retrieve bitcoin block {}", - header.block_identifier - ) - }); - } + if let Some(ref sidecar) = observer_sidecar { + let updated_blocks = sidecar.perform_bitcoin_sidecar_mutations( + blocks_to_mutate, + blocks_ids_to_rollback, + &ctx, + ); + for cache in updated_blocks.into_iter() { + bitcoin_block_store + .insert(cache.block.block_identifier.clone(), cache.clone()); + blocks_to_apply.push(cache.block); } } - blocks_to_apply = handle_blocks_pre_processing( - &block_pre_processor, - blocks_to_apply, - true, - &ctx, - ); - for header in data.confirmed_headers.iter() { match bitcoin_block_store.remove(&header.block_identifier) { - Some(block) => { - confirmed_blocks.push(block); + Some(res) => { + confirmed_blocks.push(res.block); } None => { ctx.try_log(|logger| { @@ -1091,8 +1020,8 @@ pub async fn start_observer_commands_handler( } }; - for event_handler in event_handlers.iter() { - event_handler.propagate_bitcoin_event(&chain_event).await; + if let Some(ref sidecar) = observer_sidecar { + sidecar.notify_chain_event(&chain_event, &ctx) } // process hooks let mut hooks_ids_to_deregister = vec![]; @@ -1253,9 +1182,6 @@ pub async fn start_observer_commands_handler( ctx.try_log(|logger| { slog::info!(logger, "Handling PropagateStacksChainEvent command") }); - for event_handler in event_handlers.iter() { - event_handler.propagate_stacks_event(&chain_event).await; - } let mut hooks_ids_to_deregister = vec![]; let mut requests = vec![]; let mut report = PredicateEvaluationReport::new(); @@ -1463,9 +1389,6 @@ pub async fn start_observer_commands_handler( ctx.try_log(|logger| { slog::info!(logger, "Handling NotifyBitcoinTransactionProxied command") }); - for event_handler in event_handlers.iter() { - event_handler.notify_bitcoin_transaction_proxied().await; - } if let Some(ref tx) = observer_events_tx { let _ = tx.send(ObserverEvent::NotifyBitcoinTransactionProxied); } @@ -1581,38 +1504,5 @@ pub async fn start_observer_commands_handler( Ok(()) } -fn handle_blocks_pre_processing( - block_pre_processor: &Option<(Sender, Receiver>)>, - blocks: Vec, - apply: bool, - ctx: &Context, -) -> Vec { - if let Some(ref processor) = block_pre_processor { - ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",)); - let _ = processor.0.send(match apply { - true => HandleBlock::ApplyBlocks(blocks.clone()), - false => HandleBlock::UndoBlocks(blocks.clone()), - }); - ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",)); - match processor.1.recv() { - Ok(updated_blocks) => { - ctx.try_log(|logger| slog::info!(logger, "Blocks received from pre-processor",)); - return updated_blocks; - } - Err(e) => { - ctx.try_log(|logger| { - slog::error!( - logger, - "Unable to receive block from pre-processor {}", - e.to_string() - ) - }); - return blocks; - } - } - } - blocks -} - #[cfg(test)] pub mod tests; diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs index 0f8dd3c69..1efea079d 100644 --- a/components/chainhook-sdk/src/observer/tests/mod.rs +++ b/components/chainhook-sdk/src/observer/tests/mod.rs @@ -2,20 +2,23 @@ use crate::chainhooks::types::{ BitcoinChainhookFullSpecification, BitcoinChainhookNetworkSpecification, BitcoinChainhookSpecification, BitcoinPredicateType, ChainhookConfig, ChainhookFullSpecification, ChainhookSpecification, ExactMatchingRule, HookAction, - OutputPredicate, StacksChainhookFullSpecification, StacksChainhookNetworkSpecification, - StacksChainhookSpecification, StacksContractCallBasedPredicate, StacksPredicate, + OrdinalOperations, OutputPredicate, StacksChainhookFullSpecification, + StacksChainhookNetworkSpecification, StacksChainhookSpecification, + StacksContractCallBasedPredicate, StacksPredicate, }; +use crate::indexer::fork_scratch_pad::ForkScratchPad; use crate::indexer::tests::helpers::transactions::generate_test_tx_bitcoin_p2pkh_transfer; use crate::indexer::tests::helpers::{ accounts, bitcoin_blocks, stacks_blocks, transactions::generate_test_tx_stacks_contract_call, }; use crate::observer::{ start_observer_commands_handler, ChainhookStore, EventObserverConfig, ObserverCommand, - ObserverMetrics, + ObserverMetrics, ObserverSidecar, }; use crate::utils::{AbstractBlock, Context}; use chainhook_types::{ - BitcoinBlockSignaling, BitcoinNetwork, BlockchainEvent, BlockchainUpdatedWithHeaders, + BitcoinBlockSignaling, BitcoinChainEvent, BitcoinNetwork, BlockchainEvent, + BlockchainUpdatedWithHeaders, OrdinalInscriptionRevealData, OrdinalOperation, StacksBlockUpdate, StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork, StacksNodeConfig, }; @@ -30,7 +33,6 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) { let config: EventObserverConfig = EventObserverConfig { chainhook_config: Some(ChainhookConfig::new()), bitcoin_rpc_proxy_enabled: false, - event_handlers: vec![], ingestion_port: 0, bitcoind_rpc_username: "user".into(), bitcoind_rpc_password: "user".into(), @@ -42,6 +44,7 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) { cache_path: "cache".into(), bitcoin_network: BitcoinNetwork::Regtest, stacks_network: StacksNetwork::Devnet, + data_handler_tx: None, }; let predicates = ChainhookConfig::new(); let chainhook_store = ChainhookStore { predicates }; @@ -116,6 +119,34 @@ fn bitcoin_chainhook_p2pkh( spec } +fn bitcoin_chainhook_ordinals(id: u8) -> BitcoinChainhookFullSpecification { + let mut networks = BTreeMap::new(); + networks.insert( + BitcoinNetwork::Regtest, + BitcoinChainhookNetworkSpecification { + start_block: None, + end_block: None, + blocks: None, + expire_after_occurrence: None, + predicate: BitcoinPredicateType::OrdinalsProtocol(OrdinalOperations::InscriptionFeed), + action: HookAction::Noop, + include_proof: None, + include_inputs: None, + include_outputs: None, + include_witness: None, + }, + ); + + let spec = BitcoinChainhookFullSpecification { + uuid: format!("{}", id), + name: format!("Chainhook {}", id), + owner_uuid: None, + version: 1, + networks, + }; + spec +} + fn generate_and_register_new_stacks_chainhook( observer_commands_tx: &Sender, observer_events_rx: &crossbeam_channel::Receiver, @@ -298,6 +329,37 @@ fn assert_observer_metrics_bitcoin_deregistered_predicates( ); } +fn generate_and_register_new_ordinals_chainhook( + observer_commands_tx: &Sender, + observer_events_rx: &crossbeam_channel::Receiver, + id: u8, +) -> BitcoinChainhookSpecification { + let chainhook = bitcoin_chainhook_ordinals(id); + let _ = observer_commands_tx.send(ObserverCommand::RegisterPredicate( + ChainhookFullSpecification::Bitcoin(chainhook.clone()), + )); + let mut chainhook = chainhook + .into_selected_network_specification(&BitcoinNetwork::Regtest) + .unwrap(); + chainhook.enabled = true; + let _ = observer_commands_tx.send(ObserverCommand::EnablePredicate( + ChainhookSpecification::Bitcoin(chainhook.clone()), + )); + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::PredicateRegistered(_)) => { + true + } + _ => false, + }); + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::PredicateEnabled(_)) => { + true + } + _ => false, + }); + chainhook +} + #[test] fn test_stacks_chainhook_register_deregister() { let (observer_commands_tx, observer_commands_rx) = channel(); @@ -1041,3 +1103,230 @@ fn test_bitcoin_chainhook_auto_deregister() { let _ = observer_commands_tx.send(ObserverCommand::Terminate); handle.join().expect("unable to terminate thread"); } + +#[test] +fn test_bitcoin_chainhook_through_reorg() { + let (observer_commands_tx, observer_commands_rx) = channel(); + let (block_pre_processor_in_tx, block_pre_processor_in_rx) = crossbeam_channel::unbounded(); + let (block_pre_processor_out_tx, block_pre_processor_out_rx) = crossbeam_channel::unbounded(); + + let (observer_events_tx, observer_events_rx) = crossbeam_channel::unbounded(); + let observer_metrics_rw_lock = Arc::new(RwLock::new(ObserverMetrics::default())); + let observer_metrics_rw_lock_moved = observer_metrics_rw_lock.clone(); + + let empty_ctx = Context::empty(); + + let observer_sidecar = ObserverSidecar { + bitcoin_blocks_mutator: Some((block_pre_processor_in_tx, block_pre_processor_out_rx)), + bitcoin_chain_event_notifier: None, + }; + + let handle = std::thread::spawn(move || { + let (config, chainhook_store) = generate_test_config(); + let _ = hiro_system_kit::nestable_block_on(start_observer_commands_handler( + config, + chainhook_store, + observer_commands_rx, + Some(observer_events_tx), + None, + observer_metrics_rw_lock_moved, + Some(observer_sidecar), + Context::empty(), + )); + }); + + // The block pre-processor will simulate block augmentation with new informations, which should trigger + // registered predicates + let block_pre_processor_handle = std::thread::spawn(move || { + let mut cursor: u64 = 0; + while let Ok((mut blocks, _)) = block_pre_processor_in_rx.recv() { + for b in blocks.iter_mut() { + for (tx_index, tx) in b.block.transactions.iter_mut().enumerate() { + cursor += 1; + tx.metadata + .ordinal_operations + .push(OrdinalOperation::InscriptionRevealed( + OrdinalInscriptionRevealData { + content_bytes: format!("{cursor}"), + content_type: "".to_string(), + content_length: cursor as usize, + inscription_number: cursor as i64, + inscription_fee: cursor, + inscription_output_value: cursor, + inscription_id: format!("{cursor}"), + inscription_input_index: 0, + inscriber_address: None, + ordinal_number: cursor, + ordinal_block_height: b.block.block_identifier.index, + ordinal_offset: 0, + tx_index, + transfers_pre_inscription: cursor as u32, + satpoint_post_inscription: format!("{cursor}"), + curse_type: None, + }, + )) + } + } + let _ = block_pre_processor_out_tx.send(blocks); + } + }); + + let genesis = bitcoin_blocks::generate_test_bitcoin_block(0, 1, vec![], None); + let mut fork_pad = ForkScratchPad::new(); + let _ = fork_pad.process_header(genesis.get_header(), &empty_ctx); + + // Create and register a new chainhook (wallet_2 received some sats) + let _chainhook = + generate_and_register_new_ordinals_chainhook(&observer_commands_tx, &observer_events_rx, 1); + + // registering bitcoin chainhook should increment the observer_metric's registered bitcoin hooks + assert_eq!( + 1, + observer_metrics_rw_lock + .read() + .unwrap() + .bitcoin + .registered_predicates + ); + + // Simulate a block that does not include a trigger (wallet_1 to wallet_3) + let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer( + 0, + &accounts::wallet_1_btc_address(), + &accounts::wallet_3_btc_address(), + 3, + )]; + let block = bitcoin_blocks::generate_test_bitcoin_block(0, 2, transactions, None); + let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone())); + + let chain_event = fork_pad + .process_header(block.get_header(), &empty_ctx) + .unwrap() + .unwrap(); + let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event)); + // Should signal that no hook were triggered + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::PredicatesTriggered(len)) => { + assert_eq!(len, 1); + true + } + Ok(event) => { + println!("Unexpected event: {:?}", event); + false + } + Err(e) => { + println!("Error: {:?}", e); + false + } + }); + + // 1) Should kick off predicate event + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => { + assert_eq!(payload.apply.len(), 1); + assert_eq!(payload.apply[0].block.transactions.len(), 1); + true + } + _ => false, + }); + + // 2) Should kick off bitcoin chain event + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithBlocks(_), _))) => { + true + } + _ => false, + }); + + // Simulate a block that does include a trigger (wallet_1 to wallet_2) + let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer( + 0, + &accounts::wallet_1_btc_address(), + &accounts::wallet_2_btc_address(), + 3, + )]; + let block = bitcoin_blocks::generate_test_bitcoin_block(1, 2, transactions, None); + let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone())); + let chain_event = fork_pad + .process_header(block.get_header(), &empty_ctx) + .unwrap() + .unwrap(); + let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event)); + + // Should signal that no hook were triggered + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::PredicatesTriggered(len)) => { + assert_eq!(len, 1); + true + } + _ => false, + }); + + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => { + assert_eq!(payload.rollback.len(), 1); + assert_eq!(payload.rollback[0].block.transactions.len(), 1); + assert_eq!(payload.apply.len(), 1); + assert_eq!(payload.apply[0].block.transactions.len(), 1); + true + } + _ => false, + }); + + // Should propagate chain event + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithReorg(_), _))) => { + true + } + _ => false, + }); + + // Simulate a block that does include a trigger (wallet_1 to wallet_2) + let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer( + 0, + &accounts::wallet_1_btc_address(), + &accounts::wallet_2_btc_address(), + 3, + )]; + let block = bitcoin_blocks::generate_test_bitcoin_block(0, 3, transactions, None); + let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone())); + let chain_event = fork_pad + .process_header(block.get_header(), &empty_ctx) + .unwrap() + .unwrap(); + let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event)); + + // Should signal that no hook were triggered + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::PredicatesTriggered(len)) => { + assert_eq!(len, 1); + true + } + _ => false, + }); + + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => { + assert_eq!(payload.rollback.len(), 1); + assert_eq!(payload.rollback[0].block.transactions.len(), 1); + assert_eq!(payload.apply.len(), 2); + assert_eq!(payload.apply[0].block.transactions.len(), 1); + true + } + _ => false, + }); + + // Should propagate block + assert!(match observer_events_rx.recv() { + Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithReorg(_), _))) => { + true + } + _ => false, + }); + + let _ = observer_commands_tx.send(ObserverCommand::Terminate); + handle.join().expect("unable to terminate thread"); + block_pre_processor_handle + .join() + .expect("unable to terminate thread"); +} diff --git a/components/chainhook-sdk/src/observer/zmq.rs b/components/chainhook-sdk/src/observer/zmq.rs new file mode 100644 index 000000000..0f53cc740 --- /dev/null +++ b/components/chainhook-sdk/src/observer/zmq.rs @@ -0,0 +1,155 @@ +use chainhook_types::BitcoinBlockSignaling; +use hiro_system_kit::slog; +use std::sync::mpsc::Sender; +use zmq::Socket; + +use crate::{ + indexer::{ + bitcoin::{build_http_client, download_and_parse_block_with_retry}, + fork_scratch_pad::ForkScratchPad, + }, + utils::Context, +}; +use std::collections::VecDeque; + +use super::{EventObserverConfig, ObserverCommand}; + +fn new_zmq_socket() -> Socket { + let context = zmq::Context::new(); + let socket = context.socket(zmq::SUB).unwrap(); + assert!(socket.set_subscribe(b"hashblock").is_ok()); + assert!(socket.set_rcvhwm(0).is_ok()); + // We override the OS default behavior: + assert!(socket.set_tcp_keepalive(1).is_ok()); + // The keepalive routine will wait for 5 minutes + assert!(socket.set_tcp_keepalive_idle(300).is_ok()); + // And then resend it every 60 seconds + assert!(socket.set_tcp_keepalive_intvl(60).is_ok()); + // 120 times + assert!(socket.set_tcp_keepalive_cnt(120).is_ok()); + socket +} + +pub async fn start_zeromq_runloop( + config: &EventObserverConfig, + observer_commands_tx: Sender, + ctx: &Context, +) { + let BitcoinBlockSignaling::ZeroMQ(ref bitcoind_zmq_url) = config.bitcoin_block_signaling else { + unreachable!() + }; + + let bitcoind_zmq_url = bitcoind_zmq_url.clone(); + let bitcoin_config = config.get_bitcoin_config(); + let http_client = build_http_client(); + + ctx.try_log(|logger| { + slog::info!( + logger, + "Waiting for ZMQ connection acknowledgment from bitcoind" + ) + }); + + let mut socket = new_zmq_socket(); + assert!(socket.connect(&bitcoind_zmq_url).is_ok()); + ctx.try_log(|logger| slog::info!(logger, "Waiting for ZMQ messages from bitcoind")); + + let mut bitcoin_blocks_pool = ForkScratchPad::new(); + + loop { + let msg = match socket.recv_multipart(0) { + Ok(msg) => msg, + Err(e) => { + ctx.try_log(|logger| { + slog::error!(logger, "Unable to receive ZMQ message: {}", e.to_string()) + }); + socket = new_zmq_socket(); + assert!(socket.connect(&bitcoind_zmq_url).is_ok()); + continue; + } + }; + let (topic, data, _sequence) = (&msg[0], &msg[1], &msg[2]); + + if !topic.eq(b"hashblock") { + ctx.try_log(|logger| slog::error!(logger, "Topic not supported",)); + continue; + } + + let block_hash = hex::encode(data); + + ctx.try_log(|logger| slog::info!(logger, "Bitcoin block hash announced #{block_hash}",)); + + let mut block_hashes: VecDeque = VecDeque::new(); + block_hashes.push_front(block_hash); + + while let Some(block_hash) = block_hashes.pop_front() { + let block = match download_and_parse_block_with_retry( + &http_client, + &block_hash, + &bitcoin_config, + &ctx, + ) + .await + { + Ok(block) => block, + Err(e) => { + ctx.try_log(|logger| { + slog::warn!( + logger, + "unable to download_and_parse_block: {}", + e.to_string() + ) + }); + continue; + } + }; + + let header = block.get_block_header(); + ctx.try_log(|logger| { + slog::info!( + logger, + "Bitcoin block #{} dispatched for processing", + block.height + ) + }); + + let _ = observer_commands_tx.send(ObserverCommand::ProcessBitcoinBlock(block)); + + if bitcoin_blocks_pool.can_process_header(&header) { + match bitcoin_blocks_pool.process_header(header, &ctx) { + Ok(Some(event)) => { + let _ = observer_commands_tx + .send(ObserverCommand::PropagateBitcoinChainEvent(event)); + } + Err(e) => { + ctx.try_log(|logger| { + slog::warn!(logger, "Unable to append block: {:?}", e) + }); + } + Ok(None) => { + ctx.try_log(|logger| slog::warn!(logger, "Unable to append block")); + } + } + } else { + // Handle a behaviour specific to ZMQ usage in bitcoind. + // Considering a simple re-org: + // A (1) - B1 (2) - C1 (3) + // \ B2 (4) - C2 (5) - D2 (6) + // When D2 is being discovered (making A -> B2 -> C2 -> D2 the new canonical fork) + // it looks like ZMQ is only publishing D2. + // Without additional operation, we end up with a block that we can't append. + let parent_block_hash = header + .parent_block_identifier + .get_hash_bytes_str() + .to_string(); + ctx.try_log(|logger| { + slog::info!( + logger, + "Possible re-org detected, retrieving parent block {parent_block_hash}" + ) + }); + block_hashes.push_front(parent_block_hash); + } + } + } +} diff --git a/components/chainhook-types-rs/Cargo.toml b/components/chainhook-types-rs/Cargo.toml index d4c97de44..de2f3263f 100644 --- a/components/chainhook-types-rs/Cargo.toml +++ b/components/chainhook-types-rs/Cargo.toml @@ -2,7 +2,7 @@ name = "chainhook-types" description = "Bitcoin and Stacks data schemas, based on the Rosetta specification" license = "MIT" -version = "1.0.11" +version = "1.0.12" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs index 63db84ca4..39ba9c005 100644 --- a/components/chainhook-types-rs/src/rosetta.rs +++ b/components/chainhook-types-rs/src/rosetta.rs @@ -14,6 +14,16 @@ pub struct BlockIdentifier { pub hash: String, } +impl BlockIdentifier { + pub fn get_hash_bytes_str(&self) -> &str { + &self.hash[2..] + } + + pub fn get_hash_bytes(&self) -> Vec { + hex::decode(&self.get_hash_bytes_str()).unwrap() + } +} + impl Display for BlockIdentifier { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(