From fd3627f2232cf7d50c69e762877d29c0d352ffe7 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Mon, 21 Aug 2023 18:01:13 +0200
Subject: [PATCH 01/20] chore: tweak logs

---
 components/chainhook-cli/src/cli/mod.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/components/chainhook-cli/src/cli/mod.rs b/components/chainhook-cli/src/cli/mod.rs
index 28eb37783..64f136478 100644
--- a/components/chainhook-cli/src/cli/mod.rs
+++ b/components/chainhook-cli/src/cli/mod.rs
@@ -273,14 +273,14 @@ pub fn main() {
     let opts: Opts = match Opts::try_parse() {
         Ok(opts) => opts,
         Err(e) => {
-            println!("{}", e);
+            error!(ctx.expect_logger(), "{e}");
             process::exit(1);
         }
     };
 
-    match hiro_system_kit::nestable_block_on(handle_command(opts, ctx)) {
+    match hiro_system_kit::nestable_block_on(handle_command(opts, ctx.clone())) {
         Err(e) => {
-            println!("{e}");
+            error!(ctx.expect_logger(), "{e}");
             process::exit(1);
         }
         Ok(_) => {}

From 61d9e2da6c38dfe13d1c59d152f268e2050e78ea Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Mon, 21 Aug 2023 18:02:04 +0200
Subject: [PATCH 02/20] chore: add helper to BlockIdentifier

---
 components/chainhook-types-rs/src/rosetta.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/components/chainhook-types-rs/src/rosetta.rs b/components/chainhook-types-rs/src/rosetta.rs
index 63db84ca4..39ba9c005 100644
--- a/components/chainhook-types-rs/src/rosetta.rs
+++ b/components/chainhook-types-rs/src/rosetta.rs
@@ -14,6 +14,16 @@ pub struct BlockIdentifier {
     pub hash: String,
 }
 
+impl BlockIdentifier {
+    pub fn get_hash_bytes_str(&self) -> &str {
+        &self.hash[2..]
+    }
+
+    pub fn get_hash_bytes(&self) -> Vec<u8> {
+        hex::decode(&self.get_hash_bytes_str()).unwrap()
+    }
+}
+
 impl Display for BlockIdentifier {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(

From ef568b9cf3cd603395015fd7ef5bb691beb719d5 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Mon, 21 Aug 2023 22:09:10 +0200
Subject: [PATCH 03/20] refactor: pass block instead of blocks

---
 components/chainhook-sdk/src/observer/mod.rs | 105 +++++++++++--------
 1 file changed, 60 insertions(+), 45 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index e40b47c35..5b8ee8480 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -33,7 +33,7 @@ use rocket::config::{self, Config, LogLevel};
 use rocket::data::{Limits, ToByteUnit};
 use rocket::serde::Deserialize;
 use rocket::Shutdown;
-use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
 use std::error::Error;
 use std::net::{IpAddr, Ipv4Addr};
 use std::path::PathBuf;
@@ -397,7 +397,7 @@ pub fn start_event_observer(
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<Vec<BitcoinBlockData>>)>,
+    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     match config.bitcoin_block_signaling {
@@ -458,7 +458,7 @@ pub async fn start_bitcoin_event_observer(
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<Vec<BitcoinBlockData>>)>,
+    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let chainhook_store = ChainhookStore::new();
@@ -496,7 +496,7 @@ pub async fn start_stacks_event_observer(
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<Vec<BitcoinBlockData>>)>,
+    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let indexer_config = IndexerConfig {
@@ -794,8 +794,8 @@ pub fn gather_proofs<'a>(
 }
 
 pub enum HandleBlock {
-    ApplyBlocks(Vec<BitcoinBlockData>),
-    UndoBlocks(Vec<BitcoinBlockData>),
+    ApplyBlocks(BitcoinBlockData),
+    UndoBlocks(BitcoinBlockData),
 }
 
 pub async fn start_observer_commands_handler(
@@ -805,7 +805,7 @@ pub async fn start_observer_commands_handler(
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
     ingestion_shutdown: Option<Shutdown>,
     observer_metrics: Arc<RwLock<ObserverMetrics>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<Vec<BitcoinBlockData>>)>,
+    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let mut chainhooks_occurrences_tracker: HashMap<String, u64> = HashMap::new();
@@ -906,9 +906,21 @@ pub async fn start_observer_commands_handler(
                         let mut new_blocks = vec![];
 
                         for header in data.new_headers.iter() {
-                            match bitcoin_block_store.get(&header.block_identifier) {
+                            match bitcoin_block_store.remove(&header.block_identifier) {
                                 Some(block) => {
-                                    new_blocks.push(block.clone());
+                                    // Time for pre-processing
+                                    let (_updated, updated_block) = handle_block_pre_processing(
+                                        &block_pre_processor,
+                                        block,
+                                        true,
+                                        &ctx,
+                                    );
+                                    // Keep a copy of the pre-processed version
+                                    bitcoin_block_store.insert(
+                                        updated_block.block_identifier.clone(),
+                                        updated_block.clone(),
+                                    );
+                                    new_blocks.push(updated_block);
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -922,13 +934,6 @@ pub async fn start_observer_commands_handler(
                             }
                         }
 
-                        new_blocks = handle_blocks_pre_processing(
-                            &block_pre_processor,
-                            new_blocks,
-                            true,
-                            &ctx,
-                        );
-
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
                                 Some(block) => {
@@ -973,9 +978,21 @@ pub async fn start_observer_commands_handler(
                         });
 
                         for header in data.headers_to_rollback.iter() {
-                            match bitcoin_block_store.get(&header.block_identifier) {
+                            match bitcoin_block_store.remove(&header.block_identifier) {
                                 Some(block) => {
-                                    blocks_to_rollback.push(block.clone());
+                                    // Time for pre-processing
+                                    let (_updated, updated_block) = handle_block_pre_processing(
+                                        &block_pre_processor,
+                                        block,
+                                        false,
+                                        &ctx,
+                                    );
+                                    // Keep a copy of the pre-processed version
+                                    bitcoin_block_store.insert(
+                                        updated_block.block_identifier.clone(),
+                                        updated_block.clone(),
+                                    );
+                                    blocks_to_rollback.push(updated_block);
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -989,17 +1006,22 @@ pub async fn start_observer_commands_handler(
                             }
                         }
 
-                        blocks_to_rollback = handle_blocks_pre_processing(
-                            &block_pre_processor,
-                            blocks_to_rollback,
-                            false,
-                            &ctx,
-                        );
-
                         for header in data.headers_to_apply.iter() {
-                            match bitcoin_block_store.get_mut(&header.block_identifier) {
+                            match bitcoin_block_store.remove(&header.block_identifier) {
                                 Some(block) => {
-                                    blocks_to_apply.push(block.clone());
+                                    // Time for pre-processing
+                                    let (_updated, updated_block) = handle_block_pre_processing(
+                                        &block_pre_processor,
+                                        block,
+                                        true,
+                                        &ctx,
+                                    );
+                                    // Keep a copy of the pre-processed version
+                                    bitcoin_block_store.insert(
+                                        updated_block.block_identifier.clone(),
+                                        updated_block.clone(),
+                                    );
+                                    blocks_to_apply.push(updated_block);
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -1013,13 +1035,6 @@ pub async fn start_observer_commands_handler(
                             }
                         }
 
-                        blocks_to_apply = handle_blocks_pre_processing(
-                            &block_pre_processor,
-                            blocks_to_apply,
-                            true,
-                            &ctx,
-                        );
-
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
                                 Some(block) => {
@@ -1524,23 +1539,23 @@ pub async fn start_observer_commands_handler(
     Ok(())
 }
 
-fn handle_blocks_pre_processing(
-    block_pre_processor: &Option<(Sender<HandleBlock>, Receiver<Vec<BitcoinBlockData>>)>,
-    blocks: Vec<BitcoinBlockData>,
+fn handle_block_pre_processing(
+    block_pre_processor: &Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
+    block: BitcoinBlockData,
     apply: bool,
     ctx: &Context,
-) -> Vec<BitcoinBlockData> {
+) -> (bool, BitcoinBlockData) {
     if let Some(ref processor) = block_pre_processor {
         ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",));
         let _ = processor.0.send(match apply {
-            true => HandleBlock::ApplyBlocks(blocks.clone()),
-            false => HandleBlock::UndoBlocks(blocks.clone()),
+            true => HandleBlock::ApplyBlocks(block.clone()),
+            false => HandleBlock::UndoBlocks(block.clone()),
         });
         ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",));
         match processor.1.recv() {
-            Ok(updated_blocks) => {
-                ctx.try_log(|logger| slog::info!(logger, "Blocks received from pre-processor",));
-                return updated_blocks;
+            Ok(updated_block) => {
+                ctx.try_log(|logger| slog::info!(logger, "Block received from pre-processor",));
+                return (true, updated_block);
             }
             Err(e) => {
                 ctx.try_log(|logger| {
@@ -1550,11 +1565,11 @@ fn handle_blocks_pre_processing(
                         e.to_string()
                     )
                 });
-                return blocks;
+                return (false, block);
             }
         }
     }
-    blocks
+    return (false, block);
 }
 
 #[cfg(test)]

From 275526620209e8b7137722f9c081aa7b9dca31e5 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Mon, 21 Aug 2023 22:09:49 +0200
Subject: [PATCH 04/20] fix: ensure that the parent block was previously
 received. else, fetch it

---
 .../src/indexer/fork_scratch_pad.rs           |   9 ++
 components/chainhook-sdk/src/observer/mod.rs  | 106 ++++++++++++------
 2 files changed, 82 insertions(+), 33 deletions(-)

diff --git a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
index 3f47941c5..0c040c134 100644
--- a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
+++ b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
@@ -29,6 +29,15 @@ impl ForkScratchPad {
         }
     }
 
+    pub fn can_process_header(&self, header: &BlockHeader) -> bool {
+        if self.headers_store.is_empty() {
+            return true
+        }
+        
+        self.headers_store
+            .contains_key(&header.parent_block_identifier)
+    }
+
     pub fn process_header(
         &mut self,
         header: BlockHeader,
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index 5b8ee8480..a976a1bcd 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -697,45 +697,85 @@ pub fn start_zeromq_runloop(
                                     }
                                 };
                                 let block_hash = hex::encode(message.get(1).unwrap().to_vec());
+                                let mut block_hashes: VecDeque<String> = VecDeque::new();
+                                block_hashes.push_front(block_hash);
 
-                                let block = match download_and_parse_block_with_retry(
-                                    &http_client,
-                                    &block_hash,
-                                    &bitcoin_config,
-                                    &ctx_moved,
-                                )
-                                .await
-                                {
-                                    Ok(block) => block,
-                                    Err(e) => {
+                                while let Some(block_hash) = block_hashes.pop_front() {
+
+                                    let block = match download_and_parse_block_with_retry(
+                                        &http_client,
+                                        &block_hash,
+                                        &bitcoin_config,
+                                        &ctx_moved,
+                                    )
+                                    .await
+                                    {
+                                        Ok(block) => block,
+                                        Err(e) => {
+                                            ctx_moved.try_log(|logger| {
+                                                slog::warn!(
+                                                    logger,
+                                                    "unable to download_and_parse_block: {}",
+                                                    e.to_string()
+                                                )
+                                            });
+                                            continue;
+                                        }
+                                    };
+
+                                    let header = block.get_block_header();
+                                    ctx_moved.try_log(|logger| {
+                                        slog::info!(
+                                            logger,
+                                            "Bitcoin block #{} dispatched for processing",
+                                            block.height
+                                        )
+                                    });
+
+                                    let _ = observer_commands_tx
+                                        .send(ObserverCommand::ProcessBitcoinBlock(block));
+
+                                    if bitcoin_blocks_pool.can_process_header(&header) {
+                                        match bitcoin_blocks_pool.process_header(header, &ctx_moved) {
+                                            Ok(Some(event)) => {
+                                                let _ = observer_commands_tx
+                                                    .send(ObserverCommand::PropagateBitcoinChainEvent(event));
+                                            },
+                                            Err(e) => {
+                                                ctx_moved.try_log(|logger| {
+                                                    slog::warn!(
+                                                        logger,
+                                                        "Unable to append block: {:?}", e
+                                                    )
+                                                });
+                                            }
+                                            Ok(None) => {
+                                                ctx_moved.try_log(|logger| {
+                                                    slog::warn!(
+                                                        logger,
+                                                        "Unable to append block"
+                                                    )
+                                                });
+                                            }
+                                        }
+                                    } else {
+                                        // Handle a behaviour specific to ZMQ usage in bitcoind.
+                                        // Considering a simple re-org:
+                                        // A (1) - B1 (2) - C1 (3)
+                                        //       \ B2 (4) - C2 (5) - D2 (6)
+                                        // When D2 is being discovered (making A -> B2 -> C2 -> D2 the new canonical fork)
+                                        // it looks like ZMQ is only publishing D2.
+                                        // Without additional operation, we end up with a block that with a block that we
+                                        // can't handle.
+                                        let parent_block_hash = header.parent_block_identifier.get_hash_bytes_str().to_string();
                                         ctx_moved.try_log(|logger| {
-                                            slog::warn!(
+                                            slog::info!(
                                                 logger,
-                                                "unable to download_and_parse_block: {}",
-                                                e.to_string()
+                                                "Possible re-org detected, retrieving parent block {parent_block_hash}"
                                             )
                                         });
-                                        continue;
+                                        block_hashes.push_front(parent_block_hash);
                                     }
-                                };
-
-                                ctx_moved.try_log(|logger| {
-                                    slog::info!(
-                                        logger,
-                                        "Bitcoin block #{} dispatched for processing",
-                                        block.height
-                                    )
-                                });
-
-                                let header = block.get_block_header();
-                                let _ = observer_commands_tx
-                                    .send(ObserverCommand::ProcessBitcoinBlock(block));
-
-                                if let Ok(Some(event)) =
-                                    bitcoin_blocks_pool.process_header(header, &ctx_moved)
-                                {
-                                    let _ = observer_commands_tx
-                                        .send(ObserverCommand::PropagateBitcoinChainEvent(event));
                                 }
                             }
                         });

From 6fecfd2f41fe5bc8c672a51bcf3050c634927b84 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 22 Aug 2023 13:47:04 +0200
Subject: [PATCH 05/20] fix: remove event_handlers

---
 components/chainhook-cli/src/config/mod.rs               | 1 -
 components/chainhook-sdk/src/indexer/fork_scratch_pad.rs | 4 ++--
 components/chainhook-sdk/src/observer/mod.rs             | 8 --------
 components/chainhook-sdk/src/observer/tests/mod.rs       | 1 -
 4 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs
index 039ff97fb..7db407523 100644
--- a/components/chainhook-cli/src/config/mod.rs
+++ b/components/chainhook-cli/src/config/mod.rs
@@ -109,7 +109,6 @@ impl Config {
     pub fn get_event_observer_config(&self) -> EventObserverConfig {
         EventObserverConfig {
             bitcoin_rpc_proxy_enabled: true,
-            event_handlers: vec![],
             chainhook_config: None,
             ingestion_port: DEFAULT_INGESTION_PORT,
             bitcoind_rpc_username: self.network.bitcoind_rpc_username.clone(),
diff --git a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
index 0c040c134..ca32bba7c 100644
--- a/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
+++ b/components/chainhook-sdk/src/indexer/fork_scratch_pad.rs
@@ -31,9 +31,9 @@ impl ForkScratchPad {
 
     pub fn can_process_header(&self, header: &BlockHeader) -> bool {
         if self.headers_store.is_empty() {
-            return true
+            return true;
         }
-        
+
         self.headers_store
             .contains_key(&header.parent_block_identifier)
     }
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index a976a1bcd..df23772a7 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -1123,8 +1123,6 @@ pub async fn start_observer_commands_handler(
                     }
                 };
 
-                for event_handler in event_handlers.iter() {
-                    event_handler.propagate_bitcoin_event(&chain_event).await;
                 }
                 // process hooks
                 let mut hooks_ids_to_deregister = vec![];
@@ -1275,9 +1273,6 @@ pub async fn start_observer_commands_handler(
                 ctx.try_log(|logger| {
                     slog::info!(logger, "Handling PropagateStacksChainEvent command")
                 });
-                for event_handler in event_handlers.iter() {
-                    event_handler.propagate_stacks_event(&chain_event).await;
-                }
                 let mut hooks_ids_to_deregister = vec![];
                 let mut requests = vec![];
                 let mut report = PredicateEvaluationReport::new();
@@ -1481,9 +1476,6 @@ pub async fn start_observer_commands_handler(
                 ctx.try_log(|logger| {
                     slog::info!(logger, "Handling NotifyBitcoinTransactionProxied command")
                 });
-                for event_handler in event_handlers.iter() {
-                    event_handler.notify_bitcoin_transaction_proxied().await;
-                }
                 if let Some(ref tx) = observer_events_tx {
                     let _ = tx.send(ObserverEvent::NotifyBitcoinTransactionProxied);
                 }
diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs
index 8d4b3a7f1..72a15cdd7 100644
--- a/components/chainhook-sdk/src/observer/tests/mod.rs
+++ b/components/chainhook-sdk/src/observer/tests/mod.rs
@@ -30,7 +30,6 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) {
     let config: EventObserverConfig = EventObserverConfig {
         chainhook_config: Some(ChainhookConfig::new()),
         bitcoin_rpc_proxy_enabled: false,
-        event_handlers: vec![],
         ingestion_port: 0,
         bitcoind_rpc_username: "user".into(),
         bitcoind_rpc_password: "user".into(),

From 7cafaf318447af6c2293df5aaff2646c9da1d99f Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 22 Aug 2023 13:47:25 +0200
Subject: [PATCH 06/20] chore: dead code - BitcoinBlockPool

---
 .../src/indexer/bitcoin/blocks_pool.rs        | 366 ------------------
 .../chainhook-sdk/src/indexer/bitcoin/mod.rs  |   3 -
 2 files changed, 369 deletions(-)
 delete mode 100644 components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs

diff --git a/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs b/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs
deleted file mode 100644
index 2c0c8e130..000000000
--- a/components/chainhook-sdk/src/indexer/bitcoin/blocks_pool.rs
+++ /dev/null
@@ -1,366 +0,0 @@
-use crate::{
-    indexer::{ChainSegment, ChainSegmentIncompatibility},
-    utils::Context,
-};
-use chainhook_types::{
-    BitcoinBlockData, BitcoinChainEvent, BitcoinChainUpdatedWithBlocksData,
-    BitcoinChainUpdatedWithReorgData, BlockIdentifier,
-};
-use hiro_system_kit::slog;
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
-
-pub struct BitcoinBlockPool {
-    canonical_fork_id: usize,
-    orphans: BTreeSet<BlockIdentifier>,
-    block_store: HashMap<BlockIdentifier, BitcoinBlockData>,
-    forks: BTreeMap<usize, ChainSegment>,
-}
-
-impl BitcoinBlockPool {
-    pub fn new() -> BitcoinBlockPool {
-        let mut forks = BTreeMap::new();
-        forks.insert(0, ChainSegment::new());
-        BitcoinBlockPool {
-            canonical_fork_id: 0,
-            block_store: HashMap::new(),
-            orphans: BTreeSet::new(),
-            forks,
-        }
-    }
-
-    pub fn process_block(
-        &mut self,
-        block: BitcoinBlockData,
-        ctx: &Context,
-    ) -> Result<Option<BitcoinChainEvent>, String> {
-        ctx.try_log(|logger| {
-            slog::info!(
-                logger,
-                "Start processing Bitcoin {}",
-                block.block_identifier
-            )
-        });
-
-        // Keep block data in memory
-        let existing_entry = self
-            .block_store
-            .insert(block.block_identifier.clone(), block.clone());
-        if existing_entry.is_some() {
-            ctx.try_log(|logger| {
-                slog::warn!(
-                    logger,
-                    "Bitcoin {} has already been processed",
-                    block.block_identifier
-                )
-            });
-            return Ok(None);
-        }
-
-        for (i, fork) in self.forks.iter() {
-            ctx.try_log(|logger| slog::info!(logger, "Active fork {}: {}", i, fork));
-        }
-        // Retrieve previous canonical fork
-        let previous_canonical_fork_id = self.canonical_fork_id;
-        let previous_canonical_fork = match self.forks.get(&previous_canonical_fork_id) {
-            Some(fork) => fork.clone(),
-            None => {
-                ctx.try_log(|logger| {
-                    slog::error!(logger, "unable to retrieve previous bitcoin fork")
-                });
-                return Ok(None);
-            }
-        };
-
-        let mut fork_updated = None;
-        for (_, fork) in self.forks.iter_mut() {
-            let (block_appended, mut new_fork) = fork.try_append_block(&block, ctx);
-            if block_appended {
-                if let Some(new_fork) = new_fork.take() {
-                    let fork_id = self.forks.len();
-                    self.forks.insert(fork_id, new_fork);
-                    fork_updated = self.forks.get_mut(&fork_id);
-                } else {
-                    fork_updated = Some(fork);
-                }
-                // A block can only be added to one segment
-                break;
-            }
-        }
-
-        let fork_updated = match fork_updated.take() {
-            Some(fork) => {
-                ctx.try_log(|logger| {
-                    slog::debug!(
-                        logger,
-                        "Bitcoin {} successfully appended to {}",
-                        block.block_identifier,
-                        fork
-                    )
-                });
-                fork
-            }
-            None => {
-                ctx.try_log(|logger| {
-                    slog::debug!(
-                        logger,
-                        "Unable to process Bitcoin {} - inboxed for later",
-                        block.block_identifier
-                    )
-                });
-                self.orphans.insert(block.block_identifier.clone());
-                return Ok(None);
-            }
-        };
-
-        // Process former orphans
-        let orphans = self.orphans.clone();
-        let mut orphans_to_untrack = HashSet::new();
-
-        let mut at_least_one_orphan_appended = true;
-        // As long as we are successful appending blocks that were previously unprocessable,
-        // Keep looping on this backlog
-        let mut applied = HashSet::new();
-        let mut forks_created = vec![];
-        while at_least_one_orphan_appended {
-            at_least_one_orphan_appended = false;
-            for orphan_block_identifier in orphans.iter() {
-                if applied.contains(orphan_block_identifier) {
-                    continue;
-                }
-                let block = match self.block_store.get(orphan_block_identifier) {
-                    Some(block) => block.clone(),
-                    None => continue,
-                };
-
-                let (orphan_appended, mut new_fork) = fork_updated.try_append_block(&block, ctx);
-                if orphan_appended {
-                    applied.insert(orphan_block_identifier);
-                    orphans_to_untrack.insert(orphan_block_identifier);
-                    if let Some(new_fork) = new_fork.take() {
-                        forks_created.push(new_fork);
-                    }
-                }
-                at_least_one_orphan_appended = at_least_one_orphan_appended || orphan_appended;
-            }
-        }
-
-        // Update orphans
-        for orphan in orphans_to_untrack.into_iter() {
-            ctx.try_log(|logger| slog::info!(logger, "Dequeuing orphan {}", orphan));
-            self.orphans.remove(orphan);
-        }
-
-        // Select canonical fork
-        let mut canonical_fork_id = 0;
-        let mut highest_height = 0;
-        for (fork_id, fork) in self.forks.iter() {
-            ctx.try_log(|logger| slog::info!(logger, "Active fork: {} - {}", fork_id, fork));
-            if fork.get_length() >= highest_height {
-                highest_height = fork.get_length();
-                canonical_fork_id = *fork_id;
-            }
-        }
-        ctx.try_log(|logger| {
-            slog::info!(
-                logger,
-                "Active fork selected as canonical: {}",
-                canonical_fork_id
-            )
-        });
-
-        self.canonical_fork_id = canonical_fork_id;
-        // Generate chain event from the previous and current canonical forks
-        let canonical_fork = self.forks.get(&canonical_fork_id).unwrap().clone();
-        if canonical_fork.eq(&previous_canonical_fork) {
-            ctx.try_log(|logger| slog::info!(logger, "Canonical fork unchanged"));
-            return Ok(None);
-        }
-
-        let res = self.generate_block_chain_event(&canonical_fork, &previous_canonical_fork, ctx);
-        let mut chain_event = match res {
-            Ok(chain_event) => chain_event,
-            Err(ChainSegmentIncompatibility::ParentBlockUnknown) => {
-                self.canonical_fork_id = previous_canonical_fork_id;
-                return Ok(None);
-            }
-            _ => return Ok(None),
-        };
-
-        self.collect_and_prune_confirmed_blocks(&mut chain_event, ctx);
-
-        Ok(Some(chain_event))
-    }
-
-    pub fn collect_and_prune_confirmed_blocks(
-        &mut self,
-        chain_event: &mut BitcoinChainEvent,
-        ctx: &Context,
-    ) {
-        let (tip, confirmed_blocks) = match chain_event {
-            BitcoinChainEvent::ChainUpdatedWithBlocks(ref mut event) => {
-                match event.new_blocks.last() {
-                    Some(tip) => (tip.block_identifier.clone(), &mut event.confirmed_blocks),
-                    None => return,
-                }
-            }
-            BitcoinChainEvent::ChainUpdatedWithReorg(ref mut event) => {
-                match event.blocks_to_apply.last() {
-                    Some(tip) => (tip.block_identifier.clone(), &mut event.confirmed_blocks),
-                    None => return,
-                }
-            }
-        };
-
-        let mut forks_to_prune = vec![];
-        let mut ancestor_identifier = &tip;
-
-        // Retrieve the whole canonical segment present in memory, ascending order
-        // [1] ... [6] [7]
-        let canonical_segment = {
-            let mut segment = vec![];
-            while let Some(ancestor) = self.block_store.get(&ancestor_identifier) {
-                ancestor_identifier = &ancestor.parent_block_identifier;
-                segment.push(ancestor.block_identifier.clone());
-            }
-            segment
-        };
-        if canonical_segment.len() < 7 {
-            return;
-        }
-        // Any block beyond 6th ancestor is considered as confirmed and can be pruned
-        let cut_off = &canonical_segment[5];
-
-        // Prune forks using the confirmed block
-        let mut blocks_to_prune = vec![];
-        for (fork_id, fork) in self.forks.iter_mut() {
-            let mut res = fork.prune_confirmed_blocks(&cut_off);
-            blocks_to_prune.append(&mut res);
-            if fork.block_ids.is_empty() {
-                forks_to_prune.push(*fork_id);
-            }
-        }
-
-        // Prune orphans using the confirmed block
-        let iter = self.orphans.clone().into_iter();
-        for orphan in iter {
-            if orphan.index < cut_off.index {
-                self.orphans.remove(&orphan);
-                blocks_to_prune.push(orphan);
-            }
-        }
-
-        for confirmed_block in canonical_segment[6..].into_iter() {
-            let block = match self.block_store.remove(confirmed_block) {
-                None => {
-                    ctx.try_log(|logger| {
-                        slog::error!(logger, "unable to retrieve data for {}", confirmed_block)
-                    });
-                    return;
-                }
-                Some(block) => block,
-            };
-            confirmed_blocks.push(block);
-        }
-
-        // Prune data
-        for block_to_prune in blocks_to_prune {
-            self.block_store.remove(&block_to_prune);
-        }
-        for fork_id in forks_to_prune {
-            self.forks.remove(&fork_id);
-        }
-        confirmed_blocks.reverse();
-    }
-
-    pub fn generate_block_chain_event(
-        &mut self,
-        canonical_segment: &ChainSegment,
-        other_segment: &ChainSegment,
-        ctx: &Context,
-    ) -> Result<BitcoinChainEvent, ChainSegmentIncompatibility> {
-        if other_segment.is_empty() {
-            let mut new_blocks = vec![];
-            for i in 0..canonical_segment.block_ids.len() {
-                let block_identifier =
-                    &canonical_segment.block_ids[canonical_segment.block_ids.len() - 1 - i];
-                let block = match self.block_store.get(block_identifier) {
-                    Some(block) => block.clone(),
-                    None => {
-                        ctx.try_log(|logger| {
-                            slog::error!(
-                                logger,
-                                "unable to retrive Bitcoin {} from block store",
-                                block_identifier
-                            )
-                        });
-                        return Err(ChainSegmentIncompatibility::Unknown);
-                    }
-                };
-                new_blocks.push(block)
-            }
-            return Ok(BitcoinChainEvent::ChainUpdatedWithBlocks(
-                BitcoinChainUpdatedWithBlocksData {
-                    new_blocks,
-                    confirmed_blocks: vec![],
-                },
-            ));
-        }
-        if let Ok(divergence) = canonical_segment.try_identify_divergence(other_segment, false, ctx)
-        {
-            if divergence.block_ids_to_rollback.is_empty() {
-                let mut new_blocks = vec![];
-                for i in 0..divergence.block_ids_to_apply.len() {
-                    let block_identifier = &divergence.block_ids_to_apply[i];
-                    let block = match self.block_store.get(block_identifier) {
-                        Some(block) => block.clone(),
-                        None => panic!("unable to retrive block from block store"),
-                    };
-                    new_blocks.push(block)
-                }
-                return Ok(BitcoinChainEvent::ChainUpdatedWithBlocks(
-                    BitcoinChainUpdatedWithBlocksData {
-                        new_blocks,
-                        confirmed_blocks: vec![],
-                    },
-                ));
-            } else {
-                return Ok(BitcoinChainEvent::ChainUpdatedWithReorg(
-                    BitcoinChainUpdatedWithReorgData {
-                        blocks_to_rollback: divergence
-                            .block_ids_to_rollback
-                            .iter()
-                            .map(|block_id| {
-                                let block = match self.block_store.get(block_id) {
-                                    Some(block) => block.clone(),
-                                    None => panic!("unable to retrive block from block store"),
-                                };
-                                block
-                            })
-                            .collect::<Vec<_>>(),
-                        blocks_to_apply: divergence
-                            .block_ids_to_apply
-                            .iter()
-                            .map(|block_id| {
-                                let block = match self.block_store.get(block_id) {
-                                    Some(block) => block.clone(),
-                                    None => panic!("unable to retrive block from block store"),
-                                };
-                                block
-                            })
-                            .collect::<Vec<_>>(),
-                        confirmed_blocks: vec![],
-                    },
-                ));
-            }
-        }
-        ctx.try_log(|logger| {
-            slog::debug!(
-                logger,
-                "Unable to infer chain event out of {} and {}",
-                canonical_segment,
-                other_segment
-            )
-        });
-        Err(ChainSegmentIncompatibility::ParentBlockUnknown)
-    }
-}
diff --git a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs
index a6014f4a8..3bdf00718 100644
--- a/components/chainhook-sdk/src/indexer/bitcoin/mod.rs
+++ b/components/chainhook-sdk/src/indexer/bitcoin/mod.rs
@@ -1,5 +1,3 @@
-mod blocks_pool;
-
 use std::time::Duration;
 
 use crate::chainhooks::types::{
@@ -11,7 +9,6 @@ use crate::utils::Context;
 use bitcoincore_rpc::bitcoin::hashes::Hash;
 use bitcoincore_rpc::bitcoin::{self, Address, Amount, BlockHash};
 use bitcoincore_rpc_json::GetRawTransactionResultVoutScriptPubKey;
-pub use blocks_pool::BitcoinBlockPool;
 use chainhook_types::bitcoin::{OutPoint, TxIn, TxOut};
 use chainhook_types::{
     BitcoinBlockData, BitcoinBlockMetadata, BitcoinNetwork, BitcoinTransactionData,

From 7006e0faeef728250d512ffcd4923a27754eef54 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 22 Aug 2023 13:49:52 +0200
Subject: [PATCH 07/20] test: revisit sidecar implem

---
 components/chainhook-sdk/src/observer/mod.rs  | 263 +++++++---------
 .../chainhook-sdk/src/observer/tests/mod.rs   | 296 +++++++++++++++++-
 2 files changed, 403 insertions(+), 156 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index df23772a7..acd516155 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -28,7 +28,6 @@ use chainhook_types::{
 };
 use hiro_system_kit;
 use hiro_system_kit::slog;
-use reqwest::Client as HttpClient;
 use rocket::config::{self, Config, LogLevel};
 use rocket::data::{Limits, ToByteUnit};
 use rocket::serde::Deserialize;
@@ -41,7 +40,7 @@ use std::str;
 use std::str::FromStr;
 use std::sync::mpsc::{Receiver, Sender};
 use std::sync::{Arc, Mutex, RwLock};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{SystemTime, UNIX_EPOCH};
 #[cfg(feature = "zeromq")]
 use zeromq::{Socket, SocketRecv};
 
@@ -61,64 +60,10 @@ pub enum Event {
     StacksChainEvent(StacksChainEvent),
 }
 
-// TODO(lgalabru): Support for GRPC?
-#[derive(Deserialize, Debug, Clone)]
-pub enum EventHandler {
-    WebHook(String),
-}
-
-impl EventHandler {
-    async fn propagate_stacks_event(&self, stacks_event: &StacksChainEvent) {
-        match self {
-            EventHandler::WebHook(host) => {
-                let path = "chain-events/stacks";
-                let url = format!("{}/{}", host, path);
-                let body = rocket::serde::json::serde_json::to_vec(&stacks_event).unwrap_or(vec![]);
-                let http_client = HttpClient::builder()
-                    .timeout(Duration::from_secs(20))
-                    .build()
-                    .expect("Unable to build http client");
-                let _ = http_client
-                    .post(url)
-                    .header("Content-Type", "application/json")
-                    .body(body)
-                    .send()
-                    .await;
-                // TODO(lgalabru): handle response errors
-            }
-        }
-    }
-
-    async fn propagate_bitcoin_event(&self, bitcoin_event: &BitcoinChainEvent) {
-        match self {
-            EventHandler::WebHook(host) => {
-                let path = "chain-events/bitcoin";
-                let url = format!("{}/{}", host, path);
-                let body =
-                    rocket::serde::json::serde_json::to_vec(&bitcoin_event).unwrap_or(vec![]);
-                let http_client = HttpClient::builder()
-                    .timeout(Duration::from_secs(20))
-                    .build()
-                    .expect("Unable to build http client");
-                let _res = http_client
-                    .post(url)
-                    .header("Content-Type", "application/json")
-                    .body(body)
-                    .send()
-                    .await;
-                // TODO(lgalabru): handle response errors
-            }
-        }
-    }
-
-    async fn notify_bitcoin_transaction_proxied(&self) {}
-}
-
 #[derive(Debug, Clone)]
 pub struct EventObserverConfig {
     pub chainhook_config: Option<ChainhookConfig>,
     pub bitcoin_rpc_proxy_enabled: bool,
-    pub event_handlers: Vec<EventHandler>,
     pub ingestion_port: u16,
     pub bitcoind_rpc_username: String,
     pub bitcoind_rpc_password: String,
@@ -188,7 +133,6 @@ impl EventObserverConfig {
 
         let config = EventObserverConfig {
             bitcoin_rpc_proxy_enabled: false,
-            event_handlers: vec![],
             chainhook_config: None,
             ingestion_port: overrides
                 .and_then(|c| c.ingestion_port)
@@ -392,12 +336,69 @@ pub struct ObserverMetrics {
     pub stacks: ChainMetrics,
 }
 
+pub struct ObserverSidecar {
+    bitcoin_block_mutator: Option<(Sender<BitcoinBlockData>, Receiver<BitcoinBlockData>)>,
+    bitcoin_chain_event_notifier: Option<Sender<HandleBlock>>,
+}
+
+impl ObserverSidecar {
+    fn perform_bitcoin_sidecar_mutation(
+        &self,
+        block: BitcoinBlockData,
+        ctx: &Context,
+    ) -> (bool, BitcoinBlockData) {
+        if let Some(ref block_mutator) = self.bitcoin_block_mutator {
+            ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",));
+            let _ = block_mutator.0.send(block.clone());
+            ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",));
+            match block_mutator.1.recv() {
+                Ok(updated_block) => {
+                    ctx.try_log(|logger| slog::info!(logger, "Block received from pre-processor",));
+                    (true, updated_block)
+                }
+                Err(e) => {
+                    ctx.try_log(|logger| {
+                        slog::error!(
+                            logger,
+                            "Unable to receive block from pre-processor {}",
+                            e.to_string()
+                        )
+                    });
+                    (false, block)
+                }
+            }
+        } else {
+            (false, block)
+        }
+    }
+
+    fn notify_chain_event(&self, chain_event: &BitcoinChainEvent, _ctx: &Context) {
+        if let Some(ref notifier) = self.bitcoin_chain_event_notifier {
+            match chain_event {
+                BitcoinChainEvent::ChainUpdatedWithBlocks(data) => {
+                    for block in data.new_blocks.iter() {
+                        let _ = notifier.send(HandleBlock::ApplyBlock(block.clone()));
+                    }
+                }
+                BitcoinChainEvent::ChainUpdatedWithReorg(data) => {
+                    for block in data.blocks_to_rollback.iter() {
+                        let _ = notifier.send(HandleBlock::UndoBlock(block.clone()));
+                    }
+                    for block in data.blocks_to_apply.iter() {
+                        let _ = notifier.send(HandleBlock::ApplyBlock(block.clone()));
+                    }
+                }
+            }
+        }
+    }
+}
+
 pub fn start_event_observer(
     config: EventObserverConfig,
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
+    observer_sidecar: Option<ObserverSidecar>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     match config.bitcoin_block_signaling {
@@ -414,7 +415,7 @@ pub fn start_event_observer(
                     observer_commands_tx_moved,
                     observer_commands_rx,
                     observer_events_tx,
-                    block_pre_processor,
+                    observer_sidecar,
                     context_cloned,
                 );
                 let _ = hiro_system_kit::nestable_block_on(future);
@@ -431,7 +432,7 @@ pub fn start_event_observer(
                     observer_commands_tx_moved,
                     observer_commands_rx,
                     observer_events_tx,
-                    block_pre_processor,
+                    observer_sidecar,
                     context_cloned,
                 );
                 let _ = hiro_system_kit::nestable_block_on(future);
@@ -458,7 +459,7 @@ pub async fn start_bitcoin_event_observer(
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
+    observer_sidecar: Option<ObserverSidecar>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let chainhook_store = ChainhookStore::new();
@@ -485,7 +486,7 @@ pub async fn start_bitcoin_event_observer(
         observer_events_tx,
         None,
         observer_metrics_rw_lock.clone(),
-        block_pre_processor,
+        observer_sidecar,
         ctx,
     )
     .await
@@ -496,7 +497,7 @@ pub async fn start_stacks_event_observer(
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
+    observer_sidecar: Option<ObserverSidecar>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let indexer_config = IndexerConfig {
@@ -613,7 +614,7 @@ pub async fn start_stacks_event_observer(
         observer_events_tx,
         ingestion_shutdown,
         observer_metrics_rw_lock.clone(),
-        block_pre_processor,
+        observer_sidecar,
         ctx,
     )
     .await
@@ -765,8 +766,7 @@ pub fn start_zeromq_runloop(
                                         //       \ B2 (4) - C2 (5) - D2 (6)
                                         // When D2 is being discovered (making A -> B2 -> C2 -> D2 the new canonical fork)
                                         // it looks like ZMQ is only publishing D2.
-                                        // Without additional operation, we end up with a block that with a block that we
-                                        // can't handle.
+                                        // Without additional operation, we end up with a block that we can't append.
                                         let parent_block_hash = header.parent_block_identifier.get_hash_bytes_str().to_string();
                                         ctx_moved.try_log(|logger| {
                                             slog::info!(
@@ -785,12 +785,6 @@ pub fn start_zeromq_runloop(
     }
 }
 
-pub fn pre_process_bitcoin_block() {}
-
-pub fn apply_bitcoin_block() {}
-
-pub fn rollback_bitcoin_block() {}
-
 pub fn gather_proofs<'a>(
     trigger: &BitcoinTriggerChainhook<'a>,
     proofs: &mut HashMap<&'a TransactionIdentifier, String>,
@@ -834,8 +828,8 @@ pub fn gather_proofs<'a>(
 }
 
 pub enum HandleBlock {
-    ApplyBlocks(BitcoinBlockData),
-    UndoBlocks(BitcoinBlockData),
+    ApplyBlock(BitcoinBlockData),
+    UndoBlock(BitcoinBlockData),
 }
 
 pub async fn start_observer_commands_handler(
@@ -845,13 +839,13 @@ pub async fn start_observer_commands_handler(
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
     ingestion_shutdown: Option<Shutdown>,
     observer_metrics: Arc<RwLock<ObserverMetrics>>,
-    block_pre_processor: Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
+    observer_sidecar: Option<ObserverSidecar>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
     let mut chainhooks_occurrences_tracker: HashMap<String, u64> = HashMap::new();
-    let event_handlers = config.event_handlers.clone();
     let networks = (&config.bitcoin_network, &config.stacks_network);
-    let mut bitcoin_block_store: HashMap<BlockIdentifier, BitcoinBlockData> = HashMap::new();
+    let mut bitcoin_block_store: HashMap<BlockIdentifier, (BitcoinBlockData, bool)> =
+        HashMap::new();
     let http_client = build_http_client();
 
     loop {
@@ -929,10 +923,10 @@ pub async fn start_observer_commands_handler(
                         slog::warn!(logger, "unable to acquire observer_metrics_rw_lock:{}", e)
                     }),
                 };
-                bitcoin_block_store.insert(new_block.block_identifier.clone(), new_block);
+                bitcoin_block_store.insert(new_block.block_identifier.clone(), (new_block, false));
             }
             ObserverCommand::CacheBitcoinBlock(block) => {
-                bitcoin_block_store.insert(block.block_identifier.clone(), block);
+                bitcoin_block_store.insert(block.block_identifier.clone(), (block, false));
             }
             ObserverCommand::PropagateBitcoinChainEvent(blockchain_event) => {
                 ctx.try_log(|logger| {
@@ -946,23 +940,28 @@ pub async fn start_observer_commands_handler(
                         let mut new_blocks = vec![];
 
                         for header in data.new_headers.iter() {
-                            match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some(block) => {
+                            match (
+                                bitcoin_block_store.remove(&header.block_identifier),
+                                &observer_sidecar,
+                            ) {
+                                (Some((block, false)), Some(sidecar)) => {
                                     // Time for pre-processing
-                                    let (_updated, updated_block) = handle_block_pre_processing(
-                                        &block_pre_processor,
-                                        block,
-                                        true,
-                                        &ctx,
-                                    );
-                                    // Keep a copy of the pre-processed version
+                                    let (updated, updated_block) =
+                                        sidecar.perform_bitcoin_sidecar_mutation(block, &ctx);
                                     bitcoin_block_store.insert(
                                         updated_block.block_identifier.clone(),
-                                        updated_block.clone(),
+                                        (updated_block.clone(), updated),
                                     );
                                     new_blocks.push(updated_block);
                                 }
-                                None => {
+                                (Some((block, _)), _) => {
+                                    bitcoin_block_store.insert(
+                                        block.block_identifier.clone(),
+                                        (block.clone(), true),
+                                    );
+                                    new_blocks.push(block);
+                                }
+                                (None, _) => {
                                     ctx.try_log(|logger| {
                                         slog::error!(
                                             logger,
@@ -976,7 +975,7 @@ pub async fn start_observer_commands_handler(
 
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some(block) => {
+                                Some((block, _)) => {
                                     confirmed_blocks.push(block);
                                 }
                                 None => {
@@ -1018,21 +1017,9 @@ pub async fn start_observer_commands_handler(
                         });
 
                         for header in data.headers_to_rollback.iter() {
-                            match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some(block) => {
-                                    // Time for pre-processing
-                                    let (_updated, updated_block) = handle_block_pre_processing(
-                                        &block_pre_processor,
-                                        block,
-                                        false,
-                                        &ctx,
-                                    );
-                                    // Keep a copy of the pre-processed version
-                                    bitcoin_block_store.insert(
-                                        updated_block.block_identifier.clone(),
-                                        updated_block.clone(),
-                                    );
-                                    blocks_to_rollback.push(updated_block);
+                            match bitcoin_block_store.get(&header.block_identifier) {
+                                Some((block, _)) => {
+                                    blocks_to_rollback.push(block.clone());
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -1047,23 +1034,28 @@ pub async fn start_observer_commands_handler(
                         }
 
                         for header in data.headers_to_apply.iter() {
-                            match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some(block) => {
+                            match (
+                                bitcoin_block_store.remove(&header.block_identifier),
+                                &observer_sidecar,
+                            ) {
+                                (Some((block, false)), Some(sidecar)) => {
                                     // Time for pre-processing
-                                    let (_updated, updated_block) = handle_block_pre_processing(
-                                        &block_pre_processor,
-                                        block,
-                                        true,
-                                        &ctx,
-                                    );
-                                    // Keep a copy of the pre-processed version
+                                    let (updated, updated_block) =
+                                        sidecar.perform_bitcoin_sidecar_mutation(block, &ctx);
                                     bitcoin_block_store.insert(
                                         updated_block.block_identifier.clone(),
-                                        updated_block.clone(),
+                                        (updated_block.clone(), updated),
                                     );
                                     blocks_to_apply.push(updated_block);
                                 }
-                                None => {
+                                (Some((block, _)), _) => {
+                                    bitcoin_block_store.insert(
+                                        block.block_identifier.clone(),
+                                        (block.clone(), true),
+                                    );
+                                    blocks_to_apply.push(block);
+                                }
+                                (None, _) => {
                                     ctx.try_log(|logger| {
                                         slog::error!(
                                             logger,
@@ -1077,7 +1069,7 @@ pub async fn start_observer_commands_handler(
 
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some(block) => {
+                                Some((block, _)) => {
                                     confirmed_blocks.push(block);
                                 }
                                 None => {
@@ -1123,6 +1115,8 @@ pub async fn start_observer_commands_handler(
                     }
                 };
 
+                if let Some(ref sidecar) = observer_sidecar {
+                    sidecar.notify_chain_event(&chain_event, &ctx)
                 }
                 // process hooks
                 let mut hooks_ids_to_deregister = vec![];
@@ -1571,38 +1565,5 @@ pub async fn start_observer_commands_handler(
     Ok(())
 }
 
-fn handle_block_pre_processing(
-    block_pre_processor: &Option<(Sender<HandleBlock>, Receiver<BitcoinBlockData>)>,
-    block: BitcoinBlockData,
-    apply: bool,
-    ctx: &Context,
-) -> (bool, BitcoinBlockData) {
-    if let Some(ref processor) = block_pre_processor {
-        ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",));
-        let _ = processor.0.send(match apply {
-            true => HandleBlock::ApplyBlocks(block.clone()),
-            false => HandleBlock::UndoBlocks(block.clone()),
-        });
-        ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",));
-        match processor.1.recv() {
-            Ok(updated_block) => {
-                ctx.try_log(|logger| slog::info!(logger, "Block received from pre-processor",));
-                return (true, updated_block);
-            }
-            Err(e) => {
-                ctx.try_log(|logger| {
-                    slog::error!(
-                        logger,
-                        "Unable to receive block from pre-processor {}",
-                        e.to_string()
-                    )
-                });
-                return (false, block);
-            }
-        }
-    }
-    return (false, block);
-}
-
 #[cfg(test)]
 pub mod tests;
diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs
index 72a15cdd7..2c2353878 100644
--- a/components/chainhook-sdk/src/observer/tests/mod.rs
+++ b/components/chainhook-sdk/src/observer/tests/mod.rs
@@ -2,22 +2,24 @@ use crate::chainhooks::types::{
     BitcoinChainhookFullSpecification, BitcoinChainhookNetworkSpecification,
     BitcoinChainhookSpecification, BitcoinPredicateType, ChainhookConfig,
     ChainhookFullSpecification, ChainhookSpecification, ExactMatchingRule, HookAction,
-    OutputPredicate, StacksChainhookFullSpecification, StacksChainhookNetworkSpecification,
-    StacksChainhookSpecification, StacksContractCallBasedPredicate, StacksPredicate,
+    OrdinalOperations, OutputPredicate, StacksChainhookFullSpecification,
+    StacksChainhookNetworkSpecification, StacksChainhookSpecification,
+    StacksContractCallBasedPredicate, StacksPredicate,
 };
+use crate::indexer::fork_scratch_pad::ForkScratchPad;
 use crate::indexer::tests::helpers::transactions::generate_test_tx_bitcoin_p2pkh_transfer;
 use crate::indexer::tests::helpers::{
     accounts, bitcoin_blocks, stacks_blocks, transactions::generate_test_tx_stacks_contract_call,
 };
 use crate::observer::{
     start_observer_commands_handler, ChainhookStore, EventObserverConfig, ObserverCommand,
-    ObserverMetrics,
+    ObserverMetrics, ObserverSidecar,
 };
 use crate::utils::{AbstractBlock, Context};
 use chainhook_types::{
     BitcoinBlockSignaling, BitcoinNetwork, BlockchainEvent, BlockchainUpdatedWithHeaders,
-    StacksBlockUpdate, StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork,
-    StacksNodeConfig,
+    OrdinalInscriptionRevealData, OrdinalOperation, StacksBlockUpdate, StacksChainEvent,
+    StacksChainUpdatedWithBlocksData, StacksNetwork, StacksNodeConfig, BitcoinChainEvent,
 };
 use hiro_system_kit;
 use std::collections::BTreeMap;
@@ -115,6 +117,34 @@ fn bitcoin_chainhook_p2pkh(
     spec
 }
 
+fn bitcoin_chainhook_ordinals(id: u8) -> BitcoinChainhookFullSpecification {
+    let mut networks = BTreeMap::new();
+    networks.insert(
+        BitcoinNetwork::Regtest,
+        BitcoinChainhookNetworkSpecification {
+            start_block: None,
+            end_block: None,
+            blocks: None,
+            expire_after_occurrence: None,
+            predicate: BitcoinPredicateType::OrdinalsProtocol(OrdinalOperations::InscriptionFeed),
+            action: HookAction::Noop,
+            include_proof: None,
+            include_inputs: None,
+            include_outputs: None,
+            include_witness: None,
+        },
+    );
+
+    let spec = BitcoinChainhookFullSpecification {
+        uuid: format!("{}", id),
+        name: format!("Chainhook {}", id),
+        owner_uuid: None,
+        version: 1,
+        networks,
+    };
+    spec
+}
+
 fn generate_and_register_new_stacks_chainhook(
     observer_commands_tx: &Sender<ObserverCommand>,
     observer_events_rx: &crossbeam_channel::Receiver<ObserverEvent>,
@@ -201,6 +231,37 @@ fn generate_and_register_new_bitcoin_chainhook(
     chainhook
 }
 
+fn generate_and_register_new_ordinals_chainhook(
+    observer_commands_tx: &Sender<ObserverCommand>,
+    observer_events_rx: &crossbeam_channel::Receiver<ObserverEvent>,
+    id: u8,
+) -> BitcoinChainhookSpecification {
+    let chainhook = bitcoin_chainhook_ordinals(id);
+    let _ = observer_commands_tx.send(ObserverCommand::RegisterPredicate(
+        ChainhookFullSpecification::Bitcoin(chainhook.clone()),
+    ));
+    let mut chainhook = chainhook
+        .into_selected_network_specification(&BitcoinNetwork::Regtest)
+        .unwrap();
+    chainhook.enabled = true;
+    let _ = observer_commands_tx.send(ObserverCommand::EnablePredicate(
+        ChainhookSpecification::Bitcoin(chainhook.clone()),
+    ));
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::PredicateRegistered(_)) => {
+            true
+        }
+        _ => false,
+    });
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::PredicateEnabled(_)) => {
+            true
+        }
+        _ => false,
+    });
+    chainhook
+}
+
 #[test]
 fn test_stacks_chainhook_register_deregister() {
     let (observer_commands_tx, observer_commands_rx) = channel();
@@ -1158,3 +1219,228 @@ fn test_bitcoin_chainhook_auto_deregister() {
     let _ = observer_commands_tx.send(ObserverCommand::Terminate);
     handle.join().expect("unable to terminate thread");
 }
+
+#[test]
+fn test_bitcoin_chainhook_through_reorg() {
+    let (observer_commands_tx, observer_commands_rx) = channel();
+    let (block_pre_processor_in_tx, block_pre_processor_in_rx) = channel();
+    let (block_pre_processor_out_tx, block_pre_processor_out_rx) = channel();
+
+    let (observer_events_tx, observer_events_rx) = crossbeam_channel::unbounded();
+    let observer_metrics_rw_lock = Arc::new(RwLock::new(ObserverMetrics::default()));
+    let observer_metrics_rw_lock_moved = observer_metrics_rw_lock.clone();
+
+    let empty_ctx = Context::empty();
+
+    let observer_sidecar = ObserverSidecar {
+        bitcoin_block_mutator: Some((block_pre_processor_in_tx, block_pre_processor_out_rx)),
+        bitcoin_chain_event_notifier: None,
+    };
+
+    let handle = std::thread::spawn(move || {
+        let (config, chainhook_store) = generate_test_config();
+        let _ = hiro_system_kit::nestable_block_on(start_observer_commands_handler(
+            config,
+            chainhook_store,
+            observer_commands_rx,
+            Some(observer_events_tx),
+            None,
+            observer_metrics_rw_lock_moved,
+            Some(observer_sidecar),
+            Context::empty(),
+        ));
+    });
+
+    // The block pre-processor will simulate block augmentation with new informations, which should trigger
+    // registered predicates
+    let block_pre_processor_handle = std::thread::spawn(move || {
+        let mut cursor: u64 = 0;
+        while let Ok(mut block) = block_pre_processor_in_rx.recv() {
+            for (tx_index, tx) in block.transactions.iter_mut().enumerate() {
+                cursor += 1;
+                tx.metadata
+                    .ordinal_operations
+                    .push(OrdinalOperation::InscriptionRevealed(
+                        OrdinalInscriptionRevealData {
+                            content_bytes: format!("{cursor}"),
+                            content_type: "".to_string(),
+                            content_length: cursor as usize,
+                            inscription_number: cursor as i64,
+                            inscription_fee: cursor,
+                            inscription_output_value: cursor,
+                            inscription_id: format!("{cursor}"),
+                            inscription_input_index: 0,
+                            inscriber_address: None,
+                            ordinal_number: cursor,
+                            ordinal_block_height: block.block_identifier.index,
+                            ordinal_offset: 0,
+                            tx_index,
+                            transfers_pre_inscription: cursor as u32,
+                            satpoint_post_inscription: format!("{cursor}"),
+                            curse_type: None,
+                        },
+                    ))
+            }
+            let _ = block_pre_processor_out_tx.send(block);
+        }
+    });
+
+    let genesis = bitcoin_blocks::generate_test_bitcoin_block(0, 1, vec![], None);
+    let mut fork_pad = ForkScratchPad::new();
+    let _ = fork_pad.process_header(genesis.get_header(), &empty_ctx);
+
+    // Create and register a new chainhook (wallet_2 received some sats)
+    let _chainhook =
+        generate_and_register_new_ordinals_chainhook(&observer_commands_tx, &observer_events_rx, 1);
+
+    // registering bitcoin chainhook should increment the observer_metric's registered bitcoin hooks
+    assert_eq!(
+        1,
+        observer_metrics_rw_lock
+            .read()
+            .unwrap()
+            .bitcoin
+            .registered_predicates
+    );
+
+    // Simulate a block that does not include a trigger (wallet_1 to wallet_3)
+    let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer(
+        0,
+        &accounts::wallet_1_btc_address(),
+        &accounts::wallet_3_btc_address(),
+        3,
+    )];
+    let block = bitcoin_blocks::generate_test_bitcoin_block(0, 2, transactions, None);
+    let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone()));
+
+    let chain_event = fork_pad
+        .process_header(block.get_header(), &empty_ctx)
+        .unwrap()
+        .unwrap();
+    let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event));
+    // Should signal that no hook were triggered
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::PredicatesTriggered(len)) => {
+            assert_eq!(len, 1);
+            true
+        }
+        Ok(event) => {
+            println!("Unexpected event: {:?}", event);
+            false
+        }
+        Err(e) => {
+            println!("Error: {:?}", e);
+            false
+        }
+    });
+
+    // 1) Should kick off predicate event
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => {
+            assert_eq!(payload.apply.len(), 1);
+            assert_eq!(payload.apply[0].block.transactions.len(), 1);
+            true
+        }
+        _ => false,
+    });
+
+    // 2) Should kick off bitcoin chain event
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithBlocks(_), _))) => {
+            true
+        }
+        _ => false,
+    });
+
+    // Simulate a block that does include a trigger (wallet_1 to wallet_2)
+    let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer(
+        0,
+        &accounts::wallet_1_btc_address(),
+        &accounts::wallet_2_btc_address(),
+        3,
+    )];
+    let block = bitcoin_blocks::generate_test_bitcoin_block(1, 2, transactions, None);
+    let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone()));
+    let chain_event = fork_pad
+        .process_header(block.get_header(), &empty_ctx)
+        .unwrap()
+        .unwrap();
+    let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event));
+
+    // Should signal that no hook were triggered
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::PredicatesTriggered(len)) => {
+            assert_eq!(len, 1);
+            true
+        }
+        _ => false,
+    });
+
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => {
+            assert_eq!(payload.rollback.len(), 1);
+            assert_eq!(payload.rollback[0].block.transactions.len(), 1);
+            assert_eq!(payload.apply.len(), 1);
+            assert_eq!(payload.apply[0].block.transactions.len(), 1);
+            true
+        }
+        _ => false,
+    });
+
+    // Should propagate chain event
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithReorg(_), _))) => {
+            true
+        }
+        _ => false,
+    });
+
+    // Simulate a block that does include a trigger (wallet_1 to wallet_2)
+    let transactions = vec![generate_test_tx_bitcoin_p2pkh_transfer(
+        0,
+        &accounts::wallet_1_btc_address(),
+        &accounts::wallet_2_btc_address(),
+        3,
+    )];
+    let block = bitcoin_blocks::generate_test_bitcoin_block(0, 3, transactions, None);
+    let _ = observer_commands_tx.send(ObserverCommand::CacheBitcoinBlock(block.clone()));
+    let chain_event = fork_pad
+        .process_header(block.get_header(), &empty_ctx)
+        .unwrap()
+        .unwrap();
+    let _ = observer_commands_tx.send(ObserverCommand::PropagateBitcoinChainEvent(chain_event));
+
+    // Should signal that no hook were triggered
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::PredicatesTriggered(len)) => {
+            assert_eq!(len, 1);
+            true
+        }
+        _ => false,
+    });
+
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinPredicateTriggered(payload)) => {
+            assert_eq!(payload.rollback.len(), 1);
+            assert_eq!(payload.rollback[0].block.transactions.len(), 1);
+            assert_eq!(payload.apply.len(), 2);
+            assert_eq!(payload.apply[0].block.transactions.len(), 1);
+            true
+        }
+        _ => false,
+    });
+
+    // Should propagate block
+    assert!(match observer_events_rx.recv() {
+        Ok(ObserverEvent::BitcoinChainEvent((BitcoinChainEvent::ChainUpdatedWithReorg(_), _))) => {
+            true
+        }
+        _ => false,
+    });
+
+    let _ = observer_commands_tx.send(ObserverCommand::Terminate);
+    handle.join().expect("unable to terminate thread");
+    block_pre_processor_handle
+        .join()
+        .expect("unable to terminate thread");
+}

From 67a34dcb2f7dab546bb88bd1a6ed098109953531 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 22 Aug 2023 16:49:33 +0200
Subject: [PATCH 08/20] fix: revisit approach

---
 components/chainhook-sdk/src/observer/mod.rs  | 202 +++++++++---------
 .../chainhook-sdk/src/observer/tests/mod.rs   |  63 +++---
 2 files changed, 136 insertions(+), 129 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index acd516155..6f73470b4 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -336,25 +336,37 @@ pub struct ObserverMetrics {
     pub stacks: ChainMetrics,
 }
 
+#[derive(Debug, Clone)]
+pub struct BitcoinBlockDataCached {
+    pub block: BitcoinBlockData,
+    pub processed_by_sidecar: bool,
+}
+
 pub struct ObserverSidecar {
-    bitcoin_block_mutator: Option<(Sender<BitcoinBlockData>, Receiver<BitcoinBlockData>)>,
-    bitcoin_chain_event_notifier: Option<Sender<HandleBlock>>,
+    pub bitcoin_blocks_mutator: Option<(
+        Sender<(Vec<BitcoinBlockDataCached>, Vec<BlockIdentifier>)>,
+        Receiver<Vec<BitcoinBlockDataCached>>,
+    )>,
+    pub bitcoin_chain_event_notifier: Option<Sender<HandleBlock>>,
 }
 
 impl ObserverSidecar {
-    fn perform_bitcoin_sidecar_mutation(
+    fn perform_bitcoin_sidecar_mutations(
         &self,
-        block: BitcoinBlockData,
+        blocks: Vec<BitcoinBlockDataCached>,
+        blocks_ids_to_rollback: Vec<BlockIdentifier>,
         ctx: &Context,
-    ) -> (bool, BitcoinBlockData) {
-        if let Some(ref block_mutator) = self.bitcoin_block_mutator {
+    ) -> Vec<BitcoinBlockDataCached> {
+        if let Some(ref block_mutator) = self.bitcoin_blocks_mutator {
             ctx.try_log(|logger| slog::info!(logger, "Sending blocks to pre-processor",));
-            let _ = block_mutator.0.send(block.clone());
+            let _ = block_mutator
+                .0
+                .send((blocks.clone(), blocks_ids_to_rollback));
             ctx.try_log(|logger| slog::info!(logger, "Waiting for blocks from pre-processor",));
             match block_mutator.1.recv() {
-                Ok(updated_block) => {
+                Ok(updated_blocks) => {
                     ctx.try_log(|logger| slog::info!(logger, "Block received from pre-processor",));
-                    (true, updated_block)
+                    updated_blocks
                 }
                 Err(e) => {
                     ctx.try_log(|logger| {
@@ -364,11 +376,11 @@ impl ObserverSidecar {
                             e.to_string()
                         )
                     });
-                    (false, block)
+                    blocks
                 }
             }
         } else {
-            (false, block)
+            blocks
         }
     }
 
@@ -844,9 +856,12 @@ pub async fn start_observer_commands_handler(
 ) -> Result<(), Box<dyn Error>> {
     let mut chainhooks_occurrences_tracker: HashMap<String, u64> = HashMap::new();
     let networks = (&config.bitcoin_network, &config.stacks_network);
-    let mut bitcoin_block_store: HashMap<BlockIdentifier, (BitcoinBlockData, bool)> =
-        HashMap::new();
+    let mut bitcoin_block_store: HashMap<BlockIdentifier, BitcoinBlockDataCached> = HashMap::new();
     let http_client = build_http_client();
+    let store_update_required = observer_sidecar
+        .as_ref()
+        .and_then(|s| s.bitcoin_blocks_mutator.as_ref())
+        .is_some();
 
     loop {
         let command = match observer_commands_rx.recv() {
@@ -872,7 +887,7 @@ pub async fn start_observer_commands_handler(
             }
             ObserverCommand::ProcessBitcoinBlock(mut block_data) => {
                 let block_hash = block_data.hash.to_string();
-                let new_block = loop {
+                let block = loop {
                     match standardize_bitcoin_block(
                         block_data.clone(),
                         &config.bitcoin_network,
@@ -910,8 +925,8 @@ pub async fn start_observer_commands_handler(
                 };
                 match observer_metrics.write() {
                     Ok(mut metrics) => {
-                        if new_block.block_identifier.index > metrics.bitcoin.tip_height {
-                            metrics.bitcoin.tip_height = new_block.block_identifier.index;
+                        if block.block_identifier.index > metrics.bitcoin.tip_height {
+                            metrics.bitcoin.tip_height = block.block_identifier.index;
                         }
                         metrics.bitcoin.last_block_ingestion_at = SystemTime::now()
                             .duration_since(UNIX_EPOCH)
@@ -923,10 +938,22 @@ pub async fn start_observer_commands_handler(
                         slog::warn!(logger, "unable to acquire observer_metrics_rw_lock:{}", e)
                     }),
                 };
-                bitcoin_block_store.insert(new_block.block_identifier.clone(), (new_block, false));
+                bitcoin_block_store.insert(
+                    block.block_identifier.clone(),
+                    BitcoinBlockDataCached {
+                        block,
+                        processed_by_sidecar: false,
+                    },
+                );
             }
             ObserverCommand::CacheBitcoinBlock(block) => {
-                bitcoin_block_store.insert(block.block_identifier.clone(), (block, false));
+                bitcoin_block_store.insert(
+                    block.block_identifier.clone(),
+                    BitcoinBlockDataCached {
+                        block,
+                        processed_by_sidecar: false,
+                    },
+                );
             }
             ObserverCommand::PropagateBitcoinChainEvent(blockchain_event) => {
                 ctx.try_log(|logger| {
@@ -937,46 +964,40 @@ pub async fn start_observer_commands_handler(
                 // Update Chain event before propagation
                 let chain_event = match blockchain_event {
                     BlockchainEvent::BlockchainUpdatedWithHeaders(data) => {
+                        let mut blocks_to_mutate = vec![];
                         let mut new_blocks = vec![];
 
                         for header in data.new_headers.iter() {
-                            match (
-                                bitcoin_block_store.remove(&header.block_identifier),
-                                &observer_sidecar,
-                            ) {
-                                (Some((block, false)), Some(sidecar)) => {
-                                    // Time for pre-processing
-                                    let (updated, updated_block) =
-                                        sidecar.perform_bitcoin_sidecar_mutation(block, &ctx);
-                                    bitcoin_block_store.insert(
-                                        updated_block.block_identifier.clone(),
-                                        (updated_block.clone(), updated),
-                                    );
-                                    new_blocks.push(updated_block);
-                                }
-                                (Some((block, _)), _) => {
-                                    bitcoin_block_store.insert(
-                                        block.block_identifier.clone(),
-                                        (block.clone(), true),
-                                    );
-                                    new_blocks.push(block);
-                                }
-                                (None, _) => {
-                                    ctx.try_log(|logger| {
-                                        slog::error!(
-                                            logger,
-                                            "Unable to retrieve bitcoin block {}",
-                                            header.block_identifier
-                                        )
-                                    });
-                                }
+                            if store_update_required {
+                                let Some(block) = bitcoin_block_store.remove(&header.block_identifier) else {
+                                    continue;
+                                };
+                                blocks_to_mutate.push(block);
+                            } else {
+                                let Some(cache) = bitcoin_block_store.get(&header.block_identifier) else {
+                                    continue;
+                                };
+                                new_blocks.push(cache.block.clone());
+                            };
+                        }
+
+                        if let Some(ref sidecar) = observer_sidecar {
+                            let updated_blocks = sidecar.perform_bitcoin_sidecar_mutations(
+                                blocks_to_mutate,
+                                vec![],
+                                &ctx,
+                            );
+                            for cache in updated_blocks.into_iter() {
+                                bitcoin_block_store
+                                    .insert(cache.block.block_identifier.clone(), cache.clone());
+                                new_blocks.push(cache.block);
                             }
                         }
 
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some((block, _)) => {
-                                    confirmed_blocks.push(block);
+                                Some(res) => {
+                                    confirmed_blocks.push(res.block);
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -998,28 +1019,32 @@ pub async fn start_observer_commands_handler(
                         )
                     }
                     BlockchainEvent::BlockchainUpdatedWithReorg(data) => {
-                        let mut blocks_to_apply = vec![];
                         let mut blocks_to_rollback = vec![];
 
-                        let blocks_ids_to_rollback = data
-                            .headers_to_rollback
-                            .iter()
-                            .map(|b| b.block_identifier.index.to_string())
-                            .collect::<Vec<String>>();
-                        let blocks_ids_to_apply = data
-                            .headers_to_apply
-                            .iter()
-                            .map(|b| b.block_identifier.index.to_string())
-                            .collect::<Vec<String>>();
+                        let mut blocks_to_mutate = vec![];
+                        let mut blocks_to_apply = vec![];
 
-                        ctx.try_log(|logger| {
-                            slog::info!(logger, "Bitcoin reorg detected, will rollback blocks {} and apply blocks {}", blocks_ids_to_rollback.join(", "), blocks_ids_to_apply.join(", "))
-                        });
+                        for header in data.headers_to_apply.iter() {
+                            if store_update_required {
+                                let Some(block) = bitcoin_block_store.remove(&header.block_identifier) else {
+                                    continue;
+                                };
+                                blocks_to_mutate.push(block);
+                            } else {
+                                let Some(cache) = bitcoin_block_store.get(&header.block_identifier) else {
+                                    continue;
+                                };
+                                blocks_to_apply.push(cache.block.clone());
+                            };
+                        }
+
+                        let mut blocks_ids_to_rollback: Vec<BlockIdentifier> = vec![];
 
                         for header in data.headers_to_rollback.iter() {
                             match bitcoin_block_store.get(&header.block_identifier) {
-                                Some((block, _)) => {
-                                    blocks_to_rollback.push(block.clone());
+                                Some(cache) => {
+                                    blocks_ids_to_rollback.push(header.block_identifier.clone());
+                                    blocks_to_rollback.push(cache.block.clone());
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
@@ -1033,44 +1058,23 @@ pub async fn start_observer_commands_handler(
                             }
                         }
 
-                        for header in data.headers_to_apply.iter() {
-                            match (
-                                bitcoin_block_store.remove(&header.block_identifier),
-                                &observer_sidecar,
-                            ) {
-                                (Some((block, false)), Some(sidecar)) => {
-                                    // Time for pre-processing
-                                    let (updated, updated_block) =
-                                        sidecar.perform_bitcoin_sidecar_mutation(block, &ctx);
-                                    bitcoin_block_store.insert(
-                                        updated_block.block_identifier.clone(),
-                                        (updated_block.clone(), updated),
-                                    );
-                                    blocks_to_apply.push(updated_block);
-                                }
-                                (Some((block, _)), _) => {
-                                    bitcoin_block_store.insert(
-                                        block.block_identifier.clone(),
-                                        (block.clone(), true),
-                                    );
-                                    blocks_to_apply.push(block);
-                                }
-                                (None, _) => {
-                                    ctx.try_log(|logger| {
-                                        slog::error!(
-                                            logger,
-                                            "Unable to retrieve bitcoin block {}",
-                                            header.block_identifier
-                                        )
-                                    });
-                                }
+                        if let Some(ref sidecar) = observer_sidecar {
+                            let updated_blocks = sidecar.perform_bitcoin_sidecar_mutations(
+                                blocks_to_mutate,
+                                blocks_ids_to_rollback,
+                                &ctx,
+                            );
+                            for cache in updated_blocks.into_iter() {
+                                bitcoin_block_store
+                                    .insert(cache.block.block_identifier.clone(), cache.clone());
+                                blocks_to_apply.push(cache.block);
                             }
                         }
 
                         for header in data.confirmed_headers.iter() {
                             match bitcoin_block_store.remove(&header.block_identifier) {
-                                Some((block, _)) => {
-                                    confirmed_blocks.push(block);
+                                Some(res) => {
+                                    confirmed_blocks.push(res.block);
                                 }
                                 None => {
                                     ctx.try_log(|logger| {
diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs
index 2c2353878..f14e7d3d2 100644
--- a/components/chainhook-sdk/src/observer/tests/mod.rs
+++ b/components/chainhook-sdk/src/observer/tests/mod.rs
@@ -17,9 +17,10 @@ use crate::observer::{
 };
 use crate::utils::{AbstractBlock, Context};
 use chainhook_types::{
-    BitcoinBlockSignaling, BitcoinNetwork, BlockchainEvent, BlockchainUpdatedWithHeaders,
-    OrdinalInscriptionRevealData, OrdinalOperation, StacksBlockUpdate, StacksChainEvent,
-    StacksChainUpdatedWithBlocksData, StacksNetwork, StacksNodeConfig, BitcoinChainEvent,
+    BitcoinBlockSignaling, BitcoinChainEvent, BitcoinNetwork, BlockchainEvent,
+    BlockchainUpdatedWithHeaders, OrdinalInscriptionRevealData, OrdinalOperation,
+    StacksBlockUpdate, StacksChainEvent, StacksChainUpdatedWithBlocksData, StacksNetwork,
+    StacksNodeConfig,
 };
 use hiro_system_kit;
 use std::collections::BTreeMap;
@@ -1233,7 +1234,7 @@ fn test_bitcoin_chainhook_through_reorg() {
     let empty_ctx = Context::empty();
 
     let observer_sidecar = ObserverSidecar {
-        bitcoin_block_mutator: Some((block_pre_processor_in_tx, block_pre_processor_out_rx)),
+        bitcoin_blocks_mutator: Some((block_pre_processor_in_tx, block_pre_processor_out_rx)),
         bitcoin_chain_event_notifier: None,
     };
 
@@ -1255,33 +1256,35 @@ fn test_bitcoin_chainhook_through_reorg() {
     // registered predicates
     let block_pre_processor_handle = std::thread::spawn(move || {
         let mut cursor: u64 = 0;
-        while let Ok(mut block) = block_pre_processor_in_rx.recv() {
-            for (tx_index, tx) in block.transactions.iter_mut().enumerate() {
-                cursor += 1;
-                tx.metadata
-                    .ordinal_operations
-                    .push(OrdinalOperation::InscriptionRevealed(
-                        OrdinalInscriptionRevealData {
-                            content_bytes: format!("{cursor}"),
-                            content_type: "".to_string(),
-                            content_length: cursor as usize,
-                            inscription_number: cursor as i64,
-                            inscription_fee: cursor,
-                            inscription_output_value: cursor,
-                            inscription_id: format!("{cursor}"),
-                            inscription_input_index: 0,
-                            inscriber_address: None,
-                            ordinal_number: cursor,
-                            ordinal_block_height: block.block_identifier.index,
-                            ordinal_offset: 0,
-                            tx_index,
-                            transfers_pre_inscription: cursor as u32,
-                            satpoint_post_inscription: format!("{cursor}"),
-                            curse_type: None,
-                        },
-                    ))
+        while let Ok((mut blocks, _)) = block_pre_processor_in_rx.recv() {
+            for b in blocks.iter_mut() {
+                for (tx_index, tx) in b.block.transactions.iter_mut().enumerate() {
+                    cursor += 1;
+                    tx.metadata
+                        .ordinal_operations
+                        .push(OrdinalOperation::InscriptionRevealed(
+                            OrdinalInscriptionRevealData {
+                                content_bytes: format!("{cursor}"),
+                                content_type: "".to_string(),
+                                content_length: cursor as usize,
+                                inscription_number: cursor as i64,
+                                inscription_fee: cursor,
+                                inscription_output_value: cursor,
+                                inscription_id: format!("{cursor}"),
+                                inscription_input_index: 0,
+                                inscriber_address: None,
+                                ordinal_number: cursor,
+                                ordinal_block_height: b.block.block_identifier.index,
+                                ordinal_offset: 0,
+                                tx_index,
+                                transfers_pre_inscription: cursor as u32,
+                                satpoint_post_inscription: format!("{cursor}"),
+                                curse_type: None,
+                            },
+                        ))
+                }
             }
-            let _ = block_pre_processor_out_tx.send(block);
+            let _ = block_pre_processor_out_tx.send(blocks);
         }
     });
 

From ea335530c174b8893013e6be7e0258285c4a9667 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Wed, 23 Aug 2023 10:00:54 +0200
Subject: [PATCH 09/20] fix: use crossbeam channels

---
 Cargo.lock                                         | 6 +++---
 components/chainhook-cli/Cargo.toml                | 2 +-
 components/chainhook-sdk/Cargo.toml                | 4 ++--
 components/chainhook-sdk/src/observer/mod.rs       | 9 +++++----
 components/chainhook-sdk/src/observer/tests/mod.rs | 4 ++--
 components/chainhook-types-rs/Cargo.toml           | 2 +-
 6 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8f87fc1ad..a44e40aae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -457,13 +457,13 @@ dependencies = [
 
 [[package]]
 name = "chainhook-sdk"
-version = "0.8.2"
+version = "0.8.4"
 dependencies = [
  "base58 0.2.0",
  "base64",
  "bitcoincore-rpc",
  "bitcoincore-rpc-json",
- "chainhook-types 1.0.11",
+ "chainhook-types 1.0.12",
  "clarinet-utils",
  "crossbeam-channel 0.5.8",
  "dashmap 5.4.0",
@@ -505,7 +505,7 @@ dependencies = [
 
 [[package]]
 name = "chainhook-types"
-version = "1.0.11"
+version = "1.0.12"
 dependencies = [
  "hex",
  "schemars 0.8.12",
diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml
index 77fb2c55d..1e0e88dc2 100644
--- a/components/chainhook-cli/Cargo.toml
+++ b/components/chainhook-cli/Cargo.toml
@@ -15,7 +15,7 @@ redis = "0.21.5"
 serde-redis = "0.12.0"
 hex = "0.4.3"
 rand = "0.8.5"
-chainhook-sdk = { version = "0.8.2", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" }
+chainhook-sdk = { version = "0.8.4", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" }
 clarinet-files = "1.0.1"
 hiro-system-kit = "0.1.0"
 # clarinet-files = { path = "../../../clarinet/components/clarinet-files" }
diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml
index 923946362..243641462 100644
--- a/components/chainhook-sdk/Cargo.toml
+++ b/components/chainhook-sdk/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chainhook-sdk"
-version = "0.8.2"
+version = "0.8.4"
 description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin"
 license = "GPL-3.0"
 edition = "2021"
@@ -18,7 +18,7 @@ hiro-system-kit = "0.1.0"
 # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" }
 # clarinet-utils = { version = "1", path = "../../../clarinet/components/clarinet-utils" }
 # hiro-system-kit = { version = "0.1.0", path = "../../../clarinet/components/hiro-system-kit" }
-chainhook-types = { version = "1.0.11", path = "../chainhook-types-rs" }
+chainhook-types = { version = "1.0.12", path = "../chainhook-types-rs" }
 rocket = { version = "=0.5.0-rc.3", features = ["json"] }
 bitcoincore-rpc = "0.16.0"
 bitcoincore-rpc-json = "0.16.0"
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index 6f73470b4..80a296826 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -32,7 +32,7 @@ use rocket::config::{self, Config, LogLevel};
 use rocket::data::{Limits, ToByteUnit};
 use rocket::serde::Deserialize;
 use rocket::Shutdown;
-use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::error::Error;
 use std::net::{IpAddr, Ipv4Addr};
 use std::path::PathBuf;
@@ -344,10 +344,10 @@ pub struct BitcoinBlockDataCached {
 
 pub struct ObserverSidecar {
     pub bitcoin_blocks_mutator: Option<(
-        Sender<(Vec<BitcoinBlockDataCached>, Vec<BlockIdentifier>)>,
-        Receiver<Vec<BitcoinBlockDataCached>>,
+        crossbeam_channel::Sender<(Vec<BitcoinBlockDataCached>, Vec<BlockIdentifier>)>,
+        crossbeam_channel::Receiver<Vec<BitcoinBlockDataCached>>,
     )>,
-    pub bitcoin_chain_event_notifier: Option<Sender<HandleBlock>>,
+    pub bitcoin_chain_event_notifier: Option<crossbeam_channel::Sender<HandleBlock>>,
 }
 
 impl ObserverSidecar {
@@ -662,6 +662,7 @@ pub fn start_zeromq_runloop(
     #[cfg(feature = "zeromq")]
     {
         use crate::indexer::fork_scratch_pad::ForkScratchPad;
+        use std::collections::VecDeque;
 
         if let BitcoinBlockSignaling::ZeroMQ(ref bitcoind_zmq_url) = config.bitcoin_block_signaling
         {
diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs
index f14e7d3d2..b3eea713e 100644
--- a/components/chainhook-sdk/src/observer/tests/mod.rs
+++ b/components/chainhook-sdk/src/observer/tests/mod.rs
@@ -1224,8 +1224,8 @@ fn test_bitcoin_chainhook_auto_deregister() {
 #[test]
 fn test_bitcoin_chainhook_through_reorg() {
     let (observer_commands_tx, observer_commands_rx) = channel();
-    let (block_pre_processor_in_tx, block_pre_processor_in_rx) = channel();
-    let (block_pre_processor_out_tx, block_pre_processor_out_rx) = channel();
+    let (block_pre_processor_in_tx, block_pre_processor_in_rx) = crossbeam_channel::unbounded();
+    let (block_pre_processor_out_tx, block_pre_processor_out_rx) = crossbeam_channel::unbounded();
 
     let (observer_events_tx, observer_events_rx) = crossbeam_channel::unbounded();
     let observer_metrics_rw_lock = Arc::new(RwLock::new(ObserverMetrics::default()));
diff --git a/components/chainhook-types-rs/Cargo.toml b/components/chainhook-types-rs/Cargo.toml
index d4c97de44..de2f3263f 100644
--- a/components/chainhook-types-rs/Cargo.toml
+++ b/components/chainhook-types-rs/Cargo.toml
@@ -2,7 +2,7 @@
 name = "chainhook-types"
 description = "Bitcoin and Stacks data schemas, based on the Rosetta specification"
 license = "MIT"
-version = "1.0.11"
+version = "1.0.12"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

From 5a809c63bec1c949314ecbd44ef1348286968dec Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sat, 26 Aug 2023 10:54:42 -0400
Subject: [PATCH 10/20] fix: chainhook not being registered

---
 components/chainhook-sdk/src/observer/mod.rs | 34 ++++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index 80a296826..73ddaab9f 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -107,6 +107,23 @@ impl EventObserverConfig {
         bitcoin_config
     }
 
+    pub fn get_chainhook_store(&self) -> ChainhookStore {
+        let mut chainhook_store = ChainhookStore::new();
+        // If authorization not required, we create a default ChainhookConfig
+        if let Some(ref chainhook_config) = self.chainhook_config {
+            let mut chainhook_config = chainhook_config.clone();
+            chainhook_store
+                .predicates
+                .stacks_chainhooks
+                .append(&mut chainhook_config.stacks_chainhooks);
+            chainhook_store
+                .predicates
+                .bitcoin_chainhooks
+                .append(&mut chainhook_config.bitcoin_chainhooks);
+        }
+        chainhook_store
+    }
+
     pub fn get_stacks_node_config(&self) -> &StacksNodeConfig {
         match self.bitcoin_block_signaling {
             BitcoinBlockSignaling::Stacks(ref config) => config,
@@ -474,7 +491,7 @@ pub async fn start_bitcoin_event_observer(
     observer_sidecar: Option<ObserverSidecar>,
     ctx: Context,
 ) -> Result<(), Box<dyn Error>> {
-    let chainhook_store = ChainhookStore::new();
+    let chainhook_store = config.get_chainhook_store();
 
     let observer_metrics = ObserverMetrics {
         bitcoin: ChainMetrics {
@@ -505,7 +522,7 @@ pub async fn start_bitcoin_event_observer(
 }
 
 pub async fn start_stacks_event_observer(
-    mut config: EventObserverConfig,
+    config: EventObserverConfig,
     observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
@@ -537,18 +554,7 @@ pub async fn start_stacks_event_observer(
     let bitcoin_rpc_proxy_enabled = config.bitcoin_rpc_proxy_enabled;
     let bitcoin_config = config.get_bitcoin_config();
 
-    let mut chainhook_store = ChainhookStore::new();
-    // If authorization not required, we create a default ChainhookConfig
-    if let Some(ref mut initial_chainhook_config) = config.chainhook_config {
-        chainhook_store
-            .predicates
-            .stacks_chainhooks
-            .append(&mut initial_chainhook_config.stacks_chainhooks);
-        chainhook_store
-            .predicates
-            .bitcoin_chainhooks
-            .append(&mut initial_chainhook_config.bitcoin_chainhooks);
-    }
+    let chainhook_store = config.get_chainhook_store();
 
     let indexer_rw_lock = Arc::new(RwLock::new(indexer));
 

From ee486f3571f97728d5305bdb72a303134fca1bf5 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sat, 26 Aug 2023 10:56:32 -0400
Subject: [PATCH 11/20] feat: introduce "data_handler_tx"

---
 components/chainhook-sdk/src/observer/mod.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index 73ddaab9f..f29e97fd1 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -73,6 +73,7 @@ pub struct EventObserverConfig {
     pub cache_path: String,
     pub bitcoin_network: BitcoinNetwork,
     pub stacks_network: StacksNetwork,
+    pub data_handler_tx: Option<crossbeam_channel::Sender<BitcoinChainhookOccurrencePayload>>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -187,6 +188,7 @@ impl EventObserverConfig {
                 .unwrap_or("cache".to_string()),
             bitcoin_network,
             stacks_network,
+            data_handler_tx: None,
         };
         Ok(config)
     }

From be0c22957b7345721e33d38e3bfa98794155e7a7 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sat, 26 Aug 2023 10:59:01 -0400
Subject: [PATCH 12/20] fix: build errro

---
 Cargo.lock                                         | 2 +-
 components/chainhook-sdk/Cargo.toml                | 2 +-
 components/chainhook-sdk/src/observer/tests/mod.rs | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a44e40aae..e341088b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -457,7 +457,7 @@ dependencies = [
 
 [[package]]
 name = "chainhook-sdk"
-version = "0.8.4"
+version = "0.8.5"
 dependencies = [
  "base58 0.2.0",
  "base64",
diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml
index 243641462..bec274936 100644
--- a/components/chainhook-sdk/Cargo.toml
+++ b/components/chainhook-sdk/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chainhook-sdk"
-version = "0.8.4"
+version = "0.8.5"
 description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin"
 license = "GPL-3.0"
 edition = "2021"
diff --git a/components/chainhook-sdk/src/observer/tests/mod.rs b/components/chainhook-sdk/src/observer/tests/mod.rs
index b3eea713e..d9864fafd 100644
--- a/components/chainhook-sdk/src/observer/tests/mod.rs
+++ b/components/chainhook-sdk/src/observer/tests/mod.rs
@@ -44,6 +44,7 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) {
         cache_path: "cache".into(),
         bitcoin_network: BitcoinNetwork::Regtest,
         stacks_network: StacksNetwork::Devnet,
+        data_handler_tx: None,
     };
     let predicates = ChainhookConfig::new();
     let chainhook_store = ChainhookStore { predicates };

From 85d4d91ca6276a25d0bc95e256da356758155466 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sun, 27 Aug 2023 11:04:23 -0400
Subject: [PATCH 13/20] fix: build error

---
 components/chainhook-cli/src/config/mod.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/components/chainhook-cli/src/config/mod.rs b/components/chainhook-cli/src/config/mod.rs
index 7db407523..28eecfa3d 100644
--- a/components/chainhook-cli/src/config/mod.rs
+++ b/components/chainhook-cli/src/config/mod.rs
@@ -119,6 +119,7 @@ impl Config {
             cache_path: self.storage.working_dir.clone(),
             bitcoin_network: self.network.bitcoin_network.clone(),
             stacks_network: self.network.stacks_network.clone(),
+            data_handler_tx: None,
         }
     }
 

From 4eb5a07ad350360f159b5443d0b2d665c20892bf Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sun, 27 Aug 2023 11:05:24 -0400
Subject: [PATCH 14/20] fix: migrate to finer zmq lib

---
 Cargo.lock                                   | 509 +++++++++----------
 components/chainhook-sdk/Cargo.toml          |   6 +-
 components/chainhook-sdk/src/observer/mod.rs | 159 +-----
 components/chainhook-sdk/src/observer/zmq.rs | 157 ++++++
 4 files changed, 400 insertions(+), 431 deletions(-)
 create mode 100644 components/chainhook-sdk/src/observer/zmq.rs

diff --git a/Cargo.lock b/Cargo.lock
index e341088b8..2e3978907 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8,15 +8,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
-dependencies = [
- "const-random",
-]
-
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -114,19 +105,6 @@ dependencies = [
  "syn 1.0.105",
 ]
 
-[[package]]
-name = "asynchronous-codec"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182"
-dependencies = [
- "bytes",
- "futures-sink",
- "futures-util",
- "memchr",
- "pin-project-lite",
-]
-
 [[package]]
 name = "atomic"
 version = "0.5.1"
@@ -403,10 +381,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "cfg-if"
-version = "0.1.10"
+name = "cfg-expr"
+version = "0.15.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+checksum = "b40ccee03b5175c18cde8f37e7d2a33bcef6f8ec8f7cc0d81090d1bb380949c9"
+dependencies = [
+ "smallvec 1.10.0",
+ "target-lexicon",
+]
 
 [[package]]
 name = "cfg-if"
@@ -427,7 +409,7 @@ dependencies = [
  "clarity-repl",
  "clarity-vm",
  "criterion",
- "crossbeam-channel 0.5.8",
+ "crossbeam-channel",
  "csv",
  "ctrlc",
  "flate2",
@@ -451,13 +433,13 @@ dependencies = [
  "test-case",
  "threadpool",
  "tokio",
- "toml",
- "uuid 1.3.0",
+ "toml 0.5.9",
+ "uuid",
 ]
 
 [[package]]
 name = "chainhook-sdk"
-version = "0.8.5"
+version = "0.8.6"
 dependencies = [
  "base58 0.2.0",
  "base64",
@@ -465,8 +447,8 @@ dependencies = [
  "bitcoincore-rpc-json",
  "chainhook-types 1.0.12",
  "clarinet-utils",
- "crossbeam-channel 0.5.8",
- "dashmap 5.4.0",
+ "crossbeam-channel",
+ "dashmap",
  "futures",
  "fxhash",
  "hex",
@@ -486,7 +468,7 @@ dependencies = [
  "test-case",
  "threadpool",
  "tokio",
- "zeromq",
+ "zmq",
 ]
 
 [[package]]
@@ -562,7 +544,7 @@ dependencies = [
  "bitflags 1.3.2",
  "clap_derive",
  "clap_lex",
- "indexmap",
+ "indexmap 1.9.2",
  "once_cell",
  "strsim",
  "termcolor",
@@ -625,7 +607,7 @@ dependencies = [
  "serde",
  "serde_derive",
  "tiny-hderive",
- "toml",
+ "toml 0.5.9",
  "url",
 ]
 
@@ -673,7 +655,7 @@ dependencies = [
  "sha2 0.10.6",
  "sha3 0.9.1",
  "tokio",
- "tokio-util 0.7.2",
+ "tokio-util",
 ]
 
 [[package]]
@@ -740,28 +722,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "const-random"
-version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
-dependencies = [
- "const-random-macro",
- "proc-macro-hack",
-]
-
-[[package]]
-name = "const-random-macro"
-version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
-dependencies = [
- "getrandom 0.2.8",
- "once_cell",
- "proc-macro-hack",
- "tiny-keccak",
-]
-
 [[package]]
 name = "const_fn"
 version = "0.4.9"
@@ -810,7 +770,7 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -851,26 +811,16 @@ dependencies = [
 
 [[package]]
 name = "crossbeam"
-version = "0.7.3"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
+checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c"
 dependencies = [
- "cfg-if 0.1.10",
- "crossbeam-channel 0.4.4",
- "crossbeam-deque 0.7.4",
- "crossbeam-epoch 0.8.2",
+ "cfg-if",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
  "crossbeam-queue",
- "crossbeam-utils 0.7.2",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
-dependencies = [
- "crossbeam-utils 0.7.2",
- "maybe-uninit",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -879,19 +829,8 @@ version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
 dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-utils 0.8.14",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed"
-dependencies = [
- "crossbeam-epoch 0.8.2",
- "crossbeam-utils 0.7.2",
- "maybe-uninit",
+ "cfg-if",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -900,24 +839,9 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
 dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-epoch 0.9.13",
- "crossbeam-utils 0.8.14",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
-dependencies = [
- "autocfg",
- "cfg-if 0.1.10",
- "crossbeam-utils 0.7.2",
- "lazy_static",
- "maybe-uninit",
- "memoffset 0.5.6",
- "scopeguard",
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -927,32 +851,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
 dependencies = [
  "autocfg",
- "cfg-if 1.0.0",
- "crossbeam-utils 0.8.14",
+ "cfg-if",
+ "crossbeam-utils",
  "memoffset 0.7.1",
  "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
-dependencies = [
- "cfg-if 0.1.10",
- "crossbeam-utils 0.7.2",
- "maybe-uninit",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.7.2"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
+checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
 dependencies = [
- "autocfg",
- "cfg-if 0.1.10",
- "lazy_static",
+ "cfg-if",
+ "crossbeam-utils",
 ]
 
 [[package]]
@@ -961,7 +873,7 @@ version = "0.8.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -1124,28 +1036,17 @@ dependencies = [
  "syn 1.0.105",
 ]
 
-[[package]]
-name = "dashmap"
-version = "3.11.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5"
-dependencies = [
- "ahash 0.3.8",
- "cfg-if 0.1.10",
- "num_cpus",
-]
-
 [[package]]
 name = "dashmap"
 version = "5.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "hashbrown 0.12.3",
  "lock_api",
  "once_cell",
- "parking_lot_core 0.9.5",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -1221,13 +1122,24 @@ dependencies = [
  "subtle 2.4.1",
 ]
 
+[[package]]
+name = "dircpy"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8466f8d28ca6da4c9dfbbef6ad4bff6f2fdd5e412d821025b0d3f0a9d74a8c1e"
+dependencies = [
+ "jwalk",
+ "log",
+ "walkdir",
+]
+
 [[package]]
 name = "dirs-next"
 version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "dirs-sys-next",
 ]
 
@@ -1285,7 +1197,7 @@ version = "0.8.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -1295,15 +1207,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
 
 [[package]]
-name = "enum-primitive-derive"
-version = "0.2.2"
+name = "equivalent"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c375b9c5eadb68d0a6efee2999fef292f45854c3444c86f09d8ab086ba942b0e"
-dependencies = [
- "num-traits",
- "quote",
- "syn 1.0.105",
-]
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
@@ -1380,7 +1287,7 @@ version = "3.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb21c69b9fea5e15dbc1049e4b77145dd0ba1c84019c488102de0dc4ea4b0a27"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "rustix 0.36.5",
  "windows-sys 0.42.0",
 ]
@@ -1394,7 +1301,7 @@ dependencies = [
  "atomic",
  "pear",
  "serde",
- "toml",
+ "toml 0.5.9",
  "uncased",
  "version_check",
 ]
@@ -1405,7 +1312,7 @@ version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "redox_syscall",
  "windows-sys 0.42.0",
@@ -1600,7 +1507,7 @@ version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "wasi 0.9.0+wasi-snapshot-preview1",
 ]
@@ -1611,7 +1518,7 @@ version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "js-sys",
  "libc",
  "wasi 0.11.0+wasi-snapshot-preview1",
@@ -1636,10 +1543,10 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 1.9.2",
  "slab",
  "tokio",
- "tokio-util 0.7.2",
+ "tokio-util",
  "tracing",
 ]
 
@@ -1655,7 +1562,7 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
- "ahash 0.7.6",
+ "ahash",
 ]
 
 [[package]]
@@ -1664,6 +1571,12 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
 [[package]]
 name = "hashlink"
 version = "0.7.0"
@@ -1914,6 +1827,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
 [[package]]
 name = "inlinable_string"
 version = "0.1.15"
@@ -1926,7 +1849,7 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -2012,6 +1935,16 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "jwalk"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56"
+dependencies = [
+ "crossbeam",
+ "rayon",
+]
+
 [[package]]
 name = "keccak"
 version = "0.1.3"
@@ -2045,7 +1978,7 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "winapi",
 ]
 
@@ -2187,7 +2120,7 @@ version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -2196,7 +2129,7 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "generator",
  "scoped-tls",
  "serde",
@@ -2236,15 +2169,6 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
-[[package]]
-name = "memoffset"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "memoffset"
 version = "0.6.5"
@@ -2318,7 +2242,7 @@ dependencies = [
  "mime",
  "spin 0.9.4",
  "tokio",
- "tokio-util 0.7.2",
+ "tokio-util",
  "version_check",
 ]
 
@@ -2366,7 +2290,7 @@ checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
 dependencies = [
  "bitflags 1.3.2",
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "memoffset 0.6.5",
 ]
@@ -2379,7 +2303,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
 dependencies = [
  "autocfg",
  "bitflags 1.3.2",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
 ]
 
@@ -2479,7 +2403,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
 dependencies = [
  "bitflags 1.3.2",
- "cfg-if 1.0.0",
+ "cfg-if",
  "foreign-types",
  "libc",
  "once_cell",
@@ -2529,17 +2453,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
-[[package]]
-name = "parking_lot"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.6",
-]
-
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -2547,21 +2460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.5",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
-dependencies = [
- "cfg-if 1.0.0",
- "instant",
- "libc",
- "redox_syscall",
- "smallvec 1.10.0",
- "winapi",
+ "parking_lot_core",
 ]
 
 [[package]]
@@ -2570,7 +2469,7 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec 1.10.0",
@@ -2923,9 +2822,9 @@ version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
 dependencies = [
- "crossbeam-channel 0.5.8",
- "crossbeam-deque 0.8.2",
- "crossbeam-utils 0.8.14",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
  "num_cpus",
 ]
 
@@ -3070,7 +2969,7 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-rustls",
- "tokio-util 0.7.2",
+ "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -3118,13 +3017,13 @@ dependencies = [
  "either",
  "figment",
  "futures",
- "indexmap",
+ "indexmap 1.9.2",
  "is-terminal",
  "log",
  "memchr",
  "multer",
  "num_cpus",
- "parking_lot 0.12.1",
+ "parking_lot",
  "pin-project-lite",
  "rand 0.8.5",
  "ref-cast",
@@ -3137,7 +3036,7 @@ dependencies = [
  "time 0.3.17",
  "tokio",
  "tokio-stream",
- "tokio-util 0.7.2",
+ "tokio-util",
  "ubyte",
  "version_check",
  "yansi",
@@ -3151,7 +3050,7 @@ checksum = "7093353f14228c744982e409259fb54878ba9563d08214f2d880d59ff2fc508b"
 dependencies = [
  "devise",
  "glob",
- "indexmap",
+ "indexmap 1.9.2",
  "proc-macro2",
  "quote",
  "rocket_http",
@@ -3170,7 +3069,7 @@ dependencies = [
  "futures",
  "http",
  "hyper",
- "indexmap",
+ "indexmap 1.9.2",
  "log",
  "memchr",
  "pear",
@@ -3229,7 +3128,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2288c66aeafe3b2ed227c981f364f9968fa952ef0b30e84ada4486e7ee24d00a"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "proc-macro2",
  "quote",
  "rustc_version 0.4.0",
@@ -3358,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039"
 dependencies = [
  "bitflags 1.3.2",
- "cfg-if 1.0.0",
+ "cfg-if",
  "clipboard-win",
  "dirs-next",
  "fd-lock",
@@ -3418,11 +3317,11 @@ version = "0.8.12"
 source = "git+https://github.com/hirosystems/schemars.git?branch=feat-chainhook-fixes#15fdd4711700114d57c090aad62516593bd4ca6d"
 dependencies = [
  "dyn-clone",
- "indexmap",
+ "indexmap 1.9.2",
  "schemars_derive 0.8.12",
  "serde",
  "serde_json",
- "uuid 1.3.0",
+ "uuid",
 ]
 
 [[package]]
@@ -3631,6 +3530,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_stacker"
 version = "0.1.6"
@@ -3659,11 +3567,11 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d"
 dependencies = [
- "dashmap 5.4.0",
+ "dashmap",
  "futures",
  "lazy_static",
  "log",
- "parking_lot 0.12.1",
+ "parking_lot",
  "serial_test_derive",
 ]
 
@@ -3712,7 +3620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
 dependencies = [
  "block-buffer 0.9.0",
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest 0.9.0",
  "opaque-debug 0.3.0",
@@ -3724,7 +3632,7 @@ version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "cpufeatures",
  "digest 0.10.6",
 ]
@@ -3805,7 +3713,7 @@ version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe"
 dependencies = [
- "crossbeam-channel 0.5.8",
+ "crossbeam-channel",
  "slog",
  "take_mut",
  "thread_local",
@@ -3913,7 +3821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
 dependencies = [
  "cc",
- "cfg-if 1.0.0",
+ "cfg-if",
  "libc",
  "psm",
  "winapi",
@@ -4099,6 +4007,19 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "system-deps"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30c2de8a4d8f4b823d634affc9cd2a74ec98c53a756f317e529a48046cbf71f3"
+dependencies = [
+ "cfg-expr",
+ "heck 0.4.0",
+ "pkg-config",
+ "toml 0.7.6",
+ "version-compare",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
@@ -4116,13 +4037,19 @@ dependencies = [
  "xattr",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.12.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a"
+
 [[package]]
 name = "tempfile"
 version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "fastrand",
  "libc",
  "redox_syscall",
@@ -4165,7 +4092,7 @@ version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d10394d5d1e27794f772b6fc854c7e91a2dc26e2cbf807ad523370c2a59c0cee"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "proc-macro-error",
  "proc-macro2",
  "quote",
@@ -4329,15 +4256,6 @@ dependencies = [
  "sha2 0.8.2",
 ]
 
-[[package]]
-name = "tiny-keccak"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
-dependencies = [
- "crunchy",
-]
-
 [[package]]
 name = "tinytemplate"
 version = "1.2.1"
@@ -4375,7 +4293,7 @@ dependencies = [
  "memchr",
  "mio",
  "num_cpus",
- "parking_lot 0.12.1",
+ "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
@@ -4426,21 +4344,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-util"
-version = "0.6.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-io",
- "futures-sink",
- "log",
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.7.2"
@@ -4461,8 +4364,42 @@ version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
 dependencies = [
- "indexmap",
+ "indexmap 1.9.2",
+ "serde",
+]
+
+[[package]]
+name = "toml"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.19.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
+dependencies = [
+ "indexmap 2.0.0",
  "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
 ]
 
 [[package]]
@@ -4477,7 +4414,7 @@ version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -4633,15 +4570,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
 
-[[package]]
-name = "uuid"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
-dependencies = [
- "getrandom 0.2.8",
-]
-
 [[package]]
 name = "uuid"
 version = "1.3.0"
@@ -4664,6 +4592,12 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "version-compare"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -4715,7 +4649,7 @@ version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "wasm-bindgen-macro",
 ]
 
@@ -4740,7 +4674,7 @@ version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "js-sys",
  "wasm-bindgen",
  "web-sys",
@@ -5044,6 +4978,15 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
 
+[[package]]
+name = "winnow"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winreg"
 version = "0.10.1"
@@ -5075,27 +5018,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
 
 [[package]]
-name = "zeromq"
-version = "0.3.3"
+name = "zeromq-src"
+version = "0.2.6+4.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "667ece59294ccaf617fcf2e5decc9114a06427c1f68990028b9f12d322686bdc"
+checksum = "fc120b771270365d5ed0dfb4baf1005f2243ae1ae83703265cb3504070f4160b"
 dependencies = [
- "async-trait",
- "asynchronous-codec",
- "bytes",
- "crossbeam",
- "dashmap 3.11.10",
- "enum-primitive-derive",
- "futures",
- "futures-util",
- "lazy_static",
- "log",
- "num-traits",
- "parking_lot 0.11.2",
- "rand 0.7.3",
- "regex",
- "thiserror",
- "tokio",
- "tokio-util 0.6.10",
- "uuid 0.8.2",
+ "cc",
+ "dircpy",
+]
+
+[[package]]
+name = "zmq"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd3091dd571fb84a9b3e5e5c6a807d186c411c812c8618786c3c30e5349234e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "zmq-sys",
+]
+
+[[package]]
+name = "zmq-sys"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e8351dc72494b4d7f5652a681c33634063bbad58046c1689e75270908fdc864"
+dependencies = [
+ "libc",
+ "system-deps",
+ "zeromq-src",
 ]
diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml
index bec274936..c01767ae8 100644
--- a/components/chainhook-sdk/Cargo.toml
+++ b/components/chainhook-sdk/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chainhook-sdk"
-version = "0.8.5"
+version = "0.8.6"
 description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin"
 license = "GPL-3.0"
 edition = "2021"
@@ -37,7 +37,7 @@ hyper = { version = "0.14.24", features = ["http1", "client"] }
 hex = "0.4.3"
 threadpool = "1.8.1"
 rand = "0.8.5"
-zeromq = { version = "0.3.3", default-features = false, features = ["tokio-runtime", "tcp-transport"], optional = true }
+zmq = { version = "0.10.0", optional = true }
 dashmap = "5.4.0"
 fxhash = "0.2.1"
 lazy_static = "1.4.0"
@@ -51,5 +51,5 @@ test-case = "3.1.0"
 
 [features]
 default = ["log"]
-zeromq = ["dep:zeromq"]
+zeromq = ["zmq"]
 log = ["hiro-system-kit/log"]
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index f29e97fd1..8c8f033ff 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -1,4 +1,6 @@
 mod http;
+#[cfg(feature = "zeromq")]
+mod zmq;
 
 use crate::chainhooks::bitcoin::{
     evaluate_bitcoin_chainhooks_on_chain_event, handle_bitcoin_hook_action,
@@ -41,8 +43,6 @@ use std::str::FromStr;
 use std::sync::mpsc::{Receiver, Sender};
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{SystemTime, UNIX_EPOCH};
-#[cfg(feature = "zeromq")]
-use zeromq::{Socket, SocketRecv};
 
 pub const DEFAULT_INGESTION_PORT: u16 = 20445;
 
@@ -507,7 +507,15 @@ pub async fn start_bitcoin_event_observer(
     };
     let observer_metrics_rw_lock = Arc::new(RwLock::new(observer_metrics));
 
-    start_zeromq_runloop(&config, observer_commands_tx, &ctx);
+    #[cfg(feature = "zeromq")]
+    {
+        let ctx_moved = ctx.clone();
+        let config_moved = config.clone();
+        let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || {
+            let future = zmq::start_zeromq_runloop(&config_moved, observer_commands_tx, &ctx_moved);
+            let _ = hiro_system_kit::nestable_block_on(future);
+        });
+    }
 
     // This loop is used for handling background jobs, emitted by HTTP calls.
     start_observer_commands_handler(
@@ -661,151 +669,6 @@ pub fn get_bitcoin_proof(
     }
 }
 
-#[allow(unused_variables, unused_imports)]
-pub fn start_zeromq_runloop(
-    config: &EventObserverConfig,
-    observer_commands_tx: Sender<ObserverCommand>,
-    ctx: &Context,
-) {
-    #[cfg(feature = "zeromq")]
-    {
-        use crate::indexer::fork_scratch_pad::ForkScratchPad;
-        use std::collections::VecDeque;
-
-        if let BitcoinBlockSignaling::ZeroMQ(ref bitcoind_zmq_url) = config.bitcoin_block_signaling
-        {
-            let bitcoind_zmq_url = bitcoind_zmq_url.clone();
-            let ctx_moved = ctx.clone();
-            let bitcoin_config = config.get_bitcoin_config();
-            let http_client = build_http_client();
-
-            hiro_system_kit::thread_named("Bitcoind zmq listener")
-                .spawn(move || {
-                    ctx_moved.try_log(|logger| {
-                        slog::info!(
-                            logger,
-                            "Waiting for ZMQ connection acknowledgment from bitcoind"
-                        )
-                    });
-
-                    let _: Result<(), Box<dyn Error>> =
-                        hiro_system_kit::nestable_block_on(async move {
-                            let mut socket = zeromq::SubSocket::new();
-
-                            socket
-                                .connect(&bitcoind_zmq_url)
-                                .await
-                                .expect("Failed to connect");
-
-                            socket.subscribe("").await?;
-                            ctx_moved.try_log(|logger| {
-                                slog::info!(logger, "Waiting for ZMQ messages from bitcoind")
-                            });
-
-                            let mut bitcoin_blocks_pool = ForkScratchPad::new();
-
-                            loop {
-                                let message = match socket.recv().await {
-                                    Ok(message) => message,
-                                    Err(e) => {
-                                        ctx_moved.try_log(|logger| {
-                                            slog::error!(
-                                                logger,
-                                                "Unable to receive ZMQ message: {}",
-                                                e.to_string()
-                                            )
-                                        });
-                                        continue;
-                                    }
-                                };
-                                let block_hash = hex::encode(message.get(1).unwrap().to_vec());
-                                let mut block_hashes: VecDeque<String> = VecDeque::new();
-                                block_hashes.push_front(block_hash);
-
-                                while let Some(block_hash) = block_hashes.pop_front() {
-
-                                    let block = match download_and_parse_block_with_retry(
-                                        &http_client,
-                                        &block_hash,
-                                        &bitcoin_config,
-                                        &ctx_moved,
-                                    )
-                                    .await
-                                    {
-                                        Ok(block) => block,
-                                        Err(e) => {
-                                            ctx_moved.try_log(|logger| {
-                                                slog::warn!(
-                                                    logger,
-                                                    "unable to download_and_parse_block: {}",
-                                                    e.to_string()
-                                                )
-                                            });
-                                            continue;
-                                        }
-                                    };
-
-                                    let header = block.get_block_header();
-                                    ctx_moved.try_log(|logger| {
-                                        slog::info!(
-                                            logger,
-                                            "Bitcoin block #{} dispatched for processing",
-                                            block.height
-                                        )
-                                    });
-
-                                    let _ = observer_commands_tx
-                                        .send(ObserverCommand::ProcessBitcoinBlock(block));
-
-                                    if bitcoin_blocks_pool.can_process_header(&header) {
-                                        match bitcoin_blocks_pool.process_header(header, &ctx_moved) {
-                                            Ok(Some(event)) => {
-                                                let _ = observer_commands_tx
-                                                    .send(ObserverCommand::PropagateBitcoinChainEvent(event));
-                                            },
-                                            Err(e) => {
-                                                ctx_moved.try_log(|logger| {
-                                                    slog::warn!(
-                                                        logger,
-                                                        "Unable to append block: {:?}", e
-                                                    )
-                                                });
-                                            }
-                                            Ok(None) => {
-                                                ctx_moved.try_log(|logger| {
-                                                    slog::warn!(
-                                                        logger,
-                                                        "Unable to append block"
-                                                    )
-                                                });
-                                            }
-                                        }
-                                    } else {
-                                        // Handle a behaviour specific to ZMQ usage in bitcoind.
-                                        // Considering a simple re-org:
-                                        // A (1) - B1 (2) - C1 (3)
-                                        //       \ B2 (4) - C2 (5) - D2 (6)
-                                        // When D2 is being discovered (making A -> B2 -> C2 -> D2 the new canonical fork)
-                                        // it looks like ZMQ is only publishing D2.
-                                        // Without additional operation, we end up with a block that we can't append.
-                                        let parent_block_hash = header.parent_block_identifier.get_hash_bytes_str().to_string();
-                                        ctx_moved.try_log(|logger| {
-                                            slog::info!(
-                                                logger,
-                                                "Possible re-org detected, retrieving parent block {parent_block_hash}"
-                                            )
-                                        });
-                                        block_hashes.push_front(parent_block_hash);
-                                    }
-                                }
-                            }
-                        });
-                })
-                .expect("unable to spawn thread");
-        }
-    }
-}
-
 pub fn gather_proofs<'a>(
     trigger: &BitcoinTriggerChainhook<'a>,
     proofs: &mut HashMap<&'a TransactionIdentifier, String>,
diff --git a/components/chainhook-sdk/src/observer/zmq.rs b/components/chainhook-sdk/src/observer/zmq.rs
new file mode 100644
index 000000000..70de3ff6e
--- /dev/null
+++ b/components/chainhook-sdk/src/observer/zmq.rs
@@ -0,0 +1,157 @@
+use chainhook_types::BitcoinBlockSignaling;
+use hiro_system_kit::slog;
+use std::sync::mpsc::Sender;
+use zmq::Socket;
+
+use crate::{
+    indexer::{
+        bitcoin::{build_http_client, download_and_parse_block_with_retry},
+        fork_scratch_pad::ForkScratchPad,
+    },
+    utils::Context,
+};
+use std::collections::VecDeque;
+
+use super::{EventObserverConfig, ObserverCommand};
+
+fn new_zmq_socket() -> Socket {
+    let context = zmq::Context::new();
+    let socket = context.socket(zmq::SUB).unwrap();
+    assert!(socket.set_subscribe(b"hashblock").is_ok());
+    assert!(socket.set_rcvhwm(0).is_ok());
+    // We override the OS default behavior:
+    assert!(socket.set_tcp_keepalive(1).is_ok());
+    // The keepalive routine will wait for 10 minutes
+    assert!(socket.set_tcp_keepalive_idle(300).is_ok());
+    // And then resend it every 30 seconds
+    assert!(socket.set_tcp_keepalive_intvl(30).is_ok());
+    // 100 times
+    assert!(socket.set_tcp_keepalive_cnt(100).is_ok());
+    socket
+}
+
+pub async fn start_zeromq_runloop(
+    config: &EventObserverConfig,
+    observer_commands_tx: Sender<ObserverCommand>,
+    ctx: &Context,
+) {
+    let BitcoinBlockSignaling::ZeroMQ(ref bitcoind_zmq_url) = config.bitcoin_block_signaling else {
+            unreachable!()
+        };
+
+    let bitcoind_zmq_url = bitcoind_zmq_url.clone();
+    let bitcoin_config = config.get_bitcoin_config();
+    let http_client = build_http_client();
+
+    ctx.try_log(|logger| {
+        slog::info!(
+            logger,
+            "Waiting for ZMQ connection acknowledgment from bitcoind"
+        )
+    });
+
+    let mut socket = new_zmq_socket();
+    assert!(socket.connect(&bitcoind_zmq_url).is_ok());
+    ctx.try_log(|logger| slog::info!(logger, "Waiting for ZMQ messages from bitcoind"));
+
+    let mut bitcoin_blocks_pool = ForkScratchPad::new();
+
+    loop {
+        ctx.try_log(|logger| slog::info!(logger, "Loop"));
+
+        let msg = match socket.recv_multipart(0) {
+            Ok(msg) => msg,
+            Err(e) => {
+                ctx.try_log(|logger| {
+                    slog::error!(logger, "Unable to receive ZMQ message: {}", e.to_string())
+                });
+                socket = new_zmq_socket();
+                assert!(socket.connect(&bitcoind_zmq_url).is_ok());
+                continue;
+            }
+        };
+        let (topic, data, _sequence) = (&msg[0], &msg[1], &msg[2]);
+
+        if !topic.eq(b"hashblock") {
+            ctx.try_log(|logger| slog::error!(logger, "Topic not supported",));
+            continue;
+        }
+
+        let block_hash = hex::encode(data);
+
+        ctx.try_log(|logger| slog::info!(logger, "Bitcoin block hash announced #{block_hash}",));
+
+        let mut block_hashes: VecDeque<String> = VecDeque::new();
+        block_hashes.push_front(block_hash);
+
+        while let Some(block_hash) = block_hashes.pop_front() {
+            let block = match download_and_parse_block_with_retry(
+                &http_client,
+                &block_hash,
+                &bitcoin_config,
+                &ctx,
+            )
+            .await
+            {
+                Ok(block) => block,
+                Err(e) => {
+                    ctx.try_log(|logger| {
+                        slog::warn!(
+                            logger,
+                            "unable to download_and_parse_block: {}",
+                            e.to_string()
+                        )
+                    });
+                    continue;
+                }
+            };
+
+            let header = block.get_block_header();
+            ctx.try_log(|logger| {
+                slog::info!(
+                    logger,
+                    "Bitcoin block #{} dispatched for processing",
+                    block.height
+                )
+            });
+
+            let _ = observer_commands_tx.send(ObserverCommand::ProcessBitcoinBlock(block));
+
+            if bitcoin_blocks_pool.can_process_header(&header) {
+                match bitcoin_blocks_pool.process_header(header, &ctx) {
+                    Ok(Some(event)) => {
+                        let _ = observer_commands_tx
+                            .send(ObserverCommand::PropagateBitcoinChainEvent(event));
+                    }
+                    Err(e) => {
+                        ctx.try_log(|logger| {
+                            slog::warn!(logger, "Unable to append block: {:?}", e)
+                        });
+                    }
+                    Ok(None) => {
+                        ctx.try_log(|logger| slog::warn!(logger, "Unable to append block"));
+                    }
+                }
+            } else {
+                // Handle a behaviour specific to ZMQ usage in bitcoind.
+                // Considering a simple re-org:
+                // A (1) - B1 (2) - C1 (3)
+                //       \ B2 (4) - C2 (5) - D2 (6)
+                // When D2 is being discovered (making A -> B2 -> C2 -> D2 the new canonical fork)
+                // it looks like ZMQ is only publishing D2.
+                // Without additional operation, we end up with a block that we can't append.
+                let parent_block_hash = header
+                    .parent_block_identifier
+                    .get_hash_bytes_str()
+                    .to_string();
+                ctx.try_log(|logger| {
+                    slog::info!(
+                        logger,
+                        "Possible re-org detected, retrieving parent block {parent_block_hash}"
+                    )
+                });
+                block_hashes.push_front(parent_block_hash);
+            }
+        }
+    }
+}

From e104bf0df95a967e52c5e088bab1a20544e10db7 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sun, 27 Aug 2023 11:07:56 -0400
Subject: [PATCH 15/20] chore: remove log

---
 components/chainhook-sdk/src/observer/zmq.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/zmq.rs b/components/chainhook-sdk/src/observer/zmq.rs
index 70de3ff6e..360d5bbdd 100644
--- a/components/chainhook-sdk/src/observer/zmq.rs
+++ b/components/chainhook-sdk/src/observer/zmq.rs
@@ -57,8 +57,6 @@ pub async fn start_zeromq_runloop(
     let mut bitcoin_blocks_pool = ForkScratchPad::new();
 
     loop {
-        ctx.try_log(|logger| slog::info!(logger, "Loop"));
-
         let msg = match socket.recv_multipart(0) {
             Ok(msg) => msg,
             Err(e) => {

From fc0966650a549133a68b1d5b4345b99e70d2d36a Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sun, 27 Aug 2023 11:14:21 -0400
Subject: [PATCH 16/20] chore: tweak keepalive config

---
 components/chainhook-sdk/src/observer/zmq.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/components/chainhook-sdk/src/observer/zmq.rs b/components/chainhook-sdk/src/observer/zmq.rs
index 360d5bbdd..0f53cc740 100644
--- a/components/chainhook-sdk/src/observer/zmq.rs
+++ b/components/chainhook-sdk/src/observer/zmq.rs
@@ -21,12 +21,12 @@ fn new_zmq_socket() -> Socket {
     assert!(socket.set_rcvhwm(0).is_ok());
     // We override the OS default behavior:
     assert!(socket.set_tcp_keepalive(1).is_ok());
-    // The keepalive routine will wait for 10 minutes
+    // The keepalive routine will wait for 5 minutes
     assert!(socket.set_tcp_keepalive_idle(300).is_ok());
-    // And then resend it every 30 seconds
-    assert!(socket.set_tcp_keepalive_intvl(30).is_ok());
-    // 100 times
-    assert!(socket.set_tcp_keepalive_cnt(100).is_ok());
+    // And then resend it every 60 seconds
+    assert!(socket.set_tcp_keepalive_intvl(60).is_ok());
+    // 120 times
+    assert!(socket.set_tcp_keepalive_cnt(120).is_ok());
     socket
 }
 

From 4762798f3c9b24aa31957b154824b6150eca3314 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Sun, 27 Aug 2023 11:16:36 -0400
Subject: [PATCH 17/20] chore: build warnings

---
 components/chainhook-sdk/Cargo.toml          | 2 +-
 components/chainhook-sdk/src/observer/mod.rs | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml
index c01767ae8..3c2cc3d92 100644
--- a/components/chainhook-sdk/Cargo.toml
+++ b/components/chainhook-sdk/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chainhook-sdk"
-version = "0.8.6"
+version = "0.8.7"
 description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin"
 license = "GPL-3.0"
 edition = "2021"
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index 8c8f033ff..cfce3a6dc 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -487,7 +487,7 @@ pub fn start_event_observer(
 
 pub async fn start_bitcoin_event_observer(
     config: EventObserverConfig,
-    observer_commands_tx: Sender<ObserverCommand>,
+    _observer_commands_tx: Sender<ObserverCommand>,
     observer_commands_rx: Receiver<ObserverCommand>,
     observer_events_tx: Option<crossbeam_channel::Sender<ObserverEvent>>,
     observer_sidecar: Option<ObserverSidecar>,
@@ -512,7 +512,7 @@ pub async fn start_bitcoin_event_observer(
         let ctx_moved = ctx.clone();
         let config_moved = config.clone();
         let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || {
-            let future = zmq::start_zeromq_runloop(&config_moved, observer_commands_tx, &ctx_moved);
+            let future = zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved);
             let _ = hiro_system_kit::nestable_block_on(future);
         });
     }

From d6d1cf2e50f9a136538c6abaa239dcc482aad4de Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 29 Aug 2023 10:29:30 -0400
Subject: [PATCH 18/20] chore: tweak chainhook-sdk

---
 Cargo.lock                                             |  2 +-
 components/chainhook-cli/Cargo.toml                    |  2 +-
 components/chainhook-sdk/Cargo.toml                    |  2 +-
 components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs |  4 ++--
 components/chainhook-sdk/src/observer/mod.rs           | 10 ++++++++--
 5 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2e3978907..764fe81b8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -439,7 +439,7 @@ dependencies = [
 
 [[package]]
 name = "chainhook-sdk"
-version = "0.8.6"
+version = "0.9.0"
 dependencies = [
  "base58 0.2.0",
  "base64",
diff --git a/components/chainhook-cli/Cargo.toml b/components/chainhook-cli/Cargo.toml
index 1e0e88dc2..596c9ec98 100644
--- a/components/chainhook-cli/Cargo.toml
+++ b/components/chainhook-cli/Cargo.toml
@@ -15,7 +15,7 @@ redis = "0.21.5"
 serde-redis = "0.12.0"
 hex = "0.4.3"
 rand = "0.8.5"
-chainhook-sdk = { version = "0.8.4", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" }
+chainhook-sdk = { version = "0.9.0", default-features = false, features = ["zeromq"], path = "../chainhook-sdk" }
 clarinet-files = "1.0.1"
 hiro-system-kit = "0.1.0"
 # clarinet-files = { path = "../../../clarinet/components/clarinet-files" }
diff --git a/components/chainhook-sdk/Cargo.toml b/components/chainhook-sdk/Cargo.toml
index 3c2cc3d92..96dd054ed 100644
--- a/components/chainhook-sdk/Cargo.toml
+++ b/components/chainhook-sdk/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "chainhook-sdk"
-version = "0.8.7"
+version = "0.9.0"
 description = "Stateless Transaction Indexing Engine for Stacks and Bitcoin"
 license = "GPL-3.0"
 edition = "2021"
diff --git a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs
index 1e971bebd..39485d1f9 100644
--- a/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs
+++ b/components/chainhook-sdk/src/chainhooks/bitcoin/mod.rs
@@ -24,12 +24,12 @@ pub struct BitcoinTriggerChainhook<'a> {
     pub rollback: Vec<(Vec<&'a BitcoinTransactionData>, &'a BitcoinBlockData)>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize)]
 pub struct BitcoinApplyTransactionPayload {
     pub block: BitcoinBlockData,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize)]
 pub struct BitcoinRollbackTransactionPayload {
     pub block: BitcoinBlockData,
 }
diff --git a/components/chainhook-sdk/src/observer/mod.rs b/components/chainhook-sdk/src/observer/mod.rs
index cfce3a6dc..33f031d26 100644
--- a/components/chainhook-sdk/src/observer/mod.rs
+++ b/components/chainhook-sdk/src/observer/mod.rs
@@ -60,6 +60,11 @@ pub enum Event {
     StacksChainEvent(StacksChainEvent),
 }
 
+pub enum DataHandlerEvent {
+    Process(BitcoinChainhookOccurrencePayload),
+    Terminate,
+}
+
 #[derive(Debug, Clone)]
 pub struct EventObserverConfig {
     pub chainhook_config: Option<ChainhookConfig>,
@@ -73,7 +78,7 @@ pub struct EventObserverConfig {
     pub cache_path: String,
     pub bitcoin_network: BitcoinNetwork,
     pub stacks_network: StacksNetwork,
-    pub data_handler_tx: Option<crossbeam_channel::Sender<BitcoinChainhookOccurrencePayload>>,
+    pub data_handler_tx: Option<crossbeam_channel::Sender<DataHandlerEvent>>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -512,7 +517,8 @@ pub async fn start_bitcoin_event_observer(
         let ctx_moved = ctx.clone();
         let config_moved = config.clone();
         let _ = hiro_system_kit::thread_named("ZMQ handler").spawn(move || {
-            let future = zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved);
+            let future =
+                zmq::start_zeromq_runloop(&config_moved, _observer_commands_tx, &ctx_moved);
             let _ = hiro_system_kit::nestable_block_on(future);
         });
     }

From d434c9362ec46b13f1a98d51f62d1c1938f70319 Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 12 Sep 2023 17:49:31 -0400
Subject: [PATCH 19/20] fix: workflow

---
 .github/workflows/ci.yaml | 464 ++++----------------------------------
 1 file changed, 47 insertions(+), 417 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 347580607..3c2daec2c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -1,447 +1,77 @@
 name: CI
+
 on:
-  pull_request:
   push:
     branches:
-      - main
       - develop
-      - rc/next
+      - main
+    tags-ignore:
+      - "**"
+    paths-ignore:
+      - "**/CHANGELOG.md"
+  pull_request:
   workflow_dispatch:
 
 jobs:
-  pre_run:
-    name: Cancel previous runs
+  build-publish:
     runs-on: ubuntu-latest
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@ad6cb1b847ffb509a69b745b6ee2f1d14dfe14b8
+      - uses: actions/checkout@v2
         with:
-          access_token: ${{ github.token }}
+          token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
           persist-credentials: false
 
-  get_release_info:
-    name: Get Release Info
-    runs-on: ubuntu-latest
-    needs: pre_run
-    outputs:
-      tag: ${{ steps.new_release_tag.outputs.TAG }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Get latest release
-        if: startsWith(github.ref, 'refs/heads/main')
-        id: release
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          repository: ${{ github.repository }}
-          excludes: prerelease, draft
-
-      - name: Determine if release build
-        if: startsWith(github.ref, 'refs/heads/main')
-        id: new_release_tag
-        env:
-          LATEST_RELEASE: ${{ steps.release.outputs.release }}
-        run: |
-          CARGO_VERSION=v$(grep "version" components/chainhook-cli/Cargo.toml | head -n 1 | cut -d\" -f2)
-          if [[ "${CARGO_VERSION}" != "${LATEST_RELEASE}" ]]; then
-            echo "::set-output name=TAG::${CARGO_VERSION}"
-            echo "::warning::Will create release for version: ${CARGO_VERSION}"
-          else
-            echo "::warning::Will not create a release"
-          fi
-
-  audit:
-    name: Audit and format
-    runs-on: ubuntu-latest
-    needs: pre_run
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Install Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rustfmt
-          override: true
-
-      - name: Set Cargo file permissions
-        run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/
-
-      - name: Cache cargo
-        uses: actions/cache@v2
-        with:
-          path: ~/.cargo/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install dependencies
-        run: cargo install cargo-audit
-
-      - name: Run audit
-        run: cargo audit --ignore RUSTSEC-2021-0076 --ignore RUSTSEC-2021-0119 --ignore RUSTSEC-2022-0028 --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2021-0124 --ignore RUSTSEC-2022-0040 --ignore RUSTSEC-2023-0018
-
-      - name: Run rustfmt
-        run: cargo fmt --all -- --check
-
-  # test_coverage_cargo:
-  #   name: Generate test coverage
-  #   runs-on: ubuntu-latest
-  #   needs: pre_run
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       include:
-  #         - name: clarinet
-  #           working-directory: components/clarinet-cli
-  #         - name: clarity-repl
-  #           working-directory: components/clarity-repl
-  #         - name: chainhook-cli
-  #           working-directory: components/chainhook-event-observer
-  #   steps:
-  #     - name: Checkout repository
-  #       uses: actions/checkout@v3
-  #       with:
-  #         submodules: recursive
-
-  #     - name: Install Rust toolchain stable
-  #       uses: actions-rs/toolchain@v1
-  #       with:
-  #         toolchain: stable
-  #         profile: minimal
-  #         override: true
-
-  #     - name: Cache cargo
-  #       uses: actions/cache@v2
-  #       id: cache-cargo
-  #       with:
-  #         path: |
-  #           ~/.cargo/bin/
-  #           ~/.cargo/registry/index/
-  #           ~/.cargo/registry/cache/
-  #           ~/.cargo/git/db/
-  #           target/
-  #         key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-  #     - name: Install dependencies
-  #       if: steps.cache-cargo.outputs.cache-hit != 'true'
-  #       run: RUSTC_BOOTSTRAP=1 cargo install grcov
-
-  #     - name: Install Rust toolchain
-  #       uses: actions-rs/toolchain@v1
-  #       with:
-  #         toolchain: stable
-  #         profile: minimal
-  #         components: llvm-tools-preview
-  #         override: true
-
-  #     - name: Unit Tests
-  #       env:
-  #         RUSTFLAGS: "-C instrument-coverage"
-  #         LLVM_PROFILE_FILE: "${{ matrix.name }}-%p-%m.profraw"
-  #       run: cargo build --package=clarinet-cli --locked && cargo test --package=clarinet-cli
-
-  #     - name: Generate coverage
-  #       run: grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info
-
-  #       # Run functional tests here in addition to the other jobs so we can fail fast
-  #       # Since these tests are reached much earlier in the pipeline
-  #     - name: Functional Tests
-  #       if: matrix.name == 'clarinet'
-  #       run: |
-  #         for testdir in $(ls components/clarinet-cli/examples); do
-  #           ./target/debug/clarinet test --manifest-path components/clarinet-cli/examples/${testdir}/Clarinet.toml
-  #         done
-
-  #     - name: Upload coverage report
-  #       uses: codecov/codecov-action@v1
-  #       with:
-  #         flags: unittests
-  #         name: ${{ matrix.name }}
-  #         verbose: true
-
-  dist_chainhook:
-    name: Build Chainhook Distributions
-    runs-on: ${{ matrix.os }}
-    needs: pre_run
-    # Related upstream issue:
-    # https://github.com/nagisa/rust_libloading/issues/61#issuecomment-607941377
-    #
-    # env:
-    #  CC: deny_c
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            platform: linux
-            target: x86_64-unknown-linux-gnu
-            architecture: x64
-            libc: glibc
-          - os: windows-latest
-            platform: windows
-            target: x86_64-pc-windows-msvc
-            architecture: x64
-          - os: macos-latest
-            platform: darwin
-            target: x86_64-apple-darwin
-            architecture: x64
-          - os: macos-latest
-            platform: darwin
-            target: aarch64-apple-darwin
-            architecture: arm64
-
-    steps:
-      - name: Configure git to use LF (Windows)
-        if: matrix.os == 'windows-latest'
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Install Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          target: ${{ matrix.target }}
-          profile: minimal
-          components: llvm-tools-preview
-          override: true
-
-      - name: Install wix (Windows)
-        if: matrix.os == 'windows-latest'
-        run: cargo install cargo-wix
-
-      - if: matrix.os != 'windows-latest'
-        run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/
-
-      - name: Cache cargo
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/${{ matrix.target }}/release/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      # Set environment variables required from cross compiling from macos-x86_64 to macos-arm64
-      - name: Configure macos-arm64 cross compile config
-        if: matrix.target == 'aarch64-apple-darwin'
-        run: |
-          echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV
-          echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
-
-      - name: Configure artifact names (libc)
-        if: ${{ matrix.libc }}
-        shell: bash
-        run: |
-          echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
-          echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-${{ matrix.libc }}" >> $GITHUB_ENV
-
-      - name: Configure artifact names (not libc)
-        if: ${{ ! matrix.libc }}
-        shell: bash
-        run: |
-          echo "SHORT_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}" >> $GITHUB_ENV
-          echo "PRE_GYP_TARGET_NAME=${{ matrix.platform }}-${{ matrix.architecture }}-unknown" >> $GITHUB_ENV
-
-      - name: Build - Cargo
-        if: matrix.target != 'x86_64-unknown-linux-musl'
-        run: cargo build --release --features=telemetry --locked --target ${{ matrix.target }}
-
-      - name: Code sign bin (Windows)
-        if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
-        run: |
-          $certificate_file_name = "${env:TEMP}\certificate.pfx"
-
-          $bytes_cert = [Convert]::FromBase64String('${{ secrets.WINDOWS_CODE_SIGNING_CERTIFICATE }}')
-          [IO.File]::WriteAllBytes(${certificate_file_name}, ${bytes_cert})
-
-          $signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe"
-          $bin_path = (Resolve-Path -Path "target/${{ matrix.target }}/release/chainhook.exe").Path
-          & ${signtool_path} sign `
-            /d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." `
-            /du "https://github.com/hirosystems/chainhook" `
-            /tr http://timestamp.digicert.com `
-            /td sha256 `
-            /fd sha256 `
-            -f "${certificate_file_name}" `
-            -p "${{ secrets.WINDOWS_CODE_SIGNING_PASSWORD }}" `
-            "${bin_path}"
-
-      - name: Build Installer (Windows)
-        if: matrix.os == 'windows-latest'
-        run: cargo wix -v --no-build --nocapture -p chainhook-cli
-
-      - name: Code sign installer (Windows)
-        if: startsWith(github.ref, 'refs/heads/main') && matrix.os == 'windows-latest'
+      - name: Cargo test
         run: |
-          $certificate_file_name = "${env:TEMP}\certificate.pfx"
-
-          $bytes_cert = [Convert]::FromBase64String('${{ secrets.WINDOWS_CODE_SIGNING_CERTIFICATE }}')
-          [IO.File]::WriteAllBytes(${certificate_file_name}, ${bytes_cert})
-
-          $signtool_path = ((Resolve-Path -Path "${env:ProgramFiles(x86)}/Windows Kits/10/bin/10*/x86").Path[-1]) + "/signtool.exe"
-          $msi_path = (Resolve-Path -Path "target/wix/*.msi").Path
-          & ${signtool_path} sign `
-            /d "Clarinet is a clarity runtime packaged as a command line tool, designed to facilitate smart contract understanding, development, testing and deployment." `
-            /du "https://github.com/hirosystems/chainhook" `
-            /tr http://timestamp.digicert.com `
-            /td sha256 `
-            /fd sha256 `
-            -f "${certificate_file_name}" `
-            -p "${{ secrets.WINDOWS_CODE_SIGNING_PASSWORD }}" `
-            "${msi_path}"
-
-      # Don't compress for Windows because winget can't yet unzip files
-      - name: Compress cargo artifact (Linux)
-        if: matrix.os != 'windows-latest'
-        run: tar -C target/${{ matrix.target }}/release -zcvf chainhook-${{ env.SHORT_TARGET_NAME }}.tar.gz chainhook
-
-      - name: Rename cargo artifact (Windows)
-        if: matrix.os == 'windows-latest'
-        shell: bash
-        run: mv target/wix/*.msi chainhook-${{ env.SHORT_TARGET_NAME }}.msi
-
-      # Separate uploads to prevent paths from being preserved
-      - name: Upload cargo artifacts (Linux)
-        if: matrix.os != 'windows-latest'
-        uses: actions/upload-artifact@v2
-        with:
-          name: chainhook-${{ env.SHORT_TARGET_NAME }}
-          path: chainhook-${{ env.SHORT_TARGET_NAME }}.tar.gz
-
-      - name: Upload cargo artifact (Windows)
-        if: matrix.os == 'windows-latest'
-        uses: actions/upload-artifact@v2
+          rustup update
+          cargo check
+          cargo test --all
+
+      - name: Semantic Release
+        uses: cycjimmy/semantic-release-action@v3
+        id: semantic
+        # Only run on non-PR events or only PRs that aren't from forks
+        if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }}
+          SEMANTIC_RELEASE_PACKAGE: ${{ github.event.repository.name }}
         with:
-          name: chainhook-${{ env.SHORT_TARGET_NAME }}
-          path: chainhook-${{ env.SHORT_TARGET_NAME }}.msi
+          semantic_version: 19
+          extra_plugins: |
+            @semantic-release/changelog@6.0.3
+            @semantic-release/git@10.0.1
+            conventional-changelog-conventionalcommits@6.1.0
 
-      - name: Unit Tests - Cargo
-        # can't easily run mac-arm64 tests in GH without native runners for that arch
-        if: matrix.target != 'aarch64-apple-darwin'
-        run: cargo test --release --locked --target ${{ matrix.target }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
 
-      # - name: Functional Tests (Linux)
-      #   # can't easily run mac-arm64 tests in GH without native runners for that arch
-      #   if: matrix.os != 'windows-latest' && matrix.target != 'aarch64-apple-darwin'
-      #   run: |
-      #     for testdir in $(ls components/chainhook-cli/examples); do
-      #       ./target/${{ matrix.target }}/release/clarinet test --manifest-path components/clarinet-cli/examples/${testdir}/Clarinet.toml
-      #     done
-
-      # - name: Functional Tests (Windows)
-      #   if: matrix.os == 'windows-latest'
-      #   run: |
-      #     foreach($testdir in Get-ChildItem components/clarinet-cli/examples) {
-      #       ./target/${{ matrix.target }}/release/clarinet test --manifest-path ${testdir}/Clarinet.toml
-      #     }
-
-  docker_images:
-    name: Create ${{ matrix.name }} Docker Image
-    runs-on: ubuntu-latest
-    needs:
-      - get_release_info
-      - dist_chainhook_node
-    outputs:
-      version: ${{ steps.docker_meta.outputs.version }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - name: Chainhook Node
-            description: A Stacks event observer.
-            image: ${{ github.repository_owner }}/chainhook-cli
-            artifact: chainhook-cli-linux-x64-glibc
-            dockerfile: dockerfiles/components/chainhook-cli.dockerfile
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Generate Docker tags/labels
-        id: docker_meta
+      - name: Docker Meta
+        id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ matrix.image }}
+          images: |
+            blockstack/${{ github.event.repository.name }}
+            hirosystems/${{ github.event.repository.name }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
-            type=semver,pattern={{version}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }}
-            type=semver,pattern={{major}}.{{minor}},value=${{ needs.get_release_info.outputs.tag }},enable=${{ needs.get_release_info.outputs.tag != '' }}
-          labels: |
-            org.opencontainers.image.title=${{ matrix.name }}
-            org.opencontainers.image.description=${{ matrix.description }}
+            type=semver,pattern={{version}},value=${{ steps.semantic.outputs.new_release_version }},enable=${{ steps.semantic.outputs.new_release_version != '' }}
+            type=semver,pattern={{major}}.{{minor}},value=${{ steps.semantic.outputs.new_release_version }},enable=${{ steps.semantic.outputs.new_release_version != '' }}
+            type=raw,value=latest,enable={{is_default_branch}}
 
-      - name: Login to Dockerhub
+      - name: Login to DockerHub
         uses: docker/login-action@v2
-        if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Download pre-built dist
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ matrix.artifact }}
-
-      - name: Untar pre-built dist
-        run: tar zxvf *.tar.gz
-
-      - name: Create Image
-        uses: docker/build-push-action@v3
+      - name: Build/Tag/Push Image
+        uses: docker/build-push-action@v2
         with:
           context: .
-          file: ${{ matrix.dockerfile }}
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-
-  release:
-    name: Release
-    runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/heads/main') && needs.get_release_info.outputs.tag != ''
-    needs:
-      - get_release_info
-      - audit
-      # - test_coverage_cargo
-      - docker_images
-    permissions:
-      actions: write
-      contents: write
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-
-      - name: Download pre-built dists
-        uses: actions/download-artifact@v3
-
-      - name: Tag and Release
-        uses: ncipollo/release-action@v1
-        with:
-          artifacts: "**/*.tar.gz,**/*.msi"
-          tag: ${{ needs.get_release_info.outputs.tag }}
-          commit: ${{ env.GITHUB_SHA }}
-
-      - name: Trigger pkg-version-bump workflow
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          event-type: released
-          client-payload: '{"tag": "${{ needs.get_release_info.outputs.tag }}"}'
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ./dockerfiles/components/chainhook.dockerfile
+          # Only push if (there's a new release on main branch, or if building a non-main branch) and (Only run on non-PR events or only PRs that aren't from forks)
+          push: ${{ (github.ref != 'refs/heads/master' || steps.semantic.outputs.new_release_version != '') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}

From 15da6612d852089ecdcfd8831201c3722663e5bb Mon Sep 17 00:00:00 2001
From: Ludo Galabru <ludo@hiro.so>
Date: Tue, 12 Sep 2023 17:49:54 -0400
Subject: [PATCH 20/20] chore: clean gitignore

---
 .gitignore | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index 837e005db..06ccb5f4c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,16 +12,9 @@ npm-debug.log*
 **/.requirements
 **/.cache
 **/.build
-components/stacks-devnet-js/dist
-components/stacks-devnet-js/build
 components/chainhook-types-js/dist
 *.tar.gz
 *.zip
 *.rdb
-
-components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/bin
-components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/tmp
-components/chainhook-db/examples/arkadiko-data-indexing/vault-monitor/vendor
-components/chainhook-cli/cache
-components/chainhook-cli/index.redb
+*.redb
 cache/