Skip to content

Commit

Permalink
Use JSON RPC API instead of blocksdir(#139)
Browse files Browse the repository at this point in the history
  • Loading branch information
@casey
casey authored Feb 19, 2022
1 parent a18eaac commit a400bd5
Show file tree
Hide file tree
Showing 13 changed files with 860 additions and 406 deletions.
730 changes: 718 additions & 12 deletions Cargo.lock
View file

Large diffs are not rendered by default.

12 changes: 9 additions & 3 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,28 @@ autotests = false

[dependencies]
bitcoin = "0.27.1"
bitcoincore-rpc = "0.14.0"
derive_more = "0.99.17"
dirs = "4.0.0"
env_logger = "0.9.0"
executable-path = "1.0.0"
integer-cbrt = "0.1.2"
integer-sqrt = "0.1.5"
jsonrpc = "0.12.1"
log = "0.4.14"
memmap2 = "0.5.3"
redb = { version = "0.0.4", git = "https://github.com/cberner/redb.git" }
structopt = "0.3.25"
tempfile = "3.2.0"
unindent = "0.1.7"

[dev-dependencies]
criterion = "0.3.5"
hex = "0.4.3"
jsonrpc-core = "18.0.0"
jsonrpc-core-client = "18.0.0"
jsonrpc-derive = "18.0.0"
jsonrpc-http-server = "18.0.0"
regex = "1.5.4"
tempfile = "3.2.0"
unindent = "0.1.7"

[[test]]
name = "integration"
Expand Down
201 changes: 17 additions & 184 deletions src/index.rs
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,21 @@
use super::*;
use {
super::*,
bitcoincore_rpc::{Auth, Client, RpcApi},
};

pub(crate) struct Index {
blocksdir: PathBuf,
client: Client,
database: Database,
}

impl Index {
const HASH_TO_CHILDREN: &'static str = "HASH_TO_CHILDREN";
const HASH_TO_HEIGHT: &'static str = "HASH_TO_HEIGHT";
const HASH_TO_LOCATION: &'static str = "HASH_TO_LOCATION";
const HEIGHT_TO_HASH: &'static str = "HEIGHT_TO_HASH";
const OUTPOINT_TO_ORDINAL_RANGES: &'static str = "OUTPOINT_TO_ORDINAL_RANGES";

pub(crate) fn new(blocksdir: Option<&Path>, index_size: Option<usize>) -> Result<Self> {
let blocksdir = if let Some(blocksdir) = blocksdir {
blocksdir.to_owned()
} else if cfg!(target_os = "macos") {
dirs::home_dir()
.ok_or("Unable to retrieve home directory")?
.join("Library/Application Support/Bitcoin/blocks")
} else if cfg!(target_os = "windows") {
dirs::data_dir()
.ok_or("Unable to retrieve home directory")?
.join("Bitcoin/blocks")
} else {
dirs::home_dir()
.ok_or("Unable to retrieve home directory")?
.join(".bitcoin/blocks")
};
pub(crate) fn new(index_size: Option<usize>) -> Result<Self> {
let bitcoin_core_rpc_url =
env::var("ORD_BITCOIN_CORE_RPC_URL").map_err(|err| format!("Failed to get Bitcoin Core JSON RPC URL from ORD_BITCOIN_CORE_RPC_URL environment variable: {err}"))?;

let client = Client::new(&bitcoin_core_rpc_url, Auth::None)?;

let result = unsafe { Database::open("index.redb") };

Expand All @@ -39,14 +27,7 @@ impl Index {
Err(error) => return Err(error.into()),
};

let index = Self {
database,
blocksdir,
};

index.index_blockfiles()?;

index.index_heights()?;
let index = Self { client, database };

index.index_ranges()?;

Expand Down Expand Up @@ -172,161 +153,13 @@ impl Index {
Ok(())
}

fn blockfile_path(&self, i: u64) -> PathBuf {
self.blocksdir.join(format!("blk{:05}.dat", i))
}

fn index_blockfiles(&self) -> Result {
let blockfiles = (0..)
.map(|i| self.blockfile_path(i))
.take_while(|path| path.is_file())
.count();

log::info!("Indexing {} blockfiles…", blockfiles);

for i in 0.. {
let path = self.blockfile_path(i);

if !path.is_file() {
break;
}

let blocks = unsafe { Mmap::map(&File::open(path)?)? };

let tx = self.database.begin_write()?;

let mut hash_to_children: MultimapTable<[u8], [u8]> =
tx.open_multimap_table(Self::HASH_TO_CHILDREN)?;

let mut hash_to_location: Table<[u8], u64> = tx.open_table(Self::HASH_TO_LOCATION)?;

let mut offset = 0;

let mut count = 0;

let mut genesis = false;

loop {
if offset == blocks.len() {
break;
}

let rest = &blocks[offset..];

if rest.starts_with(&[0, 0, 0, 0]) {
break;
}

let block = Self::extract_block(rest)?;

let header = BlockHeader::consensus_decode(&block[0..80])?;
let hash = header.block_hash();

if header.prev_blockhash == Default::default() {
if genesis {
return Err("Duplicate genesis block found".into());
}

let mut hash_to_height: Table<[u8], u64> = tx.open_table(Self::HASH_TO_HEIGHT)?;
let mut height_to_hash: Table<u64, [u8]> = tx.open_table(Self::HEIGHT_TO_HASH)?;

hash_to_height.insert(&hash, &0)?;
height_to_hash.insert(&0, &hash)?;

genesis = true;
}

hash_to_children.insert(&header.prev_blockhash, &hash)?;

hash_to_location.insert(&hash, &((i as u64) << 32 | offset as u64))?;

offset = offset + 8 + block.len();

count += 1;
}

log::info!("{}/{}: Processed {} blocks…", i + 1, blockfiles, count);

tx.commit()?;
}

Ok(())
}

fn index_heights(&self) -> Result {
log::info!("Indexing heights…");

let write = self.database.begin_write()?;

let read = self.database.begin_read()?;

let hash_to_children: ReadOnlyMultimapTable<[u8], [u8]> =
read.open_multimap_table(Self::HASH_TO_CHILDREN)?;

let mut hash_to_height: Table<[u8], u64> = write.open_table(Self::HASH_TO_HEIGHT)?;
let mut height_to_hash: Table<u64, [u8]> = write.open_table(Self::HEIGHT_TO_HASH)?;

let mut queue = vec![(
height_to_hash
.get(&0)?
.ok_or("Could not find genesis block in index")?
.to_vec(),
0,
)];

while let Some((block, height)) = queue.pop() {
hash_to_height.insert(block.as_ref(), &height)?;
height_to_hash.insert(&height, block.as_ref())?;

let mut iter = hash_to_children.get(&block)?;

while let Some(child) = iter.next() {
queue.push((child.to_vec(), height + 1));
}
}

write.commit()?;

Ok(())
}

fn extract_block(blocks: &[u8]) -> Result<&[u8]> {
let magic = &blocks[0..4];
if magic != Network::Bitcoin.magic().to_le_bytes() {
return Err(format!("Unknown magic bytes: {:?}", magic).into());
}

let len = u32::from_le_bytes(blocks[4..8].try_into()?) as usize;

Ok(&blocks[8..8 + len])
}

pub(crate) fn block(&self, height: u64) -> Result<Option<Block>> {
let tx = self.database.begin_read()?;

let heights_to_hash: ReadOnlyTable<u64, [u8]> = tx.open_table(Self::HEIGHT_TO_HASH)?;

match heights_to_hash.get(&height)? {
None => Ok(None),
Some(guard) => {
let hash = guard;

let hash_to_location: ReadOnlyTable<[u8], u64> = tx.open_table(Self::HASH_TO_LOCATION)?;

let location = hash_to_location
.get(hash)?
.ok_or("Could not find block location in index")?;

let path = self.blockfile_path(location >> 32);

let offset = (location & 0xFFFFFFFF) as usize;

let blocks = unsafe { Mmap::map(&File::open(path)?)? };

let bytes = Self::extract_block(&blocks[offset..])?;

Ok(Some(Block::consensus_decode(bytes)?))
}
match self.client.get_block_hash(height) {
Ok(hash) => Ok(Some(self.client.get_block(&hash)?)),
Err(bitcoincore_rpc::Error::JsonRpc(jsonrpc::error::Error::Rpc(
jsonrpc::error::RpcError { code: -8, .. },
))) => Ok(None),
Err(err) => Err(err.into()),
}
}

Expand Down
15 changes: 3 additions & 12 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,18 @@ use {
arguments::Arguments, epoch::Epoch, height::Height, index::Index, ordinal::Ordinal,
sat_point::SatPoint, subcommand::Subcommand,
},
bitcoin::{
blockdata::constants::COIN_VALUE,
consensus::{Decodable, Encodable},
Block, BlockHeader, Network, OutPoint, Transaction,
},
bitcoin::{blockdata::constants::COIN_VALUE, consensus::Encodable, Block, OutPoint, Transaction},
derive_more::{Display, FromStr},
integer_cbrt::IntegerCubeRoot,
integer_sqrt::IntegerSquareRoot,
memmap2::Mmap,
redb::{
Database, MultimapTable, ReadOnlyMultimapTable, ReadOnlyTable, ReadableMultimapTable,
ReadableTable, Table,
},
redb::{Database, ReadOnlyTable, ReadableTable, Table},
std::{
cmp::Ordering,
collections::VecDeque,
env,
fmt::{self, Display, Formatter},
fs::File,
io,
ops::{Add, AddAssign, Deref, Sub},
path::{Path, PathBuf},
process,
str::FromStr,
},
Expand Down
4 changes: 2 additions & 2 deletions src/subcommand.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ mod traits;
pub(crate) enum Subcommand {
Epochs,
Find(find::Find),
Index(index::Index),
Index,
List(list::List),
Name(name::Name),
Range(range::Range),
Expand All @@ -26,7 +26,7 @@ impl Subcommand {
match self {
Self::Epochs => epochs::run(),
Self::Find(find) => find.run(index_size),
Self::Index(index) => index.run(index_size),
Self::Index => index::run(index_size),
Self::List(list) => list.run(index_size),
Self::Name(name) => name.run(),
Self::Range(range) => range.run(),
Expand Down
4 changes: 1 addition & 3 deletions src/subcommand/find.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ use super::*;

#[derive(StructOpt)]
pub(crate) struct Find {
#[structopt(long)]
blocksdir: Option<PathBuf>,
#[structopt(long)]
as_of_height: u64,
#[structopt(long)]
Expand All @@ -13,7 +11,7 @@ pub(crate) struct Find {

impl Find {
pub(crate) fn run(self, index_size: Option<usize>) -> Result<()> {
let index = Index::new(self.blocksdir.as_deref(), index_size)?;
let index = Index::new(index_size)?;

let creation_height = self.ordinal.height().n();
let block = index.block(creation_height)?.unwrap();
Expand Down
14 changes: 3 additions & 11 deletions src/subcommand/index.rs
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,6 @@
use super::*;

#[derive(StructOpt)]
pub(crate) struct Index {
#[structopt(long)]
blocksdir: Option<PathBuf>,
}

impl Index {
pub(crate) fn run(self, index_size: Option<usize>) -> Result<()> {
crate::Index::new(self.blocksdir.as_deref(), index_size)?;
Ok(())
}
pub(crate) fn run(index_size: Option<usize>) -> Result<()> {
Index::new(index_size)?;
Ok(())
}
4 changes: 1 addition & 3 deletions src/subcommand/list.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@ use super::*;

#[derive(StructOpt)]
pub(crate) struct List {
#[structopt(long)]
blocksdir: Option<PathBuf>,
outpoint: OutPoint,
}

impl List {
pub(crate) fn run(self, index_size: Option<usize>) -> Result<()> {
let index = Index::new(self.blocksdir.as_deref(), index_size)?;
let index = Index::new(index_size)?;
let ranges = index.list(self.outpoint)?;

for (start, end) in ranges {
Expand Down
Loading

0 comments on commit a400bd5

Please sign in to comment.