Skip to content

Commit

Permalink
fix(storage/events): add upper bounds on queries used to select strategy
Browse files Browse the repository at this point in the history
Since we already have a cost limit on the event queries there's no point
in counting the exact number of matching events by the keys or range.

It's sufficient to be able to count up to reaching a limit where the
cost would be too much anyway.

This avoids issues with queries using keys of which there are a few
hundred million events in the database.
  • Loading branch information
kkovaacs committed Dec 5, 2023
1 parent 7b61edf commit 430118a
Showing 1 changed file with 12 additions and 5 deletions.
17 changes: 12 additions & 5 deletions crates/storage/src/connection/event.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ pub const PAGE_SIZE_LIMIT: usize = 1_024;
pub const KEY_FILTER_LIMIT: usize = 256;

const KEY_FILTER_COST_LIMIT: usize = 1_000_000;
const KEY_FILTER_WEIGHT: usize = 50;

const KEY_FILTER_UPPER_BOUND: usize = KEY_FILTER_COST_LIMIT / KEY_FILTER_WEIGHT;
const RANGE_FILTER_UPPER_BOUND: usize = KEY_FILTER_COST_LIMIT;

pub struct EventFilter<K: KeyFilter> {
pub from_block: Option<BlockNumber>,
Expand Down Expand Up @@ -347,8 +351,8 @@ impl KeyFilter for V03KeyFilter {
None => Ok(None),
Some(key_fts_expression) => {
let count: usize = tx.inner().query_row(
"SELECT COUNT(1) FROM starknet_events_keys_03 WHERE keys MATCH :events_match",
[&key_fts_expression],
"SELECT COUNT(1) FROM (SELECT 1 FROM starknet_events_keys_03 WHERE keys MATCH :events_match LIMIT :limit)",
params!(key_fts_expression, &(KEY_FILTER_UPPER_BOUND as u64)),
|row| row.get(0),
)?;
Ok(Some(count))
Expand Down Expand Up @@ -472,7 +476,8 @@ fn select_query_strategy(
let events_by_key_filter = number_of_events_by_key_filter(tx, keys)?;
if let Some(events_by_key_filter) = events_by_key_filter {
// shortcut if the key filter is specific enough
if events_by_key_filter < 100_000 {
// KEY_FILTER_UPPER_BOUND really means >= KEY_FILTER_UPPER_BOUND
if events_by_key_filter < KEY_FILTER_UPPER_BOUND {
tracing::trace!(
%events_by_key_filter,
"Partial queries for number of events done"
Expand All @@ -493,7 +498,6 @@ fn select_query_strategy(
"Partial queries for number of events done"
);

const KEY_FILTER_WEIGHT: usize = 50;
let weighted_events_by_key_filter =
events_by_key_filter.map(|n| n.saturating_mul(KEY_FILTER_WEIGHT));

Expand Down Expand Up @@ -536,7 +540,7 @@ fn number_of_events_in_block_range(
to_block: Option<&BlockNumber>,
contract_address: Option<&ContractAddress>,
) -> anyhow::Result<Option<usize>> {
let mut query = "SELECT COUNT(1) FROM starknet_events WHERE ".to_owned();
let mut query = "SELECT COUNT(1) FROM (SELECT 1 FROM starknet_events WHERE ".to_owned();
let mut params: Vec<(&str, rusqlite::types::ToSqlOutput<'_>)> = Vec::new();

match (from_block, to_block) {
Expand Down Expand Up @@ -566,6 +570,9 @@ fn number_of_events_in_block_range(
params.push((":contract_address", contract_address.to_sql()));
}

query.push_str(" LIMIT :limit)");
params.push((":limit", (RANGE_FILTER_UPPER_BOUND as u64).to_sql()));

let params = params
.iter()
.map(|(s, x)| (*s, x as &dyn rusqlite::ToSql))
Expand Down

0 comments on commit 430118a

Please sign in to comment.