From 77de6dbf6131080638f64d8f7cd2418c51306a71 Mon Sep 17 00:00:00 2001 From: Sergiu Popescu <44298302+sergiupopescu199@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:21:00 +0200 Subject: [PATCH] refactor(iota-indexer): add db schema diagram in README.md (#1594) * refactor(iota-indexer): add db schema diagram in README.md * refactor(iota-indexer): revert to previous formatting * refactor(iota-indexer): add margin to diagram * refactor(iota-indexer): make ERD diagram more concise * refactor(iota-indexer): describe all db schema in a separate schema.md file * refactor(iota-indexer): remove redundant tables from schema.md * refactor(iota-indexer): fix spacing in schema.rs * refactor(iota-indexer): from md files with drprint * refactor(iota-indexer): use list instead of tables in schema.md * refactor(iota-indexer): add index names in schema.md --- crates/iota-indexer/README.md | 4 ++ crates/iota-indexer/database_schema.svg | 4 ++ crates/iota-indexer/schema.md | 95 +++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 crates/iota-indexer/database_schema.svg create mode 100644 crates/iota-indexer/schema.md diff --git a/crates/iota-indexer/README.md b/crates/iota-indexer/README.md index 280defdf737..837139ea46d 100644 --- a/crates/iota-indexer/README.md +++ b/crates/iota-indexer/README.md @@ -13,6 +13,10 @@ Iota indexer is an off-fullnode service to serve data from Iota protocol, includ > - Nodes expose the `NameServiceConfig` API, whereas indexer instances do not. > - Indexer instance expose the `ExtendedApi`, but nodes do not. +## Database Schema + +For more in depth information check the [Database Schema](./schema.md). + ## Steps to run locally ### Prerequisites diff --git a/crates/iota-indexer/database_schema.svg b/crates/iota-indexer/database_schema.svg new file mode 100644 index 00000000000..8f5adc652f0 --- /dev/null +++ b/crates/iota-indexer/database_schema.svg @@ -0,0 +1,4 @@ + + + +
display
PK
object_type text NN
bcs      bytea NN
id      bytea NN
version         int2 NN
objects
PK
object_id  bytea NN
checkpoint_sequence_number ref(checkpoints)    int8 NN
coin_balance                    int8
coint_type             text
df_kind            int2
df_name             bytea
df_object_id             bytea
df_object_type                  text
object_digest                 bytea NN
object_type            text
object_version                 in8 NN
owner_id             bytea
owner_type              int2 NN
serialized_object            bytea NN
objects_snapshot
PK
object_id  ref(objects_history) bytea NN
checkpoint_sequence_number   int8 NN
coin_balance                          int8
coint_type          text
df_kind              int2
df_name           bytea
df_object_id           bytea
df_object_type              text
object_digest                       bytea
object_status             int2 NN
object_type       text
object_version                     in8 NN
owner_id               bytea
owner_type                      int2
serialized_object                          bytea
objects_history
PK
checkpoint_sequence_number ref(checkpoints) int8 NN
PK
object_id  ref(objects) bytea NN
PK
object_version  int8 NN
coin_balance                                               int8
coint_type            text
df_kind           int2
df_name   bytea
df_object_id   bytea
df_object_type      text
object_digest                bytea
object_status           int2 NN
object_type      text
owner_id    bytea
owner_type                   int2
serialized_object                 bytea
epochs
PK
epoch  int8 NN
burn_leftover_amount             int8
burn_tokens_amount              int8
epoch_commitments      bytea
epoch_end_timestamp             int8
epoch_start_timestamp int8 NN
epoch_total_transactions             int8
first_checkpoint_id ref(checkpoints) int8 NN
last_checkpoint_id ref(checkpoints)            int8
minted_tokens_amount             int8
protocol_version                 int8 NN
reference_gas_price int8 NN
stake_subsidy_amount             int8
storage_charge             int8
storage_fund_balance int8 NN
storage_rebate             int8
system_state          bytea NN
total_gas_fees             int8
total_stake int8 NN
total_stake_reward_distributed            int8
checkpoints
PK
sequence_number  int8 NN
checkpoint_digest           bytea NN
epoch      int8 NN
network_total_transactions     int8 NN
previous_checkpoint_digest    bytea
end_of_epoch        bool NN
end_of_epoch_data    bytea
tx_digests    bytea[] NN
timestamp_ms                        int8 NN
total_gas_cost                        int8 NN
computation_cost                   int8 NN
storage_cost                           int8 NN
storage_rebate                       int8 NN
non_refundable_storage_fee int8 NN
checkpoint_commitments    bytea NN
validator_signature              bytea NN
end_of_epoch_data                   bytea
transactions
PK
checkpoint_sequence_number     ref(checkpoints) int8 NN
PK
tx_sequence_number int8 NN
balance_changes                         bytea[] NN
events                                           bytea[] NN
object_changes                                 bytea[] NN
raw_effects                                             bytea NN
raw_transaction                                   bytea NN
success_command_count                    int2 NN
timestamp_ms    ref(checkpoints)                                   int8 NN
transaction_digest    ref(checkpoints)                          bytea NN
transaction_kind                                    int2 NN
events
PK
event_ sequence_number int8 NN
PK
tx_sequence_number    ref(transactions) int8 NN
bcs                        bytea NN
checkpoint_sequence_number    ref(checkpoints) int8 NN
event_type         text NN
module text NN
package            bytea NN
senders                bytea[] NN
timestamp_ms    ref(checkpoints)                            int8 NN
transaction_digest    ref(transactions)                         bytea NN
tx_senders
PK
sender bytea NN
PK
tx_sequence_number    ref(transactions).    int8 NN
tx_recipients
PK
recipient bytea NN
PK
tx_sequence_number    ref(transactions).   int8 NN
tx_calls
PK
package    ref(packages)     bytea NN
PK
tx_sequence_number    ref(transactions)   int8 NN
bcs        bytea NN
id      bytea NN
version             int2 NN
tx_input_objects
PK
object_id bytea NN
PK
tx_sequence_number     ref(transactions)    int8 NN
tx_changed_objects
PK
object_id bytea NN
PK
tx_sequence_number     ref(transactions)  int8 NN
packages
PK
package_id bytea NN
move_package bytea NN
__diesel_schema_migrations
PK
version varchar(50) NN
run_on                       timestamp NN
Legend
Database schema
ref(< table-name >)    Possible Foreign Key link
 \ No newline at end of file diff --git a/crates/iota-indexer/schema.md b/crates/iota-indexer/schema.md new file mode 100644 index 00000000000..2dad7161ac9 --- /dev/null +++ b/crates/iota-indexer/schema.md @@ -0,0 +1,95 @@ +# Database Schema + +The Indexer pulls checkpoint data from the full node and populates the tables shown in the ERD diagram: + +![Database Schema](./database_schema.svg) + +> **NOTE** +> +> Since all tables are populated from checkpoint data the entities are closely related, in the `SQL` files the foreign keys are not specified, a manual study of data is needed to understand all relations. +> +> Migrations are generated by diesel cli, the basic schema can be found [schema.rs](src/schema.rs). +> For more in depth understanding of the database tables, go to [migrations](migrations) folder, in the contained `SQL` the indexes, partitions & constraints are declared. +> +> - Tables `objects_history` & `transactions` have partitions, each partition is created based on `checkpoint_sequence_number` (related form the `checkpoints` table) it goes from `0` to `MAXVALUE` +> - `__diesel_schema_migrations` table is managed by `diesel` cli when applying migrations + +## Indexes + +### Table `checkpoints` + +| Index name | Keys | +| ------------------ | ---------------------- | +| checkpoints_epoch | epoch, sequence_number | +| checkpoints_digest | checkpoint_digest | + +### Table `events` + +| Index name | Keys | +| --------------------------------- | ----------------------------------------------------------------------- | +| events_package | package, tx_sequence_number, event_sequence_number | +| events_package_module | package, module, tx_sequence_number, event_sequence_number | +| events_event_type | event_type, text_pattern_ops, tx_sequence_number, event_sequence_number | +| events_checkpoint_sequence_number | checkpoint_sequence_number | + +### Table `objects` + +| Index name | Keys | Condition | +| ---------------------------------- | -------------------------- | --------------------------------------------------------- | +| objects_owner | owner_type, owner_id | WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL | +| objects_coin | owner_id, coin_type | WHERE coin_type IS NOT NULL AND owner_type = 1 | +| objects_checkpoint_sequence_number | checkpoint_sequence_number | | +| objects_type | object_type | | + +### Table `objects_snapshot` + +| Index name | Keys | Condition | +| ------------------------------------------- | ------------------------------- | --------------------------------------------------------- | +| objects_snapshot_checkpoint_sequence_number | checkpoint_sequence_number | | +| objects_snapshot_coin | owner_id, coin_type, object_id | WHERE coin_type IS NOT NULL AND owner_type = 1 | +| objects_snapshot_type | object_type, object_id | | +| objects_snapshot_owner | owner_type, owner_id, object_id | WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL | + +### Table `objects_history` + +| Index name | Keys | Condition | +| --------------------- | ------------------------------------------------ | --------------------------------------------------------- | +| objects_history_owner | checkpoint_sequence_number, owner_type, owner_id | WHERE owner_type BETWEEN 1 AND 2 AND owner_id IS NOT NULL | +| objects_history_coin | checkpoint_sequence_number, owner_id, coin_type | WHERE coin_type IS NOT NULL AND owner_type = 1 | +| objects_history_type | checkpoint_sequence_number, object_type | | + +### Table `transactions` + +| Index name | Keys | Condition | +| --------------------------------------- | -------------------------- | -------------------------- | +| transactions_transaction_digest | transaction_digest | | +| transactions_checkpoint_sequence_number | checkpoint_sequence_number | | +| transactions_transaction_kind | transaction_kind | WHERE transaction_kind = 1 | + +### Table `tx_calls` + +| Index name | Keys | +| --------------------------- | ----------------------------------------- | +| tx_calls_module | package, module, tx_sequence_number | +| tx_calls_func | package, module, func, tx_sequence_number | +| tx_calls_tx_sequence_number | tx_sequence_number | + +### Table `tx_senders` + +| Index name | Keys | Condition | +| ----------------------------------- | ------------------ | --------- | +| tx_senders_tx_sequence_number_index | tx_sequence_number | ASC | + +### Tables `tx_recipients` + +| Index name | Keys | Condition | +| -------------------------------------- | ------------------ | --------- | +| tx_recipients_tx_sequence_number_index | tx_sequence_number | ASC | + +## Partitions + +### Tables `transactions`, `objects_history` + +| Keys | Condition | +| :------------------------: | :-------------------------------: | +| checkpoint_sequence_number | FOR VALUES FROM (0) TO (MAXVALUE) |