Feature: full support node RPC

.env.example

@@ -1,10 +1,14 @@
 INDEXER_BRIDGED_TOKENS_FILE=mainnet.json   # full list of files you can find in repo ./build/bridged_tokens/
 INDEXER_CLASS_INTERFACES_DIR=./interfaces/ # REQUIRED
+INDEXER_DATASOURCE=sequencer               # REQUIRED: one of 'node' or 'sequencer'
 HASURA_HOST=hasura
 HASURA_POSTGRES_HOST=db
 LOG_LEVEL=info
-CACHE_ENABLED=false
 POSTGRES_PORT=5432
 POSTGRES_HOST=db
 POSTGRES_DB=starknet
-POSTGRES_PASSWORD=<TYPE_SOMETHING_STRONG> # REQUIRED
+POSTGRES_PASSWORD=<TYPE_SOMETHING_STRONG>     # REQUIRED
+STARKNET_NODE_URL=<URL_HERE>                  # REQUIRED if INDEXER_DATASOURCE=node
+STARKNET_SEQUENCER_FEEDER_GATEWAY=<URL_HERE>  # REQUIRED if INDEXER_DATASOURCE=sequencer
+NODE_APIKEY=<API_KEY_FROM_NODE_PROVIDER>      # REQUIRED if your node provider has api key. It's api key.
+NODE_HEADER_APIKEY=<HEADER_NAME>              # REQUIRED if your node provider has api key. It's header name.

build/dipdup.yml

@@ -1,32 +1,28 @@
-version: 0.0.1
+version: 0.0.2
 
 log_level: ${LOG_LEVEL:-info}
 
 indexer:
   name: ${INDEXER_NAME:-dipdup_starknet_indexer}
-
-  sequencer:
-    feeder_gateway: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway}
-    gateway: ${STARKNET_SEQUENCER_GATEWAY:-https://alpha-mainnet.starknet.io/gateway}
-    requests_per_second: ${STARKNET_SEQUENCER_RPS:-3}
-
-  node:
-    url: ${STARKNET_NODE_URL}
-    requests_per_second: ${STARKNET_NODE_RPS:-5}
-
+  datasource: ${INDEXER_DATASOURCE:-sequencer}
   threads_count: ${INDEXER_THREADS_COUNT:-10}
   start_level: ${INDEXER_START_LEVEL:-0}
   timeout: ${INDEXER_REQUEST_TIMEOUT:-10}
-  requests_per_second: ${STARKNET_RPS:-2}
   class_interfaces_dir: ${INDEXER_CLASS_INTERFACES_DIR:-./interfaces/}
   bridged_tokens_file: ${INDEXER_BRIDGED_TOKENS_FILE:-mainnet.json}
-  cache_dir: ${INDEXER_CACHE_DIR}
-  cache: ${CACHE_ENABLED:-false}
 
 grpc:
   bind: ${GRPC_BIND:-127.0.0.1:7779}
   log: true
 
+datasources:
+  node:
+    url: ${STARKNET_NODE_URL}
+    rps: ${STARKNET_NODE_RPS:-5}
+  sequencer:
+    url: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway}
+    rps: ${STARKNET_SEQUENCER_RPS:-3}
+
 database:
   kind: postgres
   host: ${POSTGRES_HOST:-db}

cmd/indexer/main.go

@@ -103,7 +103,12 @@ func main() {
 		}
 	}
 
-	indexerModule := indexer.New(cfg.Indexer, postgres)
+	indexerModule, err := indexer.New(cfg.Indexer, postgres, cfg.DataSources)
+	if err != nil {
+		log.Panic().Err(err).Msg("creating indexer module")
+		cancel()
+		return
+	}
 
 	grpcModule, err := grpc.NewServer(
 		cfg.GRPC, postgres,

internal/storage/deploy.go

@@ -39,7 +39,7 @@ type Deploy struct {
 	Hash                []byte         `comment:"Transaction hash"`
 	ContractAddressSalt []byte         `comment:"A random salt that determines the account address"`
 	ConstructorCalldata []string       `bun:",array" comment:"Raw constructor calldata"`
-	ParsedCalldata      map[string]any `comment:"Calldata parsed according to contract ABI"`
+	ParsedCalldata      map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
 	Error               *string        `bun:"error" comment:"Reverted error"`
 
 	Class     Class      `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"`
@@ -89,12 +89,15 @@ func (d Deploy) Flat() []any {
 		d.Hash,
 		d.ContractAddressSalt,
 		pq.StringArray(d.ConstructorCalldata),
+		nil,
+		d.Error,
 	}
-	parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil, d.Error)
-	} else {
-		data = append(data, string(parsed), d.Error)
+
+	if d.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[10] = string(parsed)
+		}
 	}
 	return data
 }

internal/storage/deploy_account.go

@@ -42,7 +42,7 @@ type DeployAccount struct {
 	MaxFee              decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"`
 	Nonce               decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"`
 	ConstructorCalldata []string        `bun:",array" comment:"Raw constructor calldata"`
-	ParsedCalldata      map[string]any  `comment:"Calldata parsed according to contract ABI"`
+	ParsedCalldata      map[string]any  `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
 	Error               *string         `bun:"error" comment:"Reverted error"`
 
 	Class     Class      `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"`
@@ -94,12 +94,14 @@ func (d DeployAccount) Flat() []any {
 		d.MaxFee,
 		d.Nonce,
 		pq.StringArray(d.ConstructorCalldata),
+		nil,
+		d.Error,
 	}
-	parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil, d.Error)
-	} else {
-		data = append(data, string(parsed), d.Error)
+	if d.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[12] = string(parsed)
+		}
 	}
 	return data
 }

internal/storage/event.go

@@ -48,7 +48,7 @@ type Event struct {
 	Keys       []string       `bun:",array" comment:"Raw event keys"`
 	Data       []string       `bun:",array" comment:"Raw event data"`
 	Name       string         `comment:"Event name"`
-	ParsedData map[string]any `comment:"Event data parsed according to contract ABI"`
+	ParsedData map[string]any `bun:",nullzero" comment:"Event data parsed according to contract ABI"`
 
 	From     Address `bun:"rel:belongs-to" hasura:"table:address,field:from_id,remote_field:id,type:oto,name:from"`
 	Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"`
@@ -98,13 +98,14 @@ func (e Event) Flat() []any {
 		pq.StringArray(e.Keys),
 		pq.StringArray(e.Data),
 		e.Name,
+		nil,
 	}
 
-	parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil)
-	} else {
-		data = append(data, string(parsed))
+	if e.ParsedData != nil {
+		parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[16] = string(parsed)
+		}
 	}
 	return data
 }

internal/storage/fee.go

@@ -57,7 +57,7 @@ type Fee struct {
 	Entrypoint     string         `comment:"Entrypoint name"`
 	Calldata       []string       `bun:",array" comment:"Raw calldata"`
 	Result         []string       `bun:",array" comment:"Raw result"`
-	ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"`
+	ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
 
 	Class     Class      `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"`
 	Caller    Address    `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"`
@@ -115,12 +115,14 @@ func (f Fee) Flat() []any {
 		f.Entrypoint,
 		pq.StringArray(f.Calldata),
 		pq.StringArray(f.Result),
+		nil,
 	}
-	parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil)
-	} else {
-		data = append(data, string(parsed))
+
+	if f.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[18] = string(parsed)
+		}
 	}
 	return data
 }

internal/storage/internal.go

@@ -58,8 +58,8 @@ type Internal struct {
 	Entrypoint     string         `comment:"Entrypoint name"`
 	Result         []string       `bun:",array" comment:"Raw result"`
 	Calldata       []string       `bun:",array" comment:"Raw calldata"`
-	ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"`
-	ParsedResult   map[string]any `comment:"Result parsed according to contract ABI"`
+	ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
+	ParsedResult   map[string]any `bun:",nullzero" comment:"Result parsed according to contract ABI"`
 
 	Class     Class      `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"`
 	Caller    Address    `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"`
@@ -120,20 +120,22 @@ func (i Internal) Flat() []any {
 		i.Entrypoint,
 		pq.StringArray(i.Calldata),
 		pq.StringArray(i.Result),
+		nil,
+		nil,
 	}
 
-	parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil)
-	} else {
-		data = append(data, string(parsed))
+	if i.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[20] = string(parsed)
+		}
 	}
 
-	result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil)
-	} else {
-		data = append(data, string(result))
+	if i.ParsedResult != nil {
+		result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[21] = string(result)
+		}
 	}
 	return data
 }

internal/storage/invoke.go

@@ -46,7 +46,7 @@ type Invoke struct {
 	MaxFee             decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"`
 	Nonce              decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"`
 	CallData           []string        `bun:",array" comment:"Raw calldata"`
-	ParsedCalldata     map[string]any  `comment:"Calldata parsed according to contract ABI"`
+	ParsedCalldata     map[string]any  `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
 	Error              *string         `bun:"error" comment:"Reverted error"`
 
 	Contract  Address    `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"`
@@ -98,12 +98,14 @@ func (i Invoke) Flat() []any {
 		i.MaxFee,
 		i.Nonce,
 		pq.StringArray(i.CallData),
+		nil,
+		i.Error,
 	}
-	parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil, i.Error)
-	} else {
-		data = append(data, string(parsed), i.Error)
+	if i.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[13] = string(parsed)
+		}
 	}
 
 	return data

internal/storage/l1_handler.go

@@ -44,7 +44,7 @@ type L1Handler struct {
 	MaxFee         decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"`
 	Nonce          decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"`
 	CallData       []string        `bun:",array" comment:"Raw calldata"`
-	ParsedCalldata map[string]any  `comment:"Calldata parsed according to contract ABI"`
+	ParsedCalldata map[string]any  `bun:",nullzero" comment:"Calldata parsed according to contract ABI"`
 	Error          *string         `bun:"error" comment:"Reverted error"`
 
 	Contract  Address    `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"`
@@ -94,13 +94,15 @@ func (l1 L1Handler) Flat() []any {
 		l1.MaxFee,
 		l1.Nonce,
 		pq.StringArray(l1.CallData),
+		nil,
+		l1.Error,
 	}
 
-	parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
-	if err != nil {
-		data = append(data, nil, l1.Error)
-	} else {
-		data = append(data, string(parsed), l1.Error)
+	if l1.ParsedCalldata != nil {
+		parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8())
+		if err == nil {
+			data[12] = string(parsed)
+		}
 	}
 	return data
 }

pkg/indexer/config/config.go

@@ -2,27 +2,11 @@ package config
 
 // Config - configuration structure for indexer
 type Config struct {
-	Name               string    `yaml:"name" validate:"omitempty"`
-	StartLevel         uint64    `yaml:"start_level" validate:"omitempty"`
-	ThreadsCount       int       `yaml:"threads_count" validate:"omitempty,min=1"`
-	Timeout            uint64    `yaml:"timeout" validate:"omitempty"`
-	Node               *Node     `yaml:"node" validate:"omitempty"`
-	Sequencer          Sequencer `yaml:"sequencer" validate:"required"`
-	ClassInterfacesDir string    `yaml:"class_interfaces_dir" validate:"required,dir"`
-	BridgedTokensFile  string    `yaml:"bridged_tokens_file" validate:"required,file"`
-	CacheDir           string    `yaml:"cache_dir" validate:"omitempty,dir"`
-	Cache              bool      `yaml:"cache" validate:"omitempty"`
-}
-
-// Node -
-type Node struct {
-	Url string `yaml:"url" validate:"omitempty,url"`
-	Rps int    `yaml:"requests_per_second" validate:"omitempty,min=1"`
-}
-
-// Sequencer -
-type Sequencer struct {
-	FeederGateway string `yaml:"feeder_gateway" validate:"required,url"`
-	Gateway       string `yaml:"gateway" validate:"required,url"`
-	Rps           int    `yaml:"requests_per_second" validate:"omitempty,min=1"`
+	Name               string `yaml:"name" validate:"omitempty"`
+	StartLevel         uint64 `yaml:"start_level" validate:"omitempty"`
+	ThreadsCount       int    `yaml:"threads_count" validate:"omitempty,min=1"`
+	Timeout            uint64 `yaml:"timeout" validate:"omitempty"`
+	ClassInterfacesDir string `yaml:"class_interfaces_dir" validate:"required,dir"`
+	BridgedTokensFile  string `yaml:"bridged_tokens_file" validate:"required,file"`
+	Datasource         string `yaml:"datasource" validate:"required,oneof=sequencer node"`
 }