From 203009480d935d5e52495063eec97a4a7f7e3ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20Men=C3=A9ndez?= Date: Thu, 20 Jun 2024 11:56:01 +0200 Subject: [PATCH] calculate multiple times the creation block of a contract and select the lower one, use provider manager on API layer --- api/api.go | 12 +++--- api/helpers.go | 6 +-- api/holders.go | 6 +-- api/tokens.go | 56 +++++++++++++++++++++---- cmd/census3/main.go | 2 +- scanner/providers/web3/web3_provider.go | 15 ++++--- 6 files changed, 72 insertions(+), 25 deletions(-) diff --git a/api/api.go b/api/api.go index 0e98cfce..2bf2ecd3 100644 --- a/api/api.go +++ b/api/api.go @@ -19,6 +19,7 @@ import ( "github.com/vocdoni/census3/helpers/queue" "github.com/vocdoni/census3/helpers/web3" "github.com/vocdoni/census3/scanner/providers" + "github.com/vocdoni/census3/scanner/providers/manager" web3provider "github.com/vocdoni/census3/scanner/providers/web3" "go.vocdoni.io/dvote/api/censusdb" storagelayer "go.vocdoni.io/dvote/data" @@ -40,7 +41,7 @@ type Census3APIConf struct { DataDir string GroupKey string Web3Providers *web3.Web3Pool - HolderProviders map[uint64]providers.HolderProvider + HolderProviders *manager.ProviderManager AdminToken string } @@ -53,7 +54,7 @@ type census3API struct { w3p *web3.Web3Pool storage storagelayer.Storage downloader *downloader.Downloader - holderProviders map[uint64]providers.HolderProvider + holderProviders *manager.ProviderManager cache *lru.Cache[CacheKey, any] router *httprouter.HTTProuter } @@ -238,13 +239,14 @@ func (capi *census3API) CreateInitialTokens(tokensPath string) error { // get the correct holder provider for the token type tokenType := providers.TokenTypeID(token.Type) - provider, exists := capi.holderProviders[tokenType] - if !exists { + provider, err := capi.holderProviders.GetProvider(ctx, tokenType) + if err != nil { log.Warnw("token type provided in initial list not supported, check provider is set. SKIPPING...", "tokenID", token.ID, "chainID", token.ChainID, "externalID", token.ExternalID, - "type", token.Type) + "type", token.Type, + "error", err) continue } if !provider.IsExternal() { diff --git a/api/helpers.go b/api/helpers.go index ad40477a..6ff808d0 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -272,9 +272,9 @@ func (capi *census3API) CalculateStrategyHolders(ctx context.Context, Decimals: token.Decimals, ExternalID: t.ExternalID, } - provider, exists := capi.holderProviders[token.TypeID] - if !exists { - return nil, nil, 0, fmt.Errorf("provider not found for token type id %d", token.TypeID) + provider, err := capi.holderProviders.GetProvider(ctx, token.TypeID) + if err != nil { + return nil, nil, 0, fmt.Errorf("provider not found for token type id %d: %w", token.TypeID, err) } if !provider.IsExternal() { if err := provider.SetRef(web3.Web3ProviderRef{ diff --git a/api/holders.go b/api/holders.go index e83a320b..be6d06a8 100644 --- a/api/holders.go +++ b/api/holders.go @@ -106,9 +106,9 @@ func (capi *census3API) listHoldersAtLastBlock(address common.Address, return nil, 0, ErrCantGetTokenHolders.WithErr(err) } // if the token is external, return an error - provider, exists := capi.holderProviders[tokenData.TypeID] - if !exists { - return nil, 0, ErrCantCreateCensus.With("token type not supported") + provider, err := capi.holderProviders.GetProvider(internalCtx, tokenData.TypeID) + if err != nil { + return nil, 0, ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err)) } if provider.IsExternal() { return nil, 0, ErrCantCreateCensus.With("not implemented for external providers") diff --git a/api/tokens.go b/api/tokens.go index 1ea73d2d..5021f56d 100644 --- a/api/tokens.go +++ b/api/tokens.go @@ -9,8 +9,10 @@ import ( "errors" "fmt" "math/big" + "net/http" "strconv" "strings" + "time" "github.com/ethereum/go-ethereum/common" "github.com/vocdoni/census3/db/annotations" @@ -37,6 +39,10 @@ func (capi *census3API) initTokenHandlers() error { api.MethodAccessTypePublic, capi.getToken); err != nil { return err } + if err := capi.endpoint.RegisterMethod("/tokens/startblock", "POST", + api.MethodAccessTypePublic, capi.tokenStartBlock); err != nil { + return err + } if err := capi.endpoint.RegisterMethod("/tokens/{tokenID}", "DELETE", api.MethodAccessTypeAdmin, capi.launchDeleteToken); err != nil { return err @@ -242,9 +248,9 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex defer cancel() // get the correct holder provider for the token type tokenType := providers.TokenTypeID(req.Type) - provider, exists := capi.holderProviders[tokenType] - if !exists { - return ErrCantCreateCensus.With("token type not supported") + provider, err := capi.holderProviders.GetProvider(internalCtx, tokenType) + if err != nil { + return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err)) } if !provider.IsExternal() { if err := provider.SetRef(web3.Web3ProviderRef{ @@ -541,9 +547,9 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext) atBlock := uint64(tokenData.LastBlock) tokenProgress := 100 if !tokenData.Synced { - provider, exists := capi.holderProviders[tokenData.TypeID] - if !exists { - return ErrCantCreateCensus.With("token type not supported") + provider, err := capi.holderProviders.GetProvider(internalCtx, tokenData.TypeID) + if err != nil { + return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err)) } if !provider.IsExternal() { if err := provider.SetRef(web3.Web3ProviderRef{ @@ -601,6 +607,42 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext) return ctx.Send(res, api.HTTPstatusOK) } +func (capi *census3API) tokenStartBlock(msg *api.APIdata, ctx *httprouter.HTTPContext) error { + req := Token{} + if err := json.Unmarshal(msg.Data, &req); err != nil { + log.Errorf("error unmarshalling token information: %s", err) + return ErrMalformedToken.WithErr(err) + } + tokenType := providers.TokenTypeID(req.Type) + // get token information from the database + internalCtx, cancel := context.WithTimeout(ctx.Request.Context(), getTokenTimeout) + defer cancel() + provider, err := capi.holderProviders.GetProvider(internalCtx, tokenType) + if err != nil { + return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err)) + } + if provider.IsExternal() { + return ctx.Send([]byte("type not supported"), http.StatusBadRequest) + } + if err := provider.SetRef(web3.Web3ProviderRef{ + HexAddress: common.HexToAddress(req.ID).Hex(), + ChainID: req.ChainID, + }); err != nil { + return ErrInitializingWeb3.WithErr(err) + } + go func() { + bgCtx, cancel := context.WithTimeout(context.Background(), time.Minute*10) + defer cancel() + startBlock, err := provider.CreationBlock(bgCtx, nil) + if err != nil { + log.Error(err) + return + } + log.Infow("start block calculated", "startBlock", startBlock, "tokenID", req.ID, "chainID", req.ChainID) + }() + return ctx.Send([]byte("ok"), api.HTTPstatusOK) +} + func (capi *census3API) getTokenHolder(msg *api.APIdata, ctx *httprouter.HTTPContext) error { // get contract address from the tokenID query param and decode check if // it is provided, if not return an error @@ -771,7 +813,7 @@ func (capi *census3API) enqueueTokenHoldersCSV(msg *api.APIdata, ctx *httprouter // supported types of token contracts. func (capi *census3API) getTokenTypes(msg *api.APIdata, ctx *httprouter.HTTPContext) error { supportedTypes := []string{} - for _, provider := range capi.holderProviders { + for _, provider := range capi.holderProviders.Providers(ctx.Request.Context()) { supportedTypes = append(supportedTypes, provider.TypeName()) } res, err := json.Marshal(TokenTypes{supportedTypes}) diff --git a/cmd/census3/main.go b/cmd/census3/main.go index fa89e72c..2c3ed74e 100644 --- a/cmd/census3/main.go +++ b/cmd/census3/main.go @@ -214,7 +214,7 @@ func main() { DataDir: config.dataDir, Web3Providers: w3p, GroupKey: config.connectKey, - HolderProviders: pm.Providers(ctx), + HolderProviders: pm, AdminToken: config.adminToken, }) if err != nil { diff --git a/scanner/providers/web3/web3_provider.go b/scanner/providers/web3/web3_provider.go index 67111255..c25193c6 100644 --- a/scanner/providers/web3/web3_provider.go +++ b/scanner/providers/web3/web3_provider.go @@ -63,15 +63,18 @@ func creationBlock(client *web3.Client, ctx context.Context, addr common.Address if err != nil { return 0, err } - var creationBlock uint64 + var minCreationBlock uint64 for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ { - creationBlock, err = creationBlockInRange(client, ctx, addr, 0, lastBlock) - if err == nil { - break + creationBlock, err := creationBlockInRange(client, ctx, addr, 0, lastBlock) + if err != nil { + time.Sleep(RetryWeb3Cooldown) + continue + } + if minCreationBlock == 0 || creationBlock < minCreationBlock { + minCreationBlock = creationBlock } - time.Sleep(RetryWeb3Cooldown) } - return creationBlock, err + return minCreationBlock, err } // creationBlockInRange function finds the block number of a contract between