From fe40c0f002d8cad178426021924c7298f2f2adf8 Mon Sep 17 00:00:00 2001 From: aureliusbtc <82057759+aureliusbtc@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:33:20 +0000 Subject: [PATCH 1/3] scaffold db --- services/rfq/api/db/api_db.go | 15 ++++++ services/rfq/api/db/sql/base/base.go | 31 +++++++++++ services/rfq/api/db/sql/base/doc.go | 2 + services/rfq/api/db/sql/base/model.go | 10 ++++ services/rfq/api/db/sql/doc.go | 2 + services/rfq/api/db/sql/mysql/doc.go | 2 + services/rfq/api/db/sql/mysql/store.go | 66 ++++++++++++++++++++++++ services/rfq/api/db/sql/sqlite/doc.go | 2 + services/rfq/api/db/sql/sqlite/sqlite.go | 63 ++++++++++++++++++++++ services/rfq/api/db/sql/store.go | 37 +++++++++++++ 10 files changed, 230 insertions(+) create mode 100644 services/rfq/api/db/api_db.go create mode 100644 services/rfq/api/db/sql/base/base.go create mode 100644 services/rfq/api/db/sql/base/doc.go create mode 100644 services/rfq/api/db/sql/base/model.go create mode 100644 services/rfq/api/db/sql/doc.go create mode 100644 services/rfq/api/db/sql/mysql/doc.go create mode 100644 services/rfq/api/db/sql/mysql/store.go create mode 100644 services/rfq/api/db/sql/sqlite/doc.go create mode 100644 services/rfq/api/db/sql/sqlite/sqlite.go create mode 100644 services/rfq/api/db/sql/store.go diff --git a/services/rfq/api/db/api_db.go b/services/rfq/api/db/api_db.go new file mode 100644 index 0000000000..714be11927 --- /dev/null +++ b/services/rfq/api/db/api_db.go @@ -0,0 +1,15 @@ +package db + +// ApiDBReader is the interface for reading from the database. +type ApiDBReader interface { +} + +// ApiDBWriter is the interface for writing to the database. +type ApiDBWriter interface { +} + +// ApiDB is the interface for the database service. +type ApiDB interface { + ApiDBReader + ApiDBWriter +} diff --git a/services/rfq/api/db/sql/base/base.go b/services/rfq/api/db/sql/base/base.go new file mode 100644 index 0000000000..9ba6390e67 --- /dev/null +++ b/services/rfq/api/db/sql/base/base.go @@ -0,0 +1,31 @@ +package base + +import ( + "github.com/synapsecns/sanguine/core/metrics" + "github.com/synapsecns/sanguine/services/rfq/api/db" + "gorm.io/gorm" +) + +// Store is a store that implements an underlying gorm db. +type Store struct { + db *gorm.DB + metrics metrics.Handler +} + +// NewStore creates a new store. +func NewStore(db *gorm.DB, metrics metrics.Handler) *Store { + return &Store{db: db, metrics: metrics} +} + +// DB gets the database object for mutation outside of the lib. +func (s Store) DB() *gorm.DB { + return s.db +} + +// GetAllModels gets all models to migrate. +// see: https://medium.com/@SaifAbid/slice-interfaces-8c78f8b6345d for an explanation of why we can't do this at initialization time +func GetAllModels() (allModels []interface{}) { + return allModels +} + +var _ db.ApiDB = &Store{} diff --git a/services/rfq/api/db/sql/base/doc.go b/services/rfq/api/db/sql/base/doc.go new file mode 100644 index 0000000000..9b758883a1 --- /dev/null +++ b/services/rfq/api/db/sql/base/doc.go @@ -0,0 +1,2 @@ +// Package base contains the base sql implementation +package base diff --git a/services/rfq/api/db/sql/base/model.go b/services/rfq/api/db/sql/base/model.go new file mode 100644 index 0000000000..47ff31f93d --- /dev/null +++ b/services/rfq/api/db/sql/base/model.go @@ -0,0 +1,10 @@ +package base + +// define common field names. See package docs for an explanation of why we have to do this. +// note: some models share names. In cases where they do, we run the check against all names. +// This is cheap because it's only done at startup. +func init() { + +} + +var () diff --git a/services/rfq/api/db/sql/doc.go b/services/rfq/api/db/sql/doc.go new file mode 100644 index 0000000000..9c3daf2957 --- /dev/null +++ b/services/rfq/api/db/sql/doc.go @@ -0,0 +1,2 @@ +// Package sql provides a common interface for starting sql-lite databases +package sql diff --git a/services/rfq/api/db/sql/mysql/doc.go b/services/rfq/api/db/sql/mysql/doc.go new file mode 100644 index 0000000000..a6b8106850 --- /dev/null +++ b/services/rfq/api/db/sql/mysql/doc.go @@ -0,0 +1,2 @@ +// Package mysql contains a mysql db +package mysql diff --git a/services/rfq/api/db/sql/mysql/store.go b/services/rfq/api/db/sql/mysql/store.go new file mode 100644 index 0000000000..b650ce41e7 --- /dev/null +++ b/services/rfq/api/db/sql/mysql/store.go @@ -0,0 +1,66 @@ +package mysql + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/go-log" + common_base "github.com/synapsecns/sanguine/core/dbcommon" + "github.com/synapsecns/sanguine/core/metrics" + "github.com/synapsecns/sanguine/services/rfq/api/db" + "github.com/synapsecns/sanguine/services/rfq/api/db/sql/base" + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +// Logger is the mysql logger. +var logger = log.Logger("api-mysql") + +// NewMysqlStore creates a new mysql store for a given data store. +func NewMysqlStore(ctx context.Context, dbURL string, handler metrics.Handler) (*Store, error) { + logger.Debug("create mysql store") + + gdb, err := gorm.Open(mysql.Open(dbURL), &gorm.Config{ + Logger: common_base.GetGormLogger(logger), + FullSaveAssociations: true, + NamingStrategy: NamingStrategy, + NowFunc: time.Now, + }) + + if err != nil { + return nil, fmt.Errorf("could not create mysql connection: %w", err) + } + + sqlDB, err := gdb.DB() + if err != nil { + return nil, fmt.Errorf("could not get sql db: %w", err) + } + + // fixes a timeout issue https://stackoverflow.com/a/42146536 + sqlDB.SetMaxIdleConns(MaxIdleConns) + sqlDB.SetConnMaxLifetime(time.Hour) + + handler.AddGormCallbacks(gdb) + + err = gdb.WithContext(ctx).AutoMigrate(base.GetAllModels()...) + if err != nil { + return nil, fmt.Errorf("could not migrate on mysql: %w", err) + } + + return &Store{base.NewStore(gdb, handler)}, nil +} + +// Store is the mysql store. It extends the bsae store for mysql queries. +type Store struct { + *base.Store +} + +// MaxIdleConns is exported here for testing. Tests execute too slowly with a reconnect each time. +var MaxIdleConns = 10 + +// NamingStrategy is for table prefixes. +var NamingStrategy = schema.NamingStrategy{} + +var _ db.ApiDB = &Store{} diff --git a/services/rfq/api/db/sql/sqlite/doc.go b/services/rfq/api/db/sql/sqlite/doc.go new file mode 100644 index 0000000000..d30fb340b9 --- /dev/null +++ b/services/rfq/api/db/sql/sqlite/doc.go @@ -0,0 +1,2 @@ +// Package sqlite implements the sqlite package +package sqlite diff --git a/services/rfq/api/db/sql/sqlite/sqlite.go b/services/rfq/api/db/sql/sqlite/sqlite.go new file mode 100644 index 0000000000..dffccaaf5a --- /dev/null +++ b/services/rfq/api/db/sql/sqlite/sqlite.go @@ -0,0 +1,63 @@ +package sqlite + +import ( + "context" + "fmt" + "os" + + "github.com/synapsecns/sanguine/services/rfq/api/db/sql/base" + + "github.com/ipfs/go-log" + common_base "github.com/synapsecns/sanguine/core/dbcommon" + "github.com/synapsecns/sanguine/core/metrics" + "github.com/synapsecns/sanguine/services/rfq/api/db" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// Store is the sqlite store. It extends the base store for sqlite specific queries. +type Store struct { + *base.Store +} + +var logger = log.Logger("api-sqlite") + +// NewSqliteStore creates a new sqlite data store. +func NewSqliteStore(parentCtx context.Context, dbPath string, handler metrics.Handler, skipMigrations bool) (_ *Store, err error) { + logger.Debugf("creating sqlite store at %s", dbPath) + + ctx, span := handler.Tracer().Start(parentCtx, "start-sqlite") + defer func() { + metrics.EndSpanWithErr(span, err) + }() + + // create the directory to the store if it doesn't exist + err = os.MkdirAll(dbPath, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("could not create sqlite store") + } + + logger.Warnf("api database is at %s/api.db", dbPath) + + gdb, err := gorm.Open(sqlite.Open(fmt.Sprintf("%s/%s", dbPath, "api.db")), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + Logger: common_base.GetGormLogger(logger), + FullSaveAssociations: true, + SkipDefaultTransaction: true, + }) + if err != nil { + return nil, fmt.Errorf("could not connect to db %s: %w", dbPath, err) + } + + handler.AddGormCallbacks(gdb) + + if !skipMigrations { + err = gdb.WithContext(ctx).AutoMigrate(base.GetAllModels()...) + if err != nil { + return nil, fmt.Errorf("could not migrate models: %w", err) + } + } + return &Store{base.NewStore(gdb, handler)}, nil +} + +var _ db.ApiDB = &Store{} diff --git a/services/rfq/api/db/sql/store.go b/services/rfq/api/db/sql/store.go new file mode 100644 index 0000000000..352ff5db84 --- /dev/null +++ b/services/rfq/api/db/sql/store.go @@ -0,0 +1,37 @@ +package sql + +import ( + "context" + "errors" + "fmt" + + "github.com/synapsecns/sanguine/core/dbcommon" + "github.com/synapsecns/sanguine/core/metrics" + "github.com/synapsecns/sanguine/services/rfq/api/db" + "github.com/synapsecns/sanguine/services/rfq/api/db/sql/mysql" + "github.com/synapsecns/sanguine/services/rfq/api/db/sql/sqlite" +) + +// Connect connects to the database. +func Connect(ctx context.Context, dbType dbcommon.DBType, path string, metrics metrics.Handler) (db.ApiDB, error) { + switch dbType { + case dbcommon.Mysql: + store, err := mysql.NewMysqlStore(ctx, path, metrics) + if err != nil { + return nil, fmt.Errorf("could not create mysql store: %w", err) + } + + return store, nil + case dbcommon.Sqlite: + store, err := sqlite.NewSqliteStore(ctx, path, metrics, false) + if err != nil { + return nil, fmt.Errorf("could not create sqlite store: %w", err) + } + + return store, nil + case dbcommon.Clickhouse: + return nil, errors.New("driver not supported") + default: + return nil, fmt.Errorf("unsupported driver: %s", dbType) + } +} From d2c2d743decd0a83c74eeb2e6fd275cf3d6a68a0 Mon Sep 17 00:00:00 2001 From: aureliusbtc <82057759+aureliusbtc@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:48:29 +0000 Subject: [PATCH 2/3] create quotes table and db suite test --- services/rfq/api/db/api_db_test.go | 7 ++ services/rfq/api/db/sql/base/base.go | 1 + services/rfq/api/db/sql/base/model.go | 21 +++++ services/rfq/api/db/suite_test.go | 109 ++++++++++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 services/rfq/api/db/api_db_test.go create mode 100644 services/rfq/api/db/suite_test.go diff --git a/services/rfq/api/db/api_db_test.go b/services/rfq/api/db/api_db_test.go new file mode 100644 index 0000000000..656dde4807 --- /dev/null +++ b/services/rfq/api/db/api_db_test.go @@ -0,0 +1,7 @@ +package db_test + +import "fmt" + +func (d *DBSuite) TestModelCreation() { + fmt.Println("suite started successfully") +} diff --git a/services/rfq/api/db/sql/base/base.go b/services/rfq/api/db/sql/base/base.go index 9ba6390e67..b232c1db6a 100644 --- a/services/rfq/api/db/sql/base/base.go +++ b/services/rfq/api/db/sql/base/base.go @@ -25,6 +25,7 @@ func (s Store) DB() *gorm.DB { // GetAllModels gets all models to migrate. // see: https://medium.com/@SaifAbid/slice-interfaces-8c78f8b6345d for an explanation of why we can't do this at initialization time func GetAllModels() (allModels []interface{}) { + allModels = append(allModels, &Quote{}) return allModels } diff --git a/services/rfq/api/db/sql/base/model.go b/services/rfq/api/db/sql/base/model.go index 47ff31f93d..8d8002692f 100644 --- a/services/rfq/api/db/sql/base/model.go +++ b/services/rfq/api/db/sql/base/model.go @@ -1,5 +1,11 @@ package base +import ( + "time" + + "github.com/shopspring/decimal" +) + // define common field names. See package docs for an explanation of why we have to do this. // note: some models share names. In cases where they do, we run the check against all names. // This is cheap because it's only done at startup. @@ -8,3 +14,18 @@ func init() { } var () + +type Quote struct { + // ID is the unique identifier saved of each quote provided + ID uint64 `gorm:"column:id;primaryKey;"` + // DestChainID is the chain which the relayer is willing to provide liquidity for + DestChainID uint64 `gorm:"column:dest_chain_id;index"` + // DestToken is the token address for which the relayer is providing liquidity + DestTokenAddr string `gorm:"column:token;index"` + // DestAmount is the max amount of liquidity which exists for a given destination token, provided in the destination token decimals + DestAmount decimal.Decimal `gorm:"column:dest_amount"` + // Price is the price per origin token provided for which a relayer is indicating willingness to relay + Price decimal.Decimal `gorm:"column:price"` + // UpdatedAt is the time that the quote was last upserted + UpdatedAt time.Time `gorm:"column:updated_at"` +} diff --git a/services/rfq/api/db/suite_test.go b/services/rfq/api/db/suite_test.go new file mode 100644 index 0000000000..48715001ab --- /dev/null +++ b/services/rfq/api/db/suite_test.go @@ -0,0 +1,109 @@ +package db_test + +import ( + dbSQL "database/sql" + "fmt" + "os" + "sync" + "testing" + + "github.com/Flaque/filet" + . "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/synapsecns/sanguine/core" + "github.com/synapsecns/sanguine/core/dbcommon" + "github.com/synapsecns/sanguine/core/metrics" + "github.com/synapsecns/sanguine/core/metrics/localmetrics" + "github.com/synapsecns/sanguine/core/testsuite" + "github.com/synapsecns/sanguine/services/rfq/api/db" + "github.com/synapsecns/sanguine/services/rfq/api/db/sql" + "github.com/synapsecns/sanguine/services/rfq/api/db/sql/mysql" + "github.com/synapsecns/sanguine/services/rfq/api/metadata" + "gorm.io/gorm/schema" +) + +type DBSuite struct { + *testsuite.TestSuite + dbs []db.ApiDB + metrics metrics.Handler +} + +// NewDBSuite creates a new DBSuite. +func NewDBSuite(tb testing.TB) *DBSuite { + tb.Helper() + return &DBSuite{ + TestSuite: testsuite.NewTestSuite(tb), + dbs: []db.ApiDB{}, + } +} +func (d *DBSuite) SetupSuite() { + d.TestSuite.SetupSuite() + + // don't use metrics on ci for integration tests + isCI := core.GetEnvBool("CI", false) + useMetrics := !isCI + metricsHandler := metrics.Null + + if useMetrics { + localmetrics.SetupTestJaeger(d.GetSuiteContext(), d.T()) + metricsHandler = metrics.Jaeger + } + + var err error + d.metrics, err = metrics.NewByType(d.GetSuiteContext(), metadata.BuildInfo(), metricsHandler) + Nil(d.T(), err) +} + +func (d *DBSuite) SetupTest() { + d.TestSuite.SetupTest() + + sqliteStore, err := sql.Connect(d.GetTestContext(), dbcommon.Sqlite, filet.TmpDir(d.T(), ""), d.metrics) + Nil(d.T(), err) + + d.dbs = []db.ApiDB{sqliteStore} + d.setupMysqlDB() +} + +func (d *DBSuite) setupMysqlDB() { + if os.Getenv(dbcommon.EnableMysqlTestVar) != "true" { + return + } + + mysql.NamingStrategy = schema.NamingStrategy{ + TablePrefix: fmt.Sprintf("api_%d", d.GetTestID()), + } + + // sets up the conn string to the default database + connString := dbcommon.GetTestConnString() + // sets up the myqsl db + testDB, err := dbSQL.Open("mysql", connString) + d.Require().NoError(err) + // close the db once the connection is don + defer func() { + d.Require().NoError(testDB.Close()) + }() + + mysqlStore, err := mysql.NewMysqlStore(d.GetTestContext(), connString, d.metrics) + d.Require().NoError(err) + + d.dbs = append(d.dbs, mysqlStore) +} + +func (d *DBSuite) RunOnAllDBs(testFunc func(testDB db.ApiDB)) { + d.T().Helper() + + wg := sync.WaitGroup{} + for _, testDB := range d.dbs { + wg.Add(1) + // capture the value + go func(testDB db.ApiDB) { + defer wg.Done() + testFunc(testDB) + }(testDB) + } + wg.Wait() +} + +func TestDBSuite(t *testing.T) { + suite.Run(t, NewDBSuite(t)) +} From f5eab6496087820d9000c8697c064c9d4db3bc1f Mon Sep 17 00:00:00 2001 From: aureliusbtc <82057759+aureliusbtc@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:50:39 +0000 Subject: [PATCH 3/3] add metadata to api, rework model location --- services/rfq/api/db/api_db.go | 21 ++++++++++++++++++ services/rfq/api/db/sql/base/base.go | 2 +- services/rfq/api/db/sql/base/model.go | 31 --------------------------- services/rfq/api/metadata/metadata.go | 15 +++++++++++++ 4 files changed, 37 insertions(+), 32 deletions(-) delete mode 100644 services/rfq/api/db/sql/base/model.go create mode 100644 services/rfq/api/metadata/metadata.go diff --git a/services/rfq/api/db/api_db.go b/services/rfq/api/db/api_db.go index 714be11927..ab148a91da 100644 --- a/services/rfq/api/db/api_db.go +++ b/services/rfq/api/db/api_db.go @@ -1,5 +1,26 @@ package db +import ( + "time" + + "github.com/shopspring/decimal" +) + +type QuoteModel struct { + // ID is the unique identifier saved of each quote provided + ID uint64 `gorm:"column:id;primaryKey;"` + // DestChainID is the chain which the relayer is willing to provide liquidity for + DestChainID uint64 `gorm:"column:dest_chain_id;index"` + // DestToken is the token address for which the relayer is providing liquidity + DestTokenAddr string `gorm:"column:token;index"` + // DestAmount is the max amount of liquidity which exists for a given destination token, provided in the destination token decimals + DestAmount decimal.Decimal `gorm:"column:dest_amount"` + // Price is the price per origin token provided for which a relayer is indicating willingness to relay + Price decimal.Decimal `gorm:"column:price"` + // UpdatedAt is the time that the quote was last upserted + UpdatedAt time.Time `gorm:"column:updated_at"` +} + // ApiDBReader is the interface for reading from the database. type ApiDBReader interface { } diff --git a/services/rfq/api/db/sql/base/base.go b/services/rfq/api/db/sql/base/base.go index b232c1db6a..c9bf9451c6 100644 --- a/services/rfq/api/db/sql/base/base.go +++ b/services/rfq/api/db/sql/base/base.go @@ -25,7 +25,7 @@ func (s Store) DB() *gorm.DB { // GetAllModels gets all models to migrate. // see: https://medium.com/@SaifAbid/slice-interfaces-8c78f8b6345d for an explanation of why we can't do this at initialization time func GetAllModels() (allModels []interface{}) { - allModels = append(allModels, &Quote{}) + allModels = append(allModels, &db.QuoteModel{}) return allModels } diff --git a/services/rfq/api/db/sql/base/model.go b/services/rfq/api/db/sql/base/model.go deleted file mode 100644 index 8d8002692f..0000000000 --- a/services/rfq/api/db/sql/base/model.go +++ /dev/null @@ -1,31 +0,0 @@ -package base - -import ( - "time" - - "github.com/shopspring/decimal" -) - -// define common field names. See package docs for an explanation of why we have to do this. -// note: some models share names. In cases where they do, we run the check against all names. -// This is cheap because it's only done at startup. -func init() { - -} - -var () - -type Quote struct { - // ID is the unique identifier saved of each quote provided - ID uint64 `gorm:"column:id;primaryKey;"` - // DestChainID is the chain which the relayer is willing to provide liquidity for - DestChainID uint64 `gorm:"column:dest_chain_id;index"` - // DestToken is the token address for which the relayer is providing liquidity - DestTokenAddr string `gorm:"column:token;index"` - // DestAmount is the max amount of liquidity which exists for a given destination token, provided in the destination token decimals - DestAmount decimal.Decimal `gorm:"column:dest_amount"` - // Price is the price per origin token provided for which a relayer is indicating willingness to relay - Price decimal.Decimal `gorm:"column:price"` - // UpdatedAt is the time that the quote was last upserted - UpdatedAt time.Time `gorm:"column:updated_at"` -} diff --git a/services/rfq/api/metadata/metadata.go b/services/rfq/api/metadata/metadata.go new file mode 100644 index 0000000000..70a90a0ab8 --- /dev/null +++ b/services/rfq/api/metadata/metadata.go @@ -0,0 +1,15 @@ +// Package metadata provides a metadata service for the RFQ API. +package metadata + +import "github.com/synapsecns/sanguine/core/config" + +var ( + version = config.DefaultVersion + commit = config.DefaultCommit + date = config.DefaultDate +) + +// BuildInfo returns the build info for the service. +func BuildInfo() config.BuildInfo { + return config.NewBuildInfo(version, commit, "rfq-api", date) +}