Skip to content

Commit

Permalink
sync2: add sqlstore (#6405)
Browse files Browse the repository at this point in the history
-------

## Motivation

The database-aware sync implementation is rather complex and splitting
it in parts might help the review process.
  • Loading branch information
ivan4th committed Nov 3, 2024
1 parent 8f93588 commit f6bd0a3
Show file tree
Hide file tree
Showing 16 changed files with 2,164 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ require (
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.60.1
github.com/quic-go/quic-go v0.48.1
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd
github.com/rs/cors v1.11.1
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/seehuhn/mt19937 v1.0.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,8 @@ github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
Expand Down Expand Up @@ -577,6 +579,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd h1:wW6BtayFoKaaDeIvXRE3SZVPOscSKlYD+X3bB749+zk=
github.com/rqlite/sql v0.0.0-20240312185922-ffac88a740bd/go.mod h1:ib9zVtNgRKiGuoMyUqqL5aNpk+r+++YlyiVIkclVqPg=
github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
Expand Down
10 changes: 10 additions & 0 deletions sql/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"
"sync"
"sync/atomic"
"testing"
"time"

sqlite "github.com/go-llsqlite/crawshaw"
Expand Down Expand Up @@ -236,6 +237,15 @@ func InMemory(opts ...Opt) *sqliteDatabase {
return db
}

// InMemoryTest returns an in-mem database for testing and ensures database is closed during `tb.Cleanup`.
func InMemoryTest(tb testing.TB, opts ...Opt) *sqliteDatabase {
// When using empty DB schema, we don't want to check for schema drift due to
// "PRAGMA user_version = 0;" in the initial schema retrieved from the DB.
db := InMemory(append(opts, WithNoCheckSchemaDrift())...)
tb.Cleanup(func() { db.Close() })
return db
}

// Open database with options.
//
// Database is opened in WAL mode and pragma synchronous=normal.
Expand Down
212 changes: 212 additions & 0 deletions sql/expr/expr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
// Package expr proviedes a simple SQL expression parser and builder.
// It wraps the rqlite/sql package and provides a more convenient API that contains only
// what's needed for the go-spacemesh codebase.
package expr

import (
"strings"

rsql "github.com/rqlite/sql"
)

// SQL operations.
const (
NE = rsql.NE // !=
EQ = rsql.EQ // =
LE = rsql.LE // <=
LT = rsql.LT // <
GT = rsql.GT // >
GE = rsql.GE // >=
BITAND = rsql.BITAND // &
BITOR = rsql.BITOR // |
BITNOT = rsql.BITNOT // !
LSHIFT = rsql.LSHIFT // <<
RSHIFT = rsql.RSHIFT // >>
PLUS = rsql.PLUS // +
MINUS = rsql.MINUS // -
STAR = rsql.STAR // *
SLASH = rsql.SLASH // /
REM = rsql.REM // %
CONCAT = rsql.CONCAT // ||
DOT = rsql.DOT // .
AND = rsql.AND
OR = rsql.OR
NOT = rsql.NOT
)

// Expr represents a parsed SQL expression.
type Expr = rsql.Expr

// Statement represents a parsed SQL statement.
type Statement = rsql.Statement

// MustParse parses an SQL expression and panics if there's an error.
func MustParse(s string) rsql.Expr {
expr, err := rsql.ParseExprString(s)
if err != nil {
panic("error parsing SQL expression: " + err.Error())
}
return expr
}

// MustParseStatement parses an SQL statement and panics if there's an error.
func MustParseStatement(s string) rsql.Statement {
st, err := rsql.NewParser(strings.NewReader(s)).ParseStatement()
if err != nil {
panic("error parsing SQL statement: " + err.Error())
}
return st
}

// MaybeAnd joins together several SQL expressions with AND, ignoring any nil exprs.
// If no non-nil expressions are passed, nil is returned.
// If a single non-nil expression is passed, that single expression is returned.
// Otherwise, the expressions are joined together with ANDs:
// a AND b AND c AND d.
func MaybeAnd(exprs ...Expr) Expr {
var r Expr
for _, expr := range exprs {
switch {
case expr == nil:
case r == nil:
r = expr
default:
r = Op(r, AND, expr)
}
}
return r
}

// Ident constructs SQL identifier expression for the identifier with the specified name.
func Ident(name string) *rsql.Ident {
return &rsql.Ident{Name: name}
}

// Number constructs a number literal.
func Number(value string) *rsql.NumberLit {
return &rsql.NumberLit{Value: value}
}

// TableSource constructs a Source clause for SELECT statement that corresponds to
// selecting from a single table with the specified name.
func TableSource(name string) rsql.Source {
return &rsql.QualifiedTableName{Name: Ident(name)}
}

// Op constructs a binary expression such as x + y or x < y.
func Op(x Expr, op rsql.Token, y Expr) Expr {
return &rsql.BinaryExpr{
X: x,
Op: op,
Y: y,
}
}

// Bind constructs the unnamed bind expression (?).
func Bind() Expr {
return &rsql.BindExpr{Name: "?"}
}

// Between constructs BETWEEN expression: x BETWEEN a AND b.
func Between(x, a, b Expr) Expr {
return Op(x, rsql.BETWEEN, &rsql.Range{X: a, Y: b})
}

// Call constructs a call expression with specified arguments such as max(x).
func Call(name string, args ...Expr) Expr {
return &rsql.Call{Name: Ident(name), Args: args}
}

// CountStar returns a COUNT(*) expression.
func CountStar() Expr {
return &rsql.Call{Name: Ident("count"), Star: rsql.Pos{Offset: 1}}
}

// Asc constructs an ascending ORDER BY term.
func Asc(expr Expr) *rsql.OrderingTerm {
return &rsql.OrderingTerm{X: expr}
}

// Desc constructs a descedning ORDER BY term.
func Desc(expr Expr) *rsql.OrderingTerm {
return &rsql.OrderingTerm{X: expr, Desc: rsql.Pos{Offset: 1}}
}

// SelectBuilder is used to construct a SELECT statement.
type SelectBuilder struct {
st *rsql.SelectStatement
}

// Select returns a SELECT statement builder.
func Select(columns ...any) SelectBuilder {
sb := SelectBuilder{st: &rsql.SelectStatement{}}
return sb.Columns(columns...)
}

// SelectBasedOn returns a SELECT statement builder based on the specified SELECT statement.
// The statement must be parseable, otherwise SelectBasedOn panics.
// The builder methods can be used to alter the statement.
func SelectBasedOn(st Statement) SelectBuilder {
st = rsql.CloneStatement(st)
return SelectBuilder{st: st.(*rsql.SelectStatement)}
}

// Get returns the underlying SELECT statement.
func (sb SelectBuilder) Get() *rsql.SelectStatement {
return sb.st
}

// String returns the underlying SELECT statement as a string.
func (sb SelectBuilder) String() string {
return sb.st.String()
}

// Columns sets columns in the SELECT statement.
func (sb SelectBuilder) Columns(columns ...any) SelectBuilder {
sb.st.Columns = make([]*rsql.ResultColumn, len(columns))
for n, column := range columns {
switch c := column.(type) {
case *rsql.ResultColumn:
sb.st.Columns[n] = c
case Expr:
sb.st.Columns[n] = &rsql.ResultColumn{Expr: c}
default:
panic("unexpected column type")
}
}
return sb
}

// From adds FROM clause to the SELECT statement.
func (sb SelectBuilder) From(s rsql.Source) SelectBuilder {
sb.st.Source = s
return sb
}

// From adds WHERE clause to the SELECT statement.
func (sb SelectBuilder) Where(s Expr) SelectBuilder {
sb.st.WhereExpr = s
return sb
}

// From adds ORDER BY clause to the SELECT statement.
func (sb SelectBuilder) OrderBy(terms ...*rsql.OrderingTerm) SelectBuilder {
sb.st.OrderingTerms = terms
return sb
}

// From adds LIMIT clause to the SELECT statement.
func (sb SelectBuilder) Limit(limit Expr) SelectBuilder {
sb.st.LimitExpr = limit
return sb
}

// ColumnExpr returns nth column expression from the SELECT statement.
func ColumnExpr(st Statement, n int) Expr {
return st.(*rsql.SelectStatement).Columns[n].Expr
}

// WhereExpr returns WHERE expression from the SELECT statement.
func WhereExpr(st Statement) Expr {
return st.(*rsql.SelectStatement).WhereExpr
}
Loading

0 comments on commit f6bd0a3

Please sign in to comment.