From 0dd97f66f8fdbb00a95f8d546df5154ca8c2eae1 Mon Sep 17 00:00:00 2001 From: Daylon Wilkins Date: Wed, 26 Jul 2023 15:13:14 -0700 Subject: [PATCH] Implemented Full-Text indexes --- go/gen/fb/serial/schema.go | 215 +++++++++++++- go/go.mod | 2 +- go/go.sum | 4 +- go/libraries/doltcore/doltdb/system_table.go | 14 +- .../doltcore/merge/fulltext_rebuild.go | 213 ++++++++++++++ go/libraries/doltcore/merge/fulltext_table.go | 196 ++++++++++++ go/libraries/doltcore/merge/merge.go | 8 + .../schema/encoding/schema_marshaling.go | 50 +++- .../doltcore/schema/encoding/serialization.go | 70 ++++- go/libraries/doltcore/schema/index.go | 22 ++ go/libraries/doltcore/schema/index_coll.go | 42 +++ go/libraries/doltcore/sqle/alterschema.go | 10 +- go/libraries/doltcore/sqle/database.go | 17 +- .../sqle/enginetest/dolt_engine_test.go | 9 + .../doltcore/sqle/enginetest/validation.go | 8 + .../doltcore/sqle/index/dolt_index.go | 70 +++++ go/libraries/doltcore/sqle/sqlutil/schema.go | 7 +- go/libraries/doltcore/sqle/tables.go | 278 +++++++++++++----- go/libraries/doltcore/sqle/temp_table.go | 11 +- .../doltcore/sqle/writer/noms_table_writer.go | 3 +- .../sqle/writer/prolly_table_writer.go | 7 + .../doltcore/table/editor/creation/index.go | 12 +- go/serial/schema.fbs | 15 + integration-tests/bats/fulltext.bats | 72 +++++ 24 files changed, 1237 insertions(+), 118 deletions(-) create mode 100644 go/libraries/doltcore/merge/fulltext_rebuild.go create mode 100644 go/libraries/doltcore/merge/fulltext_table.go create mode 100644 integration-tests/bats/fulltext.bats diff --git a/go/gen/fb/serial/schema.go b/go/gen/fb/serial/schema.go index 8abcc332f4d..8fa1a390a52 100644 --- a/go/gen/fb/serial/schema.go +++ b/go/gen/fb/serial/schema.go @@ -687,7 +687,48 @@ func (rcv *Index) MutateSpatialKey(n bool) bool { return rcv._tab.MutateBoolSlot(22, n) } -const IndexNumFields = 10 +func (rcv *Index) FulltextKey() bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(24)) + if o != 0 { + return rcv._tab.GetBool(o + rcv._tab.Pos) + } + return false +} + +func (rcv *Index) MutateFulltextKey(n bool) bool { + return rcv._tab.MutateBoolSlot(24, n) +} + +func (rcv *Index) FulltextInfo(obj *FulltextInfo) *FulltextInfo { + o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(FulltextInfo) + } + obj.Init(rcv._tab.Bytes, x) + return obj + } + return nil +} + +func (rcv *Index) TryFulltextInfo(obj *FulltextInfo) (*FulltextInfo, error) { + o := flatbuffers.UOffsetT(rcv._tab.Offset(26)) + if o != 0 { + x := rcv._tab.Indirect(o + rcv._tab.Pos) + if obj == nil { + obj = new(FulltextInfo) + } + obj.Init(rcv._tab.Bytes, x) + if FulltextInfoNumFields < obj.Table().NumFields() { + return nil, flatbuffers.ErrTableHasUnknownFields + } + return obj, nil + } + return nil, nil +} + +const IndexNumFields = 12 func IndexStart(builder *flatbuffers.Builder) { builder.StartObject(IndexNumFields) @@ -734,10 +775,182 @@ func IndexStartPrefixLengthsVector(builder *flatbuffers.Builder, numElems int) f func IndexAddSpatialKey(builder *flatbuffers.Builder, spatialKey bool) { builder.PrependBoolSlot(9, spatialKey, false) } +func IndexAddFulltextKey(builder *flatbuffers.Builder, fulltextKey bool) { + builder.PrependBoolSlot(10, fulltextKey, false) +} +func IndexAddFulltextInfo(builder *flatbuffers.Builder, fulltextInfo flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(11, flatbuffers.UOffsetT(fulltextInfo), 0) +} func IndexEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } +type FulltextInfo struct { + _tab flatbuffers.Table +} + +func InitFulltextInfoRoot(o *FulltextInfo, buf []byte, offset flatbuffers.UOffsetT) error { + n := flatbuffers.GetUOffsetT(buf[offset:]) + o.Init(buf, n+offset) + if FulltextInfoNumFields < o.Table().NumFields() { + return flatbuffers.ErrTableHasUnknownFields + } + return nil +} + +func TryGetRootAsFulltextInfo(buf []byte, offset flatbuffers.UOffsetT) (*FulltextInfo, error) { + x := &FulltextInfo{} + return x, InitFulltextInfoRoot(x, buf, offset) +} + +func GetRootAsFulltextInfo(buf []byte, offset flatbuffers.UOffsetT) *FulltextInfo { + x := &FulltextInfo{} + InitFulltextInfoRoot(x, buf, offset) + return x +} + +func TryGetSizePrefixedRootAsFulltextInfo(buf []byte, offset flatbuffers.UOffsetT) (*FulltextInfo, error) { + x := &FulltextInfo{} + return x, InitFulltextInfoRoot(x, buf, offset+flatbuffers.SizeUint32) +} + +func GetSizePrefixedRootAsFulltextInfo(buf []byte, offset flatbuffers.UOffsetT) *FulltextInfo { + x := &FulltextInfo{} + InitFulltextInfoRoot(x, buf, offset+flatbuffers.SizeUint32) + return x +} + +func (rcv *FulltextInfo) Init(buf []byte, i flatbuffers.UOffsetT) { + rcv._tab.Bytes = buf + rcv._tab.Pos = i +} + +func (rcv *FulltextInfo) Table() flatbuffers.Table { + return rcv._tab +} + +func (rcv *FulltextInfo) ConfigTable() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) PositionTable() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) DocCountTable() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) GlobalCountTable() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) RowCountTable() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) KeyType() byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + if o != 0 { + return rcv._tab.GetByte(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *FulltextInfo) MutateKeyType(n byte) bool { + return rcv._tab.MutateByteSlot(14, n) +} + +func (rcv *FulltextInfo) KeyName() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + +func (rcv *FulltextInfo) KeyPositions(j int) uint16 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.GetUint16(a + flatbuffers.UOffsetT(j*2)) + } + return 0 +} + +func (rcv *FulltextInfo) KeyPositionsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + +func (rcv *FulltextInfo) MutateKeyPositions(j int, n uint16) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) + if o != 0 { + a := rcv._tab.Vector(o) + return rcv._tab.MutateUint16(a+flatbuffers.UOffsetT(j*2), n) + } + return false +} + +const FulltextInfoNumFields = 8 + +func FulltextInfoStart(builder *flatbuffers.Builder) { + builder.StartObject(FulltextInfoNumFields) +} +func FulltextInfoAddConfigTable(builder *flatbuffers.Builder, configTable flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(configTable), 0) +} +func FulltextInfoAddPositionTable(builder *flatbuffers.Builder, positionTable flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(positionTable), 0) +} +func FulltextInfoAddDocCountTable(builder *flatbuffers.Builder, docCountTable flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(docCountTable), 0) +} +func FulltextInfoAddGlobalCountTable(builder *flatbuffers.Builder, globalCountTable flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(globalCountTable), 0) +} +func FulltextInfoAddRowCountTable(builder *flatbuffers.Builder, rowCountTable flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(rowCountTable), 0) +} +func FulltextInfoAddKeyType(builder *flatbuffers.Builder, keyType byte) { + builder.PrependByteSlot(5, keyType, 0) +} +func FulltextInfoAddKeyName(builder *flatbuffers.Builder, keyName flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(keyName), 0) +} +func FulltextInfoAddKeyPositions(builder *flatbuffers.Builder, keyPositions flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(7, flatbuffers.UOffsetT(keyPositions), 0) +} +func FulltextInfoStartKeyPositionsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(2, numElems, 2) +} +func FulltextInfoEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { + return builder.EndObject() +} + type CheckConstraint struct { _tab flatbuffers.Table } diff --git a/go/go.mod b/go/go.mod index 1981160a6dd..95d6a5b96bb 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.16.1-0.20230725191100-b5891a70d94a + github.com/dolthub/go-mysql-server v0.16.1-0.20230726215023-ca0b602ba632 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 3d5aec2558c..5938b455135 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,8 +180,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.16.1-0.20230725191100-b5891a70d94a h1:6swKWvtykzkKBBEUDFLenQH/YMwc7gXGNjymYStsS4Y= -github.com/dolthub/go-mysql-server v0.16.1-0.20230725191100-b5891a70d94a/go.mod h1:HNJKWMlO3657iU4+lnAmXoZNXu4BrKR7LC/cw+IE4QM= +github.com/dolthub/go-mysql-server v0.16.1-0.20230726215023-ca0b602ba632 h1:5gheD0zd9dqHWzy4kvdNT99E0z1QYXXE99t4EFoPwNQ= +github.com/dolthub/go-mysql-server v0.16.1-0.20230726215023-ca0b602ba632/go.mod h1:HNJKWMlO3657iU4+lnAmXoZNXu4BrKR7LC/cw+IE4QM= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= diff --git a/go/libraries/doltcore/doltdb/system_table.go b/go/libraries/doltcore/doltdb/system_table.go index c4e07d41625..f12ea48ba00 100644 --- a/go/libraries/doltcore/doltdb/system_table.go +++ b/go/libraries/doltcore/doltdb/system_table.go @@ -60,16 +60,26 @@ func HasDoltPrefix(s string) bool { return strings.HasPrefix(strings.ToLower(s), DoltNamespace) } +// IsFullTextTable returns a boolean stating whether the given table is one of the pseudo-index tables used by Full-Text +// indexes. +func IsFullTextTable(name string) bool { + return HasDoltPrefix(name) && (strings.HasSuffix(name, "_fts_config") || + strings.HasSuffix(name, "_fts_position") || + strings.HasSuffix(name, "_fts_doc_count") || + strings.HasSuffix(name, "_fts_global_count") || + strings.HasSuffix(name, "_fts_row_count")) +} + // IsReadOnlySystemTable returns whether the table name given is a system table that should not be included in command line // output (e.g. dolt status) by default. func IsReadOnlySystemTable(name string) bool { - return HasDoltPrefix(name) && !set.NewStrSet(writeableSystemTables).Contains(name) + return HasDoltPrefix(name) && !set.NewStrSet(writeableSystemTables).Contains(name) && !IsFullTextTable(name) } // IsNonAlterableSystemTable returns whether the table name given is a system table that cannot be dropped or altered // by the user. func IsNonAlterableSystemTable(name string) bool { - return IsReadOnlySystemTable(name) || strings.ToLower(name) == SchemasTableName + return (IsReadOnlySystemTable(name) && !IsFullTextTable(name)) || strings.ToLower(name) == SchemasTableName } // GetNonSystemTableNames gets non-system table names diff --git a/go/libraries/doltcore/merge/fulltext_rebuild.go b/go/libraries/doltcore/merge/fulltext_rebuild.go new file mode 100644 index 00000000000..739480280c0 --- /dev/null +++ b/go/libraries/doltcore/merge/fulltext_rebuild.go @@ -0,0 +1,213 @@ +// Copyright 2023 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "fmt" + "io" + "strings" + + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/fulltext" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" +) + +// rebuildFullTextIndexes scans the entire root and rebuilds all of the pseudo-index tables. This is not the most +// efficient way to go about it, but it at least produces the correct result. +func rebuildFullTextIndexes(ctx *sql.Context, root *doltdb.RootValue) (*doltdb.RootValue, error) { + // Grab a list of all tables on the root + allTableNames, err := root.GetTableNames(ctx) + if err != nil { + return nil, err + } + // We'll purge the data from every Full-Text table so that we may rewrite their contents + for _, tblName := range allTableNames { + if !doltdb.IsFullTextTable(tblName) { + continue + } else if strings.HasSuffix(tblName, "config") { + // We don't want to purge the config table, we'll just roll with whatever is there for now + continue + } + tbl, ok, err := root.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("attempted to purge `%s` during Full-Text merge but it could not be found", tblName) + } + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, err + } + rows, err := durable.NewEmptyIndex(ctx, tbl.ValueReadWriter(), tbl.NodeStore(), sch) + if err != nil { + return nil, err + } + tbl, err = tbl.UpdateRows(ctx, rows) + if err != nil { + return nil, err + } + root, err = root.PutTable(ctx, tblName, tbl) + if err != nil { + return nil, err + } + } + // Loop again, this time only looking at tables that declare a Full-Text index + for _, tblName := range allTableNames { + if doltdb.IsFullTextTable(tblName) { + continue + } + tbl, ok, err := root.GetTable(ctx, tblName) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("attempted to load `%s` during Full-Text merge but it could not be found", tblName) + } + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, err + } + if !sch.Indexes().ContainsFullTextIndex() { + continue + } + parentTable, err := createFulltextTable(ctx, tblName, root) + if err != nil { + return nil, err + } + + var configTable *fulltextTable + var tableSet []fulltext.TableSet + allFTDoltTables := make(map[string]*fulltextTable) + for _, idx := range sch.Indexes().AllIndexes() { + if !idx.IsFullText() { + continue + } + props := idx.FullTextProperties() + // The config table is shared, and it's not written to during this process + if configTable == nil { + configTable, err = createFulltextTable(ctx, props.ConfigTable, root) + if err != nil { + return nil, err + } + allFTDoltTables[props.ConfigTable] = configTable + } + positionTable, err := createFulltextTable(ctx, props.PositionTable, root) + if err != nil { + return nil, err + } + docCountTable, err := createFulltextTable(ctx, props.DocCountTable, root) + if err != nil { + return nil, err + } + globalCountTable, err := createFulltextTable(ctx, props.GlobalCountTable, root) + if err != nil { + return nil, err + } + rowCountTable, err := createFulltextTable(ctx, props.RowCountTable, root) + if err != nil { + return nil, err + } + allFTDoltTables[props.PositionTable] = positionTable + allFTDoltTables[props.DocCountTable] = docCountTable + allFTDoltTables[props.GlobalCountTable] = globalCountTable + allFTDoltTables[props.RowCountTable] = rowCountTable + ftIndex, err := index.ConvertFullTextToSql(ctx, "", tblName, sch, idx) + if err != nil { + return nil, err + } + tableSet = append(tableSet, fulltext.TableSet{ + Index: ftIndex.(fulltext.Index), + Position: positionTable, + DocCount: docCountTable, + GlobalCount: globalCountTable, + RowCount: rowCountTable, + }) + } + + // We'll write the entire contents of our table into the Full-Text editor + ftEditor, err := fulltext.CreateEditor(ctx, parentTable, configTable, tableSet...) + if err != nil { + return nil, err + } + err = func() error { + defer ftEditor.Close(ctx) + ftEditor.StatementBegin(ctx) + defer ftEditor.StatementComplete(ctx) + + rowIter, err := createRowIterForTable(ctx, tblName, tbl, sch) + if err != nil { + return err + } + defer rowIter.Close(ctx) + + row, err := rowIter.Next(ctx) + for ; err == nil; row, err = rowIter.Next(ctx) { + if err = ftEditor.Insert(ctx, row); err != nil { + return err + } + } + if err != nil && err != io.EOF { + return err + } + return nil + }() + if err != nil { + return nil, err + } + + // Update the root with all of the new tables' contents + for _, ftTable := range allFTDoltTables { + newTbl, err := ftTable.ApplyToTable(ctx) + if err != nil { + return nil, err + } + root, err = root.PutTable(ctx, ftTable.Name(), newTbl) + if err != nil { + return nil, err + } + } + } + return root, nil +} + +func createRowIterForTable(ctx *sql.Context, name string, t *doltdb.Table, sch schema.Schema) (sql.RowIter, error) { + sqlSch, err := sqlutil.FromDoltSchema(name, sch) + if err != nil { + return nil, err + } + + rowData, err := t.GetRowData(ctx) + if err != nil { + return nil, err + } + rows := durable.ProllyMapFromIndex(rowData) + rowCount, err := rows.Count() + if err != nil { + return nil, err + } + + iter, err := rows.FetchOrdinalRange(ctx, 0, uint64(rowCount)) + if err != nil { + return nil, err + } + + return index.NewProllyRowIter(sch, sqlSch.Schema, rows, iter, nil) +} diff --git a/go/libraries/doltcore/merge/fulltext_table.go b/go/libraries/doltcore/merge/fulltext_table.go new file mode 100644 index 00000000000..145e697c3f0 --- /dev/null +++ b/go/libraries/doltcore/merge/fulltext_table.go @@ -0,0 +1,196 @@ +// Copyright 2023 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package merge + +import ( + "fmt" + "io" + + "github.com/dolthub/go-mysql-server/memory" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/fulltext" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" + "github.com/dolthub/dolt/go/store/pool" + "github.com/dolthub/dolt/go/store/val" +) + +var sharePool = pool.NewBuffPool() + +type fulltextTable struct { + GMSTable *memory.Table + Table *doltdb.Table + Sch schema.Schema + SqlSch sql.Schema +} + +var _ fulltext.EditableTable = (*fulltextTable)(nil) + +func createFulltextTable(ctx *sql.Context, name string, root *doltdb.RootValue) (*fulltextTable, error) { + tbl, ok, err := root.GetTable(ctx, name) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("attempted to load Full-Text table `%s` during Full-Text merge but it could not be found", name) + } + sch, err := tbl.GetSchema(ctx) + if err != nil { + return nil, err + } + sqlSch, err := sqlutil.FromDoltSchema(name, sch) + if err != nil { + return nil, err + } + gmsTable := memory.NewTable(name, sqlSch, nil) + gmsTable.EnablePrimaryKeyIndexes() + return &fulltextTable{ + GMSTable: gmsTable, + Table: tbl, + Sch: sch, + SqlSch: sqlSch.Schema, + }, nil +} + +// Name implements the interface fulltext.EditableTable. +func (table *fulltextTable) Name() string { + return table.GMSTable.Name() +} + +// String implements the interface fulltext.EditableTable. +func (table *fulltextTable) String() string { + return table.GMSTable.String() +} + +// Schema implements the interface fulltext.EditableTable. +func (table *fulltextTable) Schema() sql.Schema { + return table.GMSTable.Schema() +} + +// Collation implements the interface fulltext.EditableTable. +func (table *fulltextTable) Collation() sql.CollationID { + return table.GMSTable.Collation() +} + +// Partitions implements the interface fulltext.EditableTable. +func (table *fulltextTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { + return table.GMSTable.Partitions(ctx) +} + +// PartitionRows implements the interface fulltext.EditableTable. +func (table *fulltextTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) { + return table.GMSTable.PartitionRows(ctx, partition) +} + +// Inserter implements the interface fulltext.EditableTable. +func (table *fulltextTable) Inserter(ctx *sql.Context) sql.RowInserter { + return table.GMSTable.Inserter(ctx) +} + +// Updater implements the interface fulltext.EditableTable. +func (table *fulltextTable) Updater(ctx *sql.Context) sql.RowUpdater { + return table.GMSTable.Updater(ctx) +} + +// Deleter implements the interface fulltext.EditableTable. +func (table *fulltextTable) Deleter(ctx *sql.Context) sql.RowDeleter { + return table.GMSTable.Deleter(ctx) +} + +// IndexedAccess implements the interface fulltext.EditableTable. +func (table *fulltextTable) IndexedAccess(lookup sql.IndexLookup) sql.IndexedTable { + return table.GMSTable.IndexedAccess(lookup) +} + +// GetIndexes implements the interface fulltext.EditableTable. +func (table *fulltextTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) { + return table.GMSTable.GetIndexes(ctx) +} + +// ApplyToTable writes the data from the internal GMS table to the internal Dolt table, then returns the updated Dolt +// table. The updated Dolt table is not stored. +func (table *fulltextTable) ApplyToTable(ctx *sql.Context) (*doltdb.Table, error) { + partIter, err := table.GMSTable.Partitions(ctx) + if err != nil { + return nil, err + } + rowIter := sql.NewTableRowIter(ctx, table.GMSTable, partIter) + + idx, err := table.Table.GetRowData(ctx) + if err != nil { + return nil, err + } + m := durable.ProllyMapFromIndex(idx) + keyDesc, valDesc := m.Descriptors() + keyMap, valMap := ordinalMappingsFromSchema(table.SqlSch, table.Sch) + mut := m.Mutate() + keyBld := val.NewTupleBuilder(keyDesc) + valBld := val.NewTupleBuilder(valDesc) + + sqlRow, err := rowIter.Next(ctx) + for ; err == nil; sqlRow, err = rowIter.Next(ctx) { + for to := range keyMap { + from := keyMap.MapOrdinal(to) + if err = index.PutField(ctx, mut.NodeStore(), keyBld, to, sqlRow[from]); err != nil { + return nil, err + } + } + k := keyBld.Build(sharePool) + + for to := range valMap { + from := valMap.MapOrdinal(to) + if err = index.PutField(ctx, mut.NodeStore(), valBld, to, sqlRow[from]); err != nil { + return nil, err + } + } + v := valBld.Build(sharePool) + + if err = mut.Put(ctx, k, v); err != nil { + return nil, err + } + } + if err != nil && err != io.EOF { + return nil, err + } + + mapped, err := mut.Map(ctx) + if err != nil { + return nil, err + } + return table.Table.UpdateRows(ctx, durable.IndexFromProllyMap(mapped)) +} + +func ordinalMappingsFromSchema(from sql.Schema, to schema.Schema) (km, vm val.OrdinalMapping) { + km = makeOrdinalMapping(from, to.GetPKCols()) + vm = makeOrdinalMapping(from, to.GetNonPKCols()) + return +} + +func makeOrdinalMapping(from sql.Schema, to *schema.ColCollection) (m val.OrdinalMapping) { + m = make(val.OrdinalMapping, len(to.GetColumns())) + for i := range m { + name := to.GetByIndex(i).Name + for j, col := range from { + if col.Name == name { + m[i] = j + } + } + } + return +} diff --git a/go/libraries/doltcore/merge/merge.go b/go/libraries/doltcore/merge/merge.go index ff8367c2991..fd7bdff3765 100644 --- a/go/libraries/doltcore/merge/merge.go +++ b/go/libraries/doltcore/merge/merge.go @@ -205,6 +205,9 @@ func MergeRoots( var schConflicts []SchemaConflict for _, tblName := range tblNames { + if doltdb.IsFullTextTable(tblName) { + continue + } mergedTable, stats, err := merger.MergeTable(ctx, tblName, opts, mergeOpts) if err != nil { return nil, err @@ -250,6 +253,11 @@ func MergeRoots( } } + mergedRoot, err = rebuildFullTextIndexes(ctx, mergedRoot) + if err != nil { + return nil, err + } + mergedFKColl, conflicts, err := ForeignKeysMerge(ctx, mergedRoot, ourRoot, theirRoot, ancRoot) if err != nil { return nil, err diff --git a/go/libraries/doltcore/schema/encoding/schema_marshaling.go b/go/libraries/doltcore/schema/encoding/schema_marshaling.go index 4cab1961c22..3e4282c15d7 100644 --- a/go/libraries/doltcore/schema/encoding/schema_marshaling.go +++ b/go/libraries/doltcore/schema/encoding/schema_marshaling.go @@ -148,13 +148,26 @@ func (enc encodedTypeInfo) decodeTypeInfo() (typeinfo.TypeInfo, error) { } type encodedIndex struct { - Name string `noms:"name" json:"name"` - Tags []uint64 `noms:"tags" json:"tags"` - Comment string `noms:"comment" json:"comment"` - Unique bool `noms:"unique" json:"unique"` - Spatial bool `noms:"spatial,omitempty" json:"spatial,omitempty"` - IsSystemDefined bool `noms:"hidden,omitempty" json:"hidden,omitempty"` // Was previously named Hidden, do not change noms name - PrefixLengths []uint16 `noms:"prefixLengths,omitempty" json:"prefixLengths,omitempty"` + Name string `noms:"name" json:"name"` + Tags []uint64 `noms:"tags" json:"tags"` + Comment string `noms:"comment" json:"comment"` + Unique bool `noms:"unique" json:"unique"` + Spatial bool `noms:"spatial,omitempty" json:"spatial,omitempty"` + FullText bool `noms:"fulltext,omitempty" json:"fulltext,omitempty"` + IsSystemDefined bool `noms:"hidden,omitempty" json:"hidden,omitempty"` // Was previously named Hidden, do not change noms name + PrefixLengths []uint16 `noms:"prefixLengths,omitempty" json:"prefixLengths,omitempty"` + FullTextInfo encodedFullTextInfo `noms:"fulltext_info,omitempty" json:"fulltext_info,omitempty"` +} + +type encodedFullTextInfo struct { + ConfigTable string `noms:"config_table" json:"config_table"` + PositionTable string `noms:"position_table" json:"position_table"` + DocCountTable string `noms:"doc_count_table" json:"doc_count_table"` + GlobalCountTable string `noms:"global_count_table" json:"global_count_table"` + RowCountTable string `noms:"row_count_table" json:"row_count_table"` + KeyType uint8 `noms:"key_type" json:"key_type"` + KeyName string `noms:"key_name" json:"key_name"` + KeyPositions []uint16 `noms:"key_positions" json:"key_positions"` } type encodedCheck struct { @@ -241,14 +254,26 @@ func toSchemaData(sch schema.Schema) (schemaData, error) { encodedIndexes := make([]encodedIndex, sch.Indexes().Count()) for i, index := range sch.Indexes().AllIndexes() { + props := index.FullTextProperties() encodedIndexes[i] = encodedIndex{ Name: index.Name(), Tags: index.IndexedColumnTags(), Comment: index.Comment(), Unique: index.IsUnique(), Spatial: index.IsSpatial(), + FullText: index.IsFullText(), IsSystemDefined: !index.IsUserDefined(), PrefixLengths: index.PrefixLengths(), + FullTextInfo: encodedFullTextInfo{ + ConfigTable: props.ConfigTable, + PositionTable: props.PositionTable, + DocCountTable: props.DocCountTable, + GlobalCountTable: props.GlobalCountTable, + RowCountTable: props.RowCountTable, + KeyType: props.KeyType, + KeyName: props.KeyName, + KeyPositions: props.KeyPositions, + }, } } @@ -311,8 +336,19 @@ func (sd schemaData) addChecksIndexesAndPkOrderingToSchema(sch schema.Schema) er schema.IndexProperties{ IsUnique: encodedIndex.Unique, IsSpatial: encodedIndex.Spatial, + IsFullText: encodedIndex.FullText, IsUserDefined: !encodedIndex.IsSystemDefined, Comment: encodedIndex.Comment, + FullTextProperties: schema.FullTextProperties{ + ConfigTable: encodedIndex.FullTextInfo.ConfigTable, + PositionTable: encodedIndex.FullTextInfo.PositionTable, + DocCountTable: encodedIndex.FullTextInfo.DocCountTable, + GlobalCountTable: encodedIndex.FullTextInfo.GlobalCountTable, + RowCountTable: encodedIndex.FullTextInfo.RowCountTable, + KeyType: encodedIndex.FullTextInfo.KeyType, + KeyName: encodedIndex.FullTextInfo.KeyName, + KeyPositions: encodedIndex.FullTextInfo.KeyPositions, + }, }, ) if err != nil { diff --git a/go/libraries/doltcore/schema/encoding/serialization.go b/go/libraries/doltcore/schema/encoding/serialization.go index 9bfdca808f3..208b6183ff8 100644 --- a/go/libraries/doltcore/schema/encoding/serialization.go +++ b/go/libraries/doltcore/schema/encoding/serialization.go @@ -342,6 +342,8 @@ func serializeSecondaryIndexes(b *fb.Builder, sch schema.Schema, indexes []schem } po := b.EndVector(len(prefixLengths)) + ftInfo := serializeFullTextInfo(b, idx) + serial.IndexStart(b) serial.IndexAddName(b, no) serial.IndexAddComment(b, co) @@ -352,6 +354,8 @@ func serializeSecondaryIndexes(b *fb.Builder, sch schema.Schema, indexes []schem serial.IndexAddSystemDefined(b, !idx.IsUserDefined()) serial.IndexAddPrefixLengths(b, po) serial.IndexAddSpatialKey(b, idx.IsSpatial()) + serial.IndexAddFulltextKey(b, idx.IsFullText()) + serial.IndexAddFulltextInfo(b, ftInfo) offs[i] = serial.IndexEnd(b) } @@ -371,10 +375,12 @@ func deserializeSecondaryIndexes(sch schema.Schema, s *serial.TableSchema) error name := string(idx.Name()) props := schema.IndexProperties{ - IsUnique: idx.UniqueKey(), - IsSpatial: idx.SpatialKey(), - IsUserDefined: !idx.SystemDefined(), - Comment: string(idx.Comment()), + IsUnique: idx.UniqueKey(), + IsSpatial: idx.SpatialKey(), + IsFullText: idx.FulltextKey(), + IsUserDefined: !idx.SystemDefined(), + Comment: string(idx.Comment()), + FullTextProperties: deserializeFullTextInfo(&idx), } tags := make([]uint64, idx.IndexColumnsLength()) @@ -433,6 +439,62 @@ func deserializeChecks(sch schema.Schema, s *serial.TableSchema) error { return nil } +func serializeFullTextInfo(b *fb.Builder, idx schema.Index) fb.UOffsetT { + props := idx.FullTextProperties() + + configTable := b.CreateString(props.ConfigTable) + posTable := b.CreateString(props.PositionTable) + docCountTable := b.CreateString(props.DocCountTable) + globalCountTable := b.CreateString(props.GlobalCountTable) + rowCountTable := b.CreateString(props.RowCountTable) + keyName := b.CreateString(props.KeyName) + + keyPositions := idx.FullTextProperties().KeyPositions + serial.FulltextInfoStartKeyPositionsVector(b, len(keyPositions)) + for j := len(keyPositions) - 1; j >= 0; j-- { + b.PrependUint16(keyPositions[j]) + } + keyPos := b.EndVector(len(keyPositions)) + + serial.FulltextInfoStart(b) + serial.FulltextInfoAddConfigTable(b, configTable) + serial.FulltextInfoAddPositionTable(b, posTable) + serial.FulltextInfoAddDocCountTable(b, docCountTable) + serial.FulltextInfoAddGlobalCountTable(b, globalCountTable) + serial.FulltextInfoAddRowCountTable(b, rowCountTable) + serial.FulltextInfoAddKeyType(b, props.KeyType) + serial.FulltextInfoAddKeyName(b, keyName) + serial.FulltextInfoAddKeyPositions(b, keyPos) + return serial.FulltextInfoEnd(b) +} + +func deserializeFullTextInfo(idx *serial.Index) schema.FullTextProperties { + fulltext := serial.FulltextInfo{} + if idx.FulltextInfo(&fulltext) == nil { + return schema.FullTextProperties{} + } + + var keyPositions []uint16 + keyPositionsLength := fulltext.KeyPositionsLength() + if keyPositionsLength > 0 { + keyPositions = make([]uint16, keyPositionsLength) + for j := range keyPositions { + keyPositions[j] = fulltext.KeyPositions(j) + } + } + + return schema.FullTextProperties{ + ConfigTable: string(fulltext.ConfigTable()), + PositionTable: string(fulltext.PositionTable()), + DocCountTable: string(fulltext.DocCountTable()), + GlobalCountTable: string(fulltext.GlobalCountTable()), + RowCountTable: string(fulltext.RowCountTable()), + KeyType: fulltext.KeyType(), + KeyName: string(fulltext.KeyName()), + KeyPositions: keyPositions, + } +} + func keylessSerialSchema(s *serial.TableSchema) bool { n := s.ColumnsLength() if n < 2 { diff --git a/go/libraries/doltcore/schema/index.go b/go/libraries/doltcore/schema/index.go index f48ed7464e1..9578ac8a849 100644 --- a/go/libraries/doltcore/schema/index.go +++ b/go/libraries/doltcore/schema/index.go @@ -45,6 +45,8 @@ type Index interface { IsUnique() bool // IsSpatial returns whether the given index has the SPATIAL constraint. IsSpatial() bool + // IsFullText returns whether the given index has the FULLTEXT constraint. + IsFullText() bool // IsUserDefined returns whether the given index was created by a user or automatically generated. IsUserDefined() bool // Name returns the name of the index. @@ -58,6 +60,8 @@ type Index interface { ToTableTuple(ctx context.Context, fullKey types.Tuple, format *types.NomsBinFormat) (types.Tuple, error) // PrefixLengths returns the prefix lengths for the index PrefixLengths() []uint16 + // FullTextProperties returns all properties belonging to a Full-Text index. + FullTextProperties() FullTextProperties } var _ Index = (*indexImpl)(nil) @@ -69,9 +73,11 @@ type indexImpl struct { indexColl *indexCollectionImpl isUnique bool isSpatial bool + isFullText bool isUserDefined bool comment string prefixLengths []uint16 + fullTextProps FullTextProperties } func NewIndex(name string, tags, allTags []uint64, indexColl IndexCollection, props IndexProperties) Index { @@ -87,8 +93,10 @@ func NewIndex(name string, tags, allTags []uint64, indexColl IndexCollection, pr indexColl: indexCollImpl, isUnique: props.IsUnique, isSpatial: props.IsSpatial, + isFullText: props.IsFullText, isUserDefined: props.IsUserDefined, comment: props.Comment, + fullTextProps: props.FullTextProperties, } } @@ -196,6 +204,11 @@ func (ix *indexImpl) IsSpatial() bool { return ix.isSpatial } +// IsFullText implements Index. +func (ix *indexImpl) IsFullText() bool { + return ix.isFullText +} + // IsUserDefined implements Index. func (ix *indexImpl) IsUserDefined() bool { return ix.isUserDefined @@ -278,6 +291,11 @@ func (ix *indexImpl) PrefixLengths() []uint16 { return ix.prefixLengths } +// FullTextProperties implements Index. +func (ix *indexImpl) FullTextProperties() FullTextProperties { + return ix.fullTextProps +} + // copy returns an exact copy of the calling index. func (ix *indexImpl) copy() *indexImpl { newIx := *ix @@ -289,5 +307,9 @@ func (ix *indexImpl) copy() *indexImpl { newIx.prefixLengths = make([]uint16, len(ix.prefixLengths)) _ = copy(newIx.prefixLengths, ix.prefixLengths) } + if len(newIx.fullTextProps.KeyPositions) > 0 { + newIx.fullTextProps.KeyPositions = make([]uint16, len(ix.fullTextProps.KeyPositions)) + _ = copy(newIx.fullTextProps.KeyPositions, ix.fullTextProps.KeyPositions) + } return &newIx } diff --git a/go/libraries/doltcore/schema/index_coll.go b/go/libraries/doltcore/schema/index_coll.go index 5cdad68b44c..7ea17deb32a 100644 --- a/go/libraries/doltcore/schema/index_coll.go +++ b/go/libraries/doltcore/schema/index_coll.go @@ -70,13 +70,28 @@ type IndexCollection interface { RenameIndex(oldName, newName string) (Index, error) //SetPks changes the pks or pk ordinals SetPks([]uint64) error + // ContainsFullTextIndex returns whether the collection contains at least one Full-Text index. + ContainsFullTextIndex() bool } type IndexProperties struct { IsUnique bool IsSpatial bool + IsFullText bool IsUserDefined bool Comment string + FullTextProperties +} + +type FullTextProperties struct { + ConfigTable string + PositionTable string + DocCountTable string + GlobalCountTable string + RowCountTable string + KeyType uint8 + KeyName string + KeyPositions []uint16 } type indexCollectionImpl struct { @@ -167,9 +182,11 @@ func (ixc *indexCollectionImpl) AddIndexByColTags(indexName string, tags []uint6 allTags: combineAllTags(tags, ixc.pks), isUnique: props.IsUnique, isSpatial: props.IsSpatial, + isFullText: props.IsFullText, isUserDefined: props.IsUserDefined, comment: props.Comment, prefixLengths: prefixLengths, + fullTextProps: props.FullTextProperties, } ixc.indexes[indexName] = index for _, tag := range tags { @@ -191,9 +208,11 @@ func (ixc *indexCollectionImpl) UnsafeAddIndexByColTags(indexName string, tags [ allTags: combineAllTags(tags, ixc.pks), isUnique: props.IsUnique, isSpatial: props.IsSpatial, + isFullText: props.IsFullText, isUserDefined: props.IsUserDefined, comment: props.Comment, prefixLengths: prefixLengths, + fullTextProps: props.FullTextProperties, } ixc.indexes[indexName] = index for _, tag := range tags { @@ -356,9 +375,11 @@ func (ixc *indexCollectionImpl) Merge(indexes ...Index) { indexColl: ixc, isUnique: index.IsUnique(), isSpatial: index.IsSpatial(), + isFullText: index.IsFullText(), isUserDefined: index.IsUserDefined(), comment: index.Comment(), prefixLengths: index.PrefixLengths(), + fullTextProps: index.FullTextProperties(), } ixc.AddIndex(newIndex) } @@ -462,6 +483,27 @@ func (ixc *indexCollectionImpl) SetPks(tags []uint64) error { return nil } +func (ixc *indexCollectionImpl) ContainsFullTextIndex() bool { + for _, idx := range ixc.indexes { + if idx.isFullText { + return true + } + } + return false +} + +//TODO: maybe delete? +// TableNameSlice returns the table names as a slice, which may be used to easily grab all of the tables using a for loop. +func (props FullTextProperties) TableNameSlice() []string { + return []string{ + props.ConfigTable, + props.PositionTable, + props.DocCountTable, + props.GlobalCountTable, + props.RowCountTable, + } +} + func combineAllTags(tags []uint64, pks []uint64) []uint64 { allTags := make([]uint64, len(tags)) _ = copy(allTags, tags) diff --git a/go/libraries/doltcore/sqle/alterschema.go b/go/libraries/doltcore/sqle/alterschema.go index ab879c8db68..1e23c71a0b2 100755 --- a/go/libraries/doltcore/sqle/alterschema.go +++ b/go/libraries/doltcore/sqle/alterschema.go @@ -265,10 +265,12 @@ func replaceColumnInSchema(sch schema.Schema, oldCol schema.Column, newCol schem tags, index.PrefixLengths(), schema.IndexProperties{ - IsUnique: index.IsUnique(), - IsSpatial: index.IsSpatial(), - IsUserDefined: index.IsUserDefined(), - Comment: index.Comment(), + IsUnique: index.IsUnique(), + IsSpatial: index.IsSpatial(), + IsFullText: index.IsFullText(), + IsUserDefined: index.IsUserDefined(), + Comment: index.Comment(), + FullTextProperties: index.FullTextProperties(), }) if err != nil { return nil, err diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 46b84c228c0..4a4b3b6c543 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -23,6 +23,7 @@ import ( "time" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/fulltext" "github.com/dolthub/go-mysql-server/sql/parse" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/types" @@ -76,6 +77,7 @@ var _ sql.VersionedDatabase = Database{} var _ sql.ViewDatabase = Database{} var _ sql.EventDatabase = Database{} var _ sql.AliasedDatabase = Database{} +var _ fulltext.Database = Database{} type ReadOnlyDatabase struct { Database @@ -630,7 +632,7 @@ func (db Database) getTable(ctx *sql.Context, root *doltdb.RootValue, tableName } if doltdb.IsReadOnlySystemTable(tableName) { table = readonlyTable - } else if doltdb.HasDoltPrefix(tableName) { + } else if doltdb.HasDoltPrefix(tableName) && !doltdb.IsFullTextTable(tableName) { table = &WritableDoltTable{DoltTable: readonlyTable, db: db} } else { table = &AlterableDoltTable{WritableDoltTable{DoltTable: readonlyTable, db: db}} @@ -849,7 +851,7 @@ func (db Database) CreateTable(ctx *sql.Context, tableName string, sch sql.Prima if !dtables.DoltDocsSqlSchema.Equals(sch.Schema) && !dtables.OldDoltDocsSqlSchema.Equals(sch.Schema) { return fmt.Errorf("incorrect schema for dolt_docs table") } - } else if doltdb.HasDoltPrefix(tableName) { + } else if doltdb.HasDoltPrefix(tableName) && !doltdb.IsFullTextTable(tableName) { return ErrReservedTableName.New(tableName) } @@ -881,6 +883,17 @@ func (db Database) CreateIndexedTable(ctx *sql.Context, tableName string, sch sq return db.createIndexedSqlTable(ctx, tableName, sch, idxDef, collation) } +// CreateFulltextTableNames returns a set of names that will be used to create Full-Text pseudo-index tables. +func (db Database) CreateFulltextTableNames(ctx *sql.Context, parentTable string, parentIndexName string) (fulltext.IndexTableNames, error) { + return fulltext.IndexTableNames{ + Config: fmt.Sprintf("dolt_%s_fts_config", parentTable), + Position: fmt.Sprintf("dolt_%s_%s_fts_position", parentTable, parentIndexName), + DocCount: fmt.Sprintf("dolt_%s_%s_fts_doc_count", parentTable, parentIndexName), + GlobalCount: fmt.Sprintf("dolt_%s_%s_fts_global_count", parentTable, parentIndexName), + RowCount: fmt.Sprintf("dolt_%s_%s_fts_row_count", parentTable, parentIndexName), + }, nil +} + // createSqlTable is the private version of CreateTable. It doesn't enforce any table name checks. func (db Database) createSqlTable(ctx *sql.Context, tableName string, sch sql.PrimaryKeySchema, collation sql.CollationID) error { ws, err := db.GetWorkingSet(ctx) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index 9df940ae26b..01965f26303 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -1023,6 +1023,15 @@ func TestForeignKeyBranchesPrepared(t *testing.T) { } } +func TestFulltextIndexes(t *testing.T) { + if !types.IsFormat_DOLT(types.Format_Default) { + t.Skip("FULLTEXT is not supported on the old format") + } + h := newDoltHarness(t) + defer h.Close() + enginetest.TestFulltextIndexes(t, h) +} + func TestCreateCheckConstraints(t *testing.T) { h := newDoltHarness(t) defer h.Close() diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index d3b393b0cca..5e782e708e3 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -158,6 +158,10 @@ func validateIndexConsistency( } func validateKeylessIndex(ctx context.Context, sch schema.Schema, def schema.Index, primary, secondary prolly.Map) error { + // Full-Text indexes do not make use of their internal map, so we may safely skip this check + if def.IsFullText() { + return nil + } secondary = prolly.ConvertToSecondaryKeylessIndex(secondary) idxDesc, _ := secondary.Descriptors() builder := val.NewTupleBuilder(idxDesc) @@ -215,6 +219,10 @@ func validateKeylessIndex(ctx context.Context, sch schema.Schema, def schema.Ind } func validatePkIndex(ctx context.Context, sch schema.Schema, def schema.Index, primary, secondary prolly.Map) error { + // Full-Text indexes do not make use of their internal map, so we may safely skip this check + if def.IsFullText() { + return nil + } // secondary indexes have empty values idxDesc, _ := secondary.Descriptors() builder := val.NewTupleBuilder(idxDesc) diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 95d32ff7201..318ee55325a 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -22,6 +22,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" + "github.com/dolthub/go-mysql-server/sql/fulltext" sqltypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" @@ -51,6 +52,7 @@ type DoltTableable interface { type DoltIndex interface { sql.FilteredIndex sql.OrderedIndex + fulltext.Index Schema() schema.Schema IndexSchema() schema.Schema Format() *types.NomsBinFormat @@ -373,6 +375,7 @@ func getSecondaryIndex(ctx context.Context, db, tbl string, t *doltdb.Table, sch tableSch: sch, unique: idx.IsUnique(), spatial: idx.IsSpatial(), + fulltext: idx.IsFullText(), isPk: false, comment: idx.Comment(), vrw: vrw, @@ -382,6 +385,38 @@ func getSecondaryIndex(ctx context.Context, db, tbl string, t *doltdb.Table, sch constrainedToLookupExpression: true, doltBinFormat: types.IsFormat_DOLT(vrw.Format()), prefixLengths: idx.PrefixLengths(), + fullTextProps: idx.FullTextProperties(), + }, nil +} + +// ConvertFullTextToSql converts a given Full-Text schema.Index into a sql.Index. As we do not need to write to a +// Full-Text index, we can omit all such fields. This must not be used in any other circumstance. +func ConvertFullTextToSql(ctx context.Context, db, tbl string, sch schema.Schema, idx schema.Index) (sql.Index, error) { + cols := make([]schema.Column, idx.Count()) + for i, tag := range idx.IndexedColumnTags() { + cols[i], _ = idx.GetColumn(tag) + } + + return &doltIndex{ + id: idx.Name(), + tblName: tbl, + dbName: db, + columns: cols, + indexSch: idx.Schema(), + tableSch: sch, + unique: idx.IsUnique(), + spatial: idx.IsSpatial(), + fulltext: idx.IsFullText(), + isPk: false, + comment: idx.Comment(), + vrw: nil, + ns: nil, + keyBld: nil, + order: sql.IndexOrderAsc, + constrainedToLookupExpression: true, + doltBinFormat: true, + prefixLengths: idx.PrefixLengths(), + fullTextProps: idx.FullTextProperties(), }, nil } @@ -502,6 +537,7 @@ type doltIndex struct { tableSch schema.Schema unique bool spatial bool + fulltext bool isPk bool comment string order sql.IndexOrder @@ -516,6 +552,7 @@ type doltIndex struct { doltBinFormat bool prefixLengths []uint16 + fullTextProps schema.FullTextProperties } var _ DoltIndex = (*doltIndex)(nil) @@ -846,6 +883,11 @@ func (di *doltIndex) IsSpatial() bool { return di.spatial } +// IsFullText implements sql.Index +func (di *doltIndex) IsFullText() bool { + return di.fulltext +} + // IsPrimaryKey implements DoltIndex. func (di *doltIndex) IsPrimaryKey() bool { return di.isPk @@ -890,6 +932,34 @@ func (di *doltIndex) Format() *types.NomsBinFormat { return di.vrw.Format() } +// FullTextTableNames implements sql.Index +func (di *doltIndex) FullTextTableNames(ctx *sql.Context) (fulltext.IndexTableNames, error) { + return fulltext.IndexTableNames{ + Config: di.fullTextProps.ConfigTable, + Position: di.fullTextProps.PositionTable, + DocCount: di.fullTextProps.DocCountTable, + GlobalCount: di.fullTextProps.GlobalCountTable, + RowCount: di.fullTextProps.RowCountTable, + }, nil +} + +// FullTextKeyColumns implements sql.Index +func (di *doltIndex) FullTextKeyColumns(ctx *sql.Context) (fulltext.KeyColumns, error) { + var positions []int + if len(di.fullTextProps.KeyPositions) > 0 { + positions = make([]int, len(di.fullTextProps.KeyPositions)) + for i := range positions { + positions[i] = int(di.fullTextProps.KeyPositions[i]) + } + } + + return fulltext.KeyColumns{ + Type: fulltext.KeyType(di.fullTextProps.KeyType), + Name: di.fullTextProps.KeyName, + Positions: positions, + }, nil +} + // keysToTuple returns a tuple that indicates the starting point for an index. The empty tuple will cause the index to // start at the very beginning. func (di *doltIndex) keysToTuple(ctx *sql.Context, keys []interface{}) (types.Tuple, error) { diff --git a/go/libraries/doltcore/sqle/sqlutil/schema.go b/go/libraries/doltcore/sqle/sqlutil/schema.go index 30b04f88862..2ad6f634639 100644 --- a/go/libraries/doltcore/sqle/sqlutil/schema.go +++ b/go/libraries/doltcore/sqle/sqlutil/schema.go @@ -71,9 +71,10 @@ func ParseCreateTableStatement(ctx context.Context, root *doltdb.RootValue, quer prefixes = append(prefixes, uint16(c.Length)) } props := schema.IndexProperties{ - IsUnique: idx.IsUnique(), - IsSpatial: idx.IsSpatial(), - Comment: idx.Comment, + IsUnique: idx.IsUnique(), + IsSpatial: idx.IsSpatial(), + IsFullText: idx.IsFullText(), + Comment: idx.Comment, } name := getIndexName(idx) _, err = sch.Indexes().AddIndexByColNames(name, idx.ColumnNames(), prefixes, props) diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index ad63c019704..67ac928dd47 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -29,6 +29,7 @@ import ( "sync" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/fulltext" sqltypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" @@ -462,7 +463,6 @@ func partitionRows(ctx *sql.Context, t *doltdb.Table, sqlSch sql.Schema, projCol type WritableDoltTable struct { *DoltTable db Database - ed writer.TableWriter } var _ doltTableInterface = (*WritableDoltTable)(nil) @@ -491,7 +491,6 @@ func (t *WritableDoltTable) WithProjections(colNames []string) sql.Table { return &WritableDoltTable{ DoltTable: t.DoltTable.WithProjections(colNames).(*DoltTable), db: t.db, - ed: t.ed, } } @@ -517,12 +516,90 @@ func (t *WritableDoltTable) getTableEditor(ctx *sql.Context) (ed writer.TableWri setter := ds.SetRoot ed, err = state.WriteSession().GetTableWriter(ctx, t.tableName, t.db.RevisionQualifiedName(), setter) - if err != nil { return nil, err } - return ed, nil + if t.sch.Indexes().ContainsFullTextIndex() { + ftEditor, err := t.getFullTextEditor(ctx) + if err != nil { + return nil, err + } + multiEditor, err := fulltext.CreateMultiTableEditor(ctx, ed, ftEditor) + if err != nil { + return nil, err + } + return multiEditor.(writer.TableWriter), nil + } else { + return ed, nil + } +} + +// getFullTextEditor gathers all pseudo-index tables for a Full-Text index and returns an editor that will write +// to all of them. This assumes that there are Full-Text indexes in the schema. +func (t *WritableDoltTable) getFullTextEditor(ctx *sql.Context) (fulltext.TableEditor, error) { + workingRoot, err := t.workingRoot(ctx) + if err != nil { + return fulltext.TableEditor{}, err + } + var configTable fulltext.EditableTable + var sets []fulltext.TableSet + for _, idx := range t.sch.Indexes().AllIndexes() { + if !idx.IsFullText() { + continue + } + props := idx.FullTextProperties() + // We only load the config table once since it's shared by all indexes + if configTable == nil { + tbl, ok, err := t.db.getTable(ctx, workingRoot, props.ConfigTable) + if err != nil { + return fulltext.TableEditor{}, err + } else if !ok { + return fulltext.TableEditor{}, fmt.Errorf("missing Full-Text table: %s", props.ConfigTable) + } + configTable = tbl.(fulltext.EditableTable) + } + // Load the rest of the tables + positionTable, ok, err := t.db.getTable(ctx, workingRoot, props.PositionTable) + if err != nil { + return fulltext.TableEditor{}, err + } else if !ok { + return fulltext.TableEditor{}, fmt.Errorf("missing Full-Text table: %s", props.PositionTable) + } + docCountTable, ok, err := t.db.getTable(ctx, workingRoot, props.DocCountTable) + if err != nil { + return fulltext.TableEditor{}, err + } else if !ok { + return fulltext.TableEditor{}, fmt.Errorf("missing Full-Text table: %s", props.DocCountTable) + } + globalCountTable, ok, err := t.db.getTable(ctx, workingRoot, props.GlobalCountTable) + if err != nil { + return fulltext.TableEditor{}, err + } else if !ok { + return fulltext.TableEditor{}, fmt.Errorf("missing Full-Text table: %s", props.GlobalCountTable) + } + rowCountTable, ok, err := t.db.getTable(ctx, workingRoot, props.RowCountTable) + if err != nil { + return fulltext.TableEditor{}, err + } else if !ok { + return fulltext.TableEditor{}, fmt.Errorf("missing Full-Text table: %s", props.RowCountTable) + } + // Convert the index into a sql.Index + sqlIdx, err := index.ConvertFullTextToSql(ctx, t.db.RevisionQualifiedName(), t.tableName, t.sch, idx) + if err != nil { + return fulltext.TableEditor{}, err + } + + sets = append(sets, fulltext.TableSet{ + Index: sqlIdx.(fulltext.Index), + Position: positionTable.(fulltext.EditableTable), + DocCount: docCountTable.(fulltext.EditableTable), + GlobalCount: globalCountTable.(fulltext.EditableTable), + RowCount: rowCountTable.(fulltext.EditableTable), + }) + } + + return fulltext.CreateEditor(ctx, t, configTable, sets...) } // Deleter implements sql.DeletableTable @@ -730,7 +807,7 @@ func (t *WritableDoltTable) GetNextAutoIncrementValue(ctx *sql.Context, potentia return 0, err } - return ed.GetNextAutoIncrementValue(ctx, potentialVal) + return ed.(sql.AutoIncrementGetter).GetNextAutoIncrementValue(ctx, potentialVal) } func (t *DoltTable) GetChecks(ctx *sql.Context) ([]sql.CheckDefinition, error) { @@ -1076,6 +1153,7 @@ type doltAlterableTableInterface interface { sql.PrimaryKeyAlterableTable sql.ProjectedTable sql.CollationAlterableTable + fulltext.IndexAlterableTable } var _ doltAlterableTableInterface = (*AlterableDoltTable)(nil) @@ -1324,10 +1402,12 @@ func (t *AlterableDoltTable) RewriteInserter( colNames, prefixLengths, schema.IndexProperties{ - IsUnique: index.IsUnique(), - IsSpatial: index.IsSpatial(), - IsUserDefined: index.IsUserDefined(), - Comment: index.Comment(), + IsUnique: index.IsUnique(), + IsSpatial: index.IsSpatial(), + IsFullText: index.IsFullText(), + IsUserDefined: index.IsUserDefined(), + Comment: index.Comment(), + FullTextProperties: index.FullTextProperties(), }) } } else { @@ -1781,70 +1861,7 @@ func (t *AlterableDoltTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) err return fmt.Errorf("only the following types of index constraints are supported: none, unique, spatial") } - columns := make([]string, len(idx.Columns)) - for i, indexCol := range idx.Columns { - columns[i] = indexCol.Name - } - - table, err := t.DoltTable.DoltTable(ctx) - if err != nil { - return err - } - - ret, err := creation.CreateIndex( - ctx, - table, - idx.Name, - columns, - allocatePrefixLengths(idx.Columns), - idx.Constraint == sql.IndexConstraint_Unique, - idx.Constraint == sql.IndexConstraint_Spatial, - true, - idx.Comment, - t.opts, - ) - if err != nil { - return err - } - root, err := t.getRoot(ctx) - if err != nil { - return err - } - if ret.OldIndex != nil && ret.OldIndex != ret.NewIndex { // old index was replaced, so we update foreign keys - fkc, err := root.GetForeignKeyCollection(ctx) - if err != nil { - return err - } - for _, fk := range fkc.AllKeys() { - newFk := fk - if t.tableName == fk.TableName && fk.TableIndex == ret.OldIndex.Name() { - newFk.TableIndex = ret.NewIndex.Name() - } - if t.tableName == fk.ReferencedTableName && fk.ReferencedTableIndex == ret.OldIndex.Name() { - newFk.ReferencedTableIndex = ret.NewIndex.Name() - } - fkc.RemoveKeys(fk) - err = fkc.AddKeys(newFk) - if err != nil { - return err - } - } - root, err = root.PutForeignKeyCollection(ctx, fkc) - if err != nil { - return err - } - } - newRoot, err := root.PutTable(ctx, t.tableName, ret.NewTable) - if err != nil { - return err - } - - err = t.setRoot(ctx, newRoot) - - if err != nil { - return err - } - return t.updateFromRoot(ctx, newRoot) + return t.createIndex(ctx, idx, fulltext.KeyColumns{}, fulltext.IndexTableNames{}) } // DropIndex implements sql.IndexAlterableTable @@ -1916,6 +1933,110 @@ func (t *AlterableDoltTable) RenameIndex(ctx *sql.Context, fromIndexName string, return t.updateFromRoot(ctx, newRoot) } +// CreateFulltextIndex implements fulltext.IndexAlterableTable +func (t *AlterableDoltTable) CreateFulltextIndex(ctx *sql.Context, idx sql.IndexDef, keyCols fulltext.KeyColumns, tableNames fulltext.IndexTableNames) error { + if !types.IsFormat_DOLT(t.Format()) { + return fmt.Errorf("FULLTEXT is not supported on our old format") + } + if err := dsess.CheckAccessForDb(ctx, t.db, branch_control.Permissions_Write); err != nil { + return err + } + if idx.Constraint != sql.IndexConstraint_Fulltext { + return fmt.Errorf("attempted to create non-FullText index through FullText interface") + } + + return t.createIndex(ctx, idx, keyCols, tableNames) +} + +// createIndex handles the common functionality between CreateIndex and CreateFulltextIndex. +func (t *AlterableDoltTable) createIndex(ctx *sql.Context, idx sql.IndexDef, keyCols fulltext.KeyColumns, tableNames fulltext.IndexTableNames) error { + columns := make([]string, len(idx.Columns)) + for i, indexCol := range idx.Columns { + columns[i] = indexCol.Name + } + + table, err := t.DoltTable.DoltTable(ctx) + if err != nil { + return err + } + + var keyPositions []uint16 + if len(keyCols.Positions) > 0 { + keyPositions = make([]uint16, len(keyCols.Positions)) + for i := range keyPositions { + keyPositions[i] = uint16(keyCols.Positions[i]) + } + } + + ret, err := creation.CreateIndex( + ctx, + table, + idx.Name, + columns, + allocatePrefixLengths(idx.Columns), + schema.IndexProperties{ + IsUnique: idx.Constraint == sql.IndexConstraint_Unique, + IsSpatial: idx.Constraint == sql.IndexConstraint_Spatial, + IsFullText: idx.Constraint == sql.IndexConstraint_Fulltext, + IsUserDefined: true, + Comment: idx.Comment, + FullTextProperties: schema.FullTextProperties{ + ConfigTable: tableNames.Config, + PositionTable: tableNames.Position, + DocCountTable: tableNames.DocCount, + GlobalCountTable: tableNames.GlobalCount, + RowCountTable: tableNames.RowCount, + KeyType: uint8(keyCols.Type), + KeyName: keyCols.Name, + KeyPositions: keyPositions, + }, + }, + t.opts, + ) + if err != nil { + return err + } + root, err := t.getRoot(ctx) + if err != nil { + return err + } + if ret.OldIndex != nil && ret.OldIndex != ret.NewIndex { // old index was replaced, so we update foreign keys + fkc, err := root.GetForeignKeyCollection(ctx) + if err != nil { + return err + } + for _, fk := range fkc.AllKeys() { + newFk := fk + if t.tableName == fk.TableName && fk.TableIndex == ret.OldIndex.Name() { + newFk.TableIndex = ret.NewIndex.Name() + } + if t.tableName == fk.ReferencedTableName && fk.ReferencedTableIndex == ret.OldIndex.Name() { + newFk.ReferencedTableIndex = ret.NewIndex.Name() + } + fkc.RemoveKeys(fk) + err = fkc.AddKeys(newFk) + if err != nil { + return err + } + } + root, err = root.PutForeignKeyCollection(ctx, fkc) + if err != nil { + return err + } + } + newRoot, err := root.PutTable(ctx, t.tableName, ret.NewTable) + if err != nil { + return err + } + + err = t.setRoot(ctx, newRoot) + + if err != nil { + return err + } + return t.updateFromRoot(ctx, newRoot) +} + // createForeignKey creates a doltdb.ForeignKey from a sql.ForeignKeyConstraint func (t *AlterableDoltTable) createForeignKey( ctx *sql.Context, @@ -2173,10 +2294,13 @@ func (t *AlterableDoltTable) CreateIndexForForeignKey(ctx *sql.Context, idx sql. idx.Name, columns, allocatePrefixLengths(idx.Columns), - idx.Constraint == sql.IndexConstraint_Unique, - idx.Constraint == sql.IndexConstraint_Spatial, - false, - "", + schema.IndexProperties{ + IsUnique: idx.Constraint == sql.IndexConstraint_Unique, + IsSpatial: idx.Constraint == sql.IndexConstraint_Spatial, + IsFullText: idx.Constraint == sql.IndexConstraint_Fulltext, + IsUserDefined: false, + Comment: "", + }, t.opts, ) if err != nil { diff --git a/go/libraries/doltcore/sqle/temp_table.go b/go/libraries/doltcore/sqle/temp_table.go index 7d2d7810a89..0977b0bac37 100644 --- a/go/libraries/doltcore/sqle/temp_table.go +++ b/go/libraries/doltcore/sqle/temp_table.go @@ -276,10 +276,13 @@ func (t *TempTable) CreateIndex(ctx *sql.Context, idx sql.IndexDef) error { idx.Name, cols, allocatePrefixLengths(idx.Columns), - idx.Constraint == sql.IndexConstraint_Unique, - idx.Constraint == sql.IndexConstraint_Spatial, - true, - idx.Comment, + schema.IndexProperties{ + IsUnique: idx.Constraint == sql.IndexConstraint_Unique, + IsSpatial: idx.Constraint == sql.IndexConstraint_Spatial, + IsFullText: idx.Constraint == sql.IndexConstraint_Fulltext, + IsUserDefined: true, + Comment: idx.Comment, + }, t.opts, ) if err != nil { diff --git a/go/libraries/doltcore/sqle/writer/noms_table_writer.go b/go/libraries/doltcore/sqle/writer/noms_table_writer.go index 9ecac78afce..974177553e1 100644 --- a/go/libraries/doltcore/sqle/writer/noms_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/noms_table_writer.go @@ -30,7 +30,7 @@ import ( type TableWriter interface { sql.TableEditor sql.ForeignKeyEditor - sql.AutoIncrementEditor + sql.AutoIncrementSetter } // SessionRootSetter sets the root value for the session. @@ -63,6 +63,7 @@ type nomsTableWriter struct { } var _ TableWriter = &nomsTableWriter{} +var _ sql.AutoIncrementGetter = &nomsTableWriter{} func (te *nomsTableWriter) duplicateKeyErrFunc(keyString, indexName string, k, v types.Tuple, isPk bool) error { oldRow, err := te.kvToSQLRow.ConvertKVTuplesToSqlRow(k, v) diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index 16e300ed7b9..0c58e01dd7d 100644 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -54,6 +54,7 @@ type prollyTableWriter struct { } var _ TableWriter = &prollyTableWriter{} +var _ sql.AutoIncrementGetter = &prollyTableWriter{} func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch sql.Schema, sch schema.Schema) (map[string]indexWriter, error) { s, err := t.GetIndexSet(ctx) @@ -66,6 +67,9 @@ func getSecondaryProllyIndexWriters(ctx context.Context, t *doltdb.Table, sqlSch writers := make(map[string]indexWriter) for _, def := range definitions { + if def.IsFullText() { + continue + } defName := def.Name() idxRows, err := s.GetIndex(ctx, sch, defName) if err != nil { @@ -104,6 +108,9 @@ func getSecondaryKeylessProllyWriters(ctx context.Context, t *doltdb.Table, sqlS writers := make(map[string]indexWriter) for _, def := range definitions { + if def.IsFullText() { + continue + } defName := def.Name() idxRows, err := s.GetIndex(ctx, sch, defName) if err != nil { diff --git a/go/libraries/doltcore/table/editor/creation/index.go b/go/libraries/doltcore/table/editor/creation/index.go index 2b91257ce45..32f584fd1d5 100644 --- a/go/libraries/doltcore/table/editor/creation/index.go +++ b/go/libraries/doltcore/table/editor/creation/index.go @@ -48,10 +48,7 @@ func CreateIndex( indexName string, columns []string, prefixLengths []uint16, - isUnique bool, - isSpatial bool, - isUserDefined bool, - comment string, + props schema.IndexProperties, opts editor.Options, ) (*CreateIndexReturn, error) { sch, err := table.GetSchema(ctx) @@ -102,12 +99,7 @@ func CreateIndex( indexName, realColNames, prefixLengths, - schema.IndexProperties{ - IsUnique: isUnique, - IsSpatial: isSpatial, - IsUserDefined: isUserDefined, - Comment: comment, - }, + props, ) if err != nil { return nil, err diff --git a/go/serial/schema.fbs b/go/serial/schema.fbs index d2cfc6a8ef1..659a2c24cb8 100644 --- a/go/serial/schema.fbs +++ b/go/serial/schema.fbs @@ -91,6 +91,21 @@ table Index { prefix_lengths:[uint16]; spatial_key:bool; + + // fulltext information + fulltext_key:bool; + fulltext_info:FulltextInfo; +} + +table FulltextInfo { + config_table:string; + position_table:string; + doc_count_table:string; + global_count_table:string; + row_count_table:string; + key_type:uint8; + key_name:string; + key_positions:[uint16]; } table CheckConstraint { diff --git a/integration-tests/bats/fulltext.bats b/integration-tests/bats/fulltext.bats new file mode 100644 index 00000000000..184648af43d --- /dev/null +++ b/integration-tests/bats/fulltext.bats @@ -0,0 +1,72 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common +} + +teardown() { + assert_feature_version + teardown_common +} + +@test "fulltext: basic persistence checking" { + dolt sql -q "CREATE TABLE test (pk1 BIGINT UNSIGNED, pk2 BIGINT UNSIGNED, v1 VARCHAR(200), v2 VARCHAR(200), PRIMARY KEY (pk1, pk2), FULLTEXT idx (v1, v2));" + dolt sql -q "INSERT INTO test VALUES (1, 1, 'abc', 'def pqr'), (2, 1, 'ghi', 'jkl'), (3, 1, 'mno', 'mno'), (4, 1, 'stu vwx', 'xyz zyx yzx'), (5, 1, 'ghs', 'mno shg');" + run dolt sql -q "SELECT v2 FROM test WHERE MATCH(v2, v1) AGAINST ('jkl');" -r=json + [[ "$output" =~ "{\"rows\": [{\"v2\":\"jkl\"}]}" ]] || false +} + +@test "fulltext: basic merge" { + dolt sql -q "CREATE TABLE test (pk BIGINT UNSIGNED PRIMARY KEY, v1 VARCHAR(200), v2 VARCHAR(200), FULLTEXT idx (v1, v2));" + dolt sql -q "INSERT INTO test VALUES (1, 'abc', 'def pqr'), (2, 'ghi', 'jkl'), (3, 'mno', 'mno'), (4, 'stu vwx', 'xyz zyx yzx'), (5, 'ghs', 'mno shg');" + + run dolt sql -q "SELECT * FROM dolt_test_idx_fts_global_count;" + [[ "$output" =~ "| word | global_count |" ]] || false + [[ "$output" =~ "| abc | 1 |" ]] || false + [[ "$output" =~ "| def | 1 |" ]] || false + [[ "$output" =~ "| ghi | 1 |" ]] || false + [[ "$output" =~ "| ghs | 1 |" ]] || false + [[ "$output" =~ "| jkl | 1 |" ]] || false + [[ "$output" =~ "| mno | 2 |" ]] || false + [[ "$output" =~ "| pqr | 1 |" ]] || false + [[ "$output" =~ "| shg | 1 |" ]] || false + [[ "$output" =~ "| stu | 1 |" ]] || false + [[ "$output" =~ "| vwx | 1 |" ]] || false + [[ "$output" =~ "| xyz | 1 |" ]] || false + [[ "$output" =~ "| yzx | 1 |" ]] || false + [[ "$output" =~ "| zyx | 1 |" ]] || false + + dolt add -A + dolt commit -m "Initial commit" + dolt branch other + + dolt sql -q "DELETE FROM test WHERE pk = 3;" + dolt add -A + dolt commit -m "Main commit" + + dolt checkout other + dolt sql -q "INSERT INTO test VALUES (6, 'jak', 'mno'), (7, 'mno', 'bot');" + dolt add -A + dolt commit -m "Other commit" + + dolt checkout main + dolt merge other + run dolt sql -q "SELECT * FROM dolt_test_idx_fts_global_count;" + [[ "$output" =~ "| word | global_count |" ]] || false + [[ "$output" =~ "| abc | 1 |" ]] || false + [[ "$output" =~ "| bot | 1 |" ]] || false + [[ "$output" =~ "| def | 1 |" ]] || false + [[ "$output" =~ "| ghi | 1 |" ]] || false + [[ "$output" =~ "| ghs | 1 |" ]] || false + [[ "$output" =~ "| jak | 1 |" ]] || false + [[ "$output" =~ "| jkl | 1 |" ]] || false + [[ "$output" =~ "| mno | 3 |" ]] || false + [[ "$output" =~ "| pqr | 1 |" ]] || false + [[ "$output" =~ "| shg | 1 |" ]] || false + [[ "$output" =~ "| stu | 1 |" ]] || false + [[ "$output" =~ "| vwx | 1 |" ]] || false + [[ "$output" =~ "| xyz | 1 |" ]] || false + [[ "$output" =~ "| yzx | 1 |" ]] || false + [[ "$output" =~ "| zyx | 1 |" ]] || false +}