From a55612521504a99ef8c0f2968f91a09ba8f2eab7 Mon Sep 17 00:00:00 2001 From: Sander Pick Date: Wed, 27 May 2020 16:15:35 -0700 Subject: [PATCH] Collection and index updates and fixes (#362) * db: cleanup indexing, add update collection method Signed-off-by: Sander Pick * db: adds delete collection, more index cleanup Signed-off-by: Sander Pick * db: add update collection api method and more tests Signed-off-by: Sander Pick * db: add get indexes method to api Signed-off-by: Sander Pick * db: ensure unique indexes can be unique Signed-off-by: Sander Pick * db: use token opt in collection methods Signed-off-by: Sander Pick * db: use two locks Signed-off-by: Sander Pick * db: check index path on hydration Signed-off-by: Sander Pick * review: address feedback Signed-off-by: Sander Pick --- api/client/client.go | 78 ++- api/client/client_test.go | 161 ++++- api/pb/threads.pb.go | 646 ++++++++++++++---- api/pb/threads.proto | 36 +- api/service.go | 96 ++- core/db/db.go | 21 +- db/bench_test.go | 10 +- db/collection.go | 228 +++---- db/collection_test.go | 374 ++++++++-- db/common.go | 2 +- db/db.go | 298 ++++---- db/db_test.go | 10 +- db/dispatcher.go | 6 +- db/encode.go | 8 +- db/index.go | 196 ++++-- db/listeners.go | 4 +- db/manager.go | 38 +- db/manager_test.go | 8 +- db/options.go | 128 ++-- db/query.go | 18 +- db/query_test.go | 2 +- db/testutils_test.go | 4 +- examples/e2e_counter/reader.go | 2 +- examples/e2e_counter/writer.go | 2 +- examples/local_eventstore/books/main.go | 2 +- examples/local_eventstore/json-books/main.go | 2 +- integrationtests/foldersync/client.go | 4 +- .../foldersync/foldersync_test.go | 8 +- jsonpatcher/jsonpatcher.go | 7 +- 29 files changed, 1694 insertions(+), 705 deletions(-) diff --git a/api/client/client.go b/api/client/client.go index ffd788a7..dd0d2f7a 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -152,8 +152,8 @@ func (c *Client) GetToken(ctx context.Context, identity thread.Identity) (tok th } // NewDB creates a new DB with ID. -func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManagedDBOption) error { - args := &db.NewManagedDBOptions{} +func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManagedOption) error { + args := &db.NewManagedOptions{} for _, opt := range opts { opt(args) } @@ -174,8 +174,8 @@ func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManage } // NewDBFromAddr creates a new DB with address and keys. -func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey thread.Key, opts ...db.NewManagedDBOption) error { - args := &db.NewManagedDBOptions{} +func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey thread.Key, opts ...db.NewManagedOption) error { + args := &db.NewManagedOptions{} for _, opt := range opts { opt(args) } @@ -197,9 +197,9 @@ func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey t } func collectionConfigToPb(c db.CollectionConfig) (*pb.CollectionConfig, error) { - idx := make([]*pb.CollectionConfig_IndexConfig, len(c.Indexes)) + idx := make([]*pb.Index, len(c.Indexes)) for i, index := range c.Indexes { - idx[i] = &pb.CollectionConfig_IndexConfig{ + idx[i] = &pb.Index{ Path: index.Path, Unique: index.Unique, } @@ -216,8 +216,8 @@ func collectionConfigToPb(c db.CollectionConfig) (*pb.CollectionConfig, error) { } // GetDBInfo retrives db addresses and keys. -func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.ManagedDBOption) ([]ma.Multiaddr, thread.Key, error) { - args := &db.ManagedDBOptions{} +func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.ManagedOption) ([]ma.Multiaddr, thread.Key, error) { + args := &db.ManagedOptions{} for _, opt := range opts { opt(args) } @@ -244,8 +244,8 @@ func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.Manag } // DeleteDB deletes a db. -func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.ManagedDBOption) error { - args := &db.ManagedDBOptions{} +func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.ManagedOption) error { + args := &db.ManagedOptions{} for _, opt := range opts { opt(args) } @@ -257,9 +257,8 @@ func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.Manage } // NewCollection creates a new collection. -// @todo: This should take some thread auth, but collections currently do not involve a thread. -func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedDBOption) error { - args := &db.ManagedDBOptions{} +func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedOption) error { + args := &db.ManagedOptions{} for _, opt := range opts { opt(args) } @@ -274,6 +273,59 @@ func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.Co return err } +// UpdateCollection updates an existing collection. +func (c *Client) UpdateCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedOption) error { + args := &db.ManagedOptions{} + for _, opt := range opts { + opt(args) + } + cc, err := collectionConfigToPb(config) + if err != nil { + return err + } + _, err = c.c.UpdateCollection(ctx, &pb.UpdateCollectionRequest{ + DbID: dbID.Bytes(), + Config: cc, + }) + return err +} + +// DeleteCollection deletes a collection. +func (c *Client) DeleteCollection(ctx context.Context, dbID thread.ID, name string, opts ...db.ManagedOption) error { + args := &db.ManagedOptions{} + for _, opt := range opts { + opt(args) + } + _, err := c.c.DeleteCollection(ctx, &pb.DeleteCollectionRequest{ + DbID: dbID.Bytes(), + Name: name, + }) + return err +} + +// GetCollectionIndexes returns an existing collection's indexes. +func (c *Client) GetCollectionIndexes(ctx context.Context, dbID thread.ID, name string, opts ...db.ManagedOption) ([]db.Index, error) { + args := &db.ManagedOptions{} + for _, opt := range opts { + opt(args) + } + resp, err := c.c.GetCollectionIndexes(ctx, &pb.GetCollectionIndexesRequest{ + DbID: dbID.Bytes(), + Name: name, + }) + if err != nil { + return nil, err + } + indexes := make([]db.Index, len(resp.Indexes)) + for i, index := range resp.Indexes { + indexes[i] = db.Index{ + Path: index.Path, + Unique: index.Unique, + } + } + return indexes, nil +} + // Create creates new instances of objects. func (c *Client) Create(ctx context.Context, dbID thread.ID, collectionName string, instances Instances, opts ...db.TxnOption) ([]string, error) { args := &db.TxnOptions{} diff --git a/api/client/client_test.go b/api/client/client_test.go index c97aa204..62fea97b 100644 --- a/api/client/client_test.go +++ b/api/client/client_test.go @@ -136,6 +136,83 @@ func TestClient_NewCollection(t *testing.T) { }) } +func TestClient_UpdateCollection(t *testing.T) { + t.Parallel() + client, done := setup(t) + defer done() + + id := thread.NewIDV1(thread.Raw, 32) + err := client.NewDB(context.Background(), id) + checkErr(t, err) + err = client.NewCollection(context.Background(), id, db.CollectionConfig{Name: collectionName, Schema: util.SchemaFromSchemaString(schema)}) + checkErr(t, err) + + t.Run("test update collection", func(t *testing.T) { + err = client.UpdateCollection(context.Background(), id, db.CollectionConfig{ + Name: collectionName, + Schema: util.SchemaFromSchemaString(schema2), + Indexes: []db.Index{{ + Path: "age", + Unique: false, + }}, + }) + if err != nil { + t.Fatalf("failed to update collection: %v", err) + } + _, err = client.Create(context.Background(), id, collectionName, Instances{createPerson2()}) + checkErr(t, err) + }) +} + +func TestClient_DeleteCollection(t *testing.T) { + t.Parallel() + client, done := setup(t) + defer done() + + t.Run("test delete collection", func(t *testing.T) { + id := thread.NewIDV1(thread.Raw, 32) + err := client.NewDB(context.Background(), id) + checkErr(t, err) + err = client.NewCollection(context.Background(), id, db.CollectionConfig{Name: collectionName, Schema: util.SchemaFromSchemaString(schema)}) + checkErr(t, err) + + err = client.DeleteCollection(context.Background(), id, collectionName) + if err != nil { + t.Fatalf("failed to delete collection: %v", err) + } + _, err = client.Create(context.Background(), id, collectionName, Instances{createPerson()}) + if err == nil { + t.Fatal("failed to delete collection") + } + }) +} + +func TestClient_GetCollectionIndexes(t *testing.T) { + t.Parallel() + client, done := setup(t) + defer done() + + t.Run("test get collection indexes", func(t *testing.T) { + id := thread.NewIDV1(thread.Raw, 32) + err := client.NewDB(context.Background(), id) + checkErr(t, err) + err = client.NewCollection(context.Background(), id, db.CollectionConfig{ + Name: collectionName, + Schema: util.SchemaFromSchemaString(schema), + Indexes: []db.Index{{ + Path: "lastName", + Unique: true, + }}, + }) + checkErr(t, err) + indexes, err := client.GetCollectionIndexes(context.Background(), id, collectionName) + checkErr(t, err) + if len(indexes) != 2 { + t.Fatalf("expected 2 indexes, but got %v", len(indexes)) + } + }) +} + func TestClient_Create(t *testing.T) { t.Parallel() client, done := setup(t) @@ -302,7 +379,7 @@ func TestClient_FindWithIndex(t *testing.T) { err = client.NewCollection(context.Background(), id, db.CollectionConfig{ Name: collectionName, Schema: util.SchemaFromSchemaString(schema), - Indexes: []db.IndexConfig{{ + Indexes: []db.Index{{ Path: "lastName", Unique: true, }}, @@ -698,41 +775,69 @@ func createIdentity(t *testing.T) thread.Identity { func createPerson() *Person { return &Person{ - ID: "", FirstName: "Adam", LastName: "Doe", Age: 21, } } +func createPerson2() *Person2 { + return &Person2{ + FullName: "Adam Doe", + Age: 21, + } +} + const ( collectionName = "Person" schema = `{ - "$id": "https://example.com/person.schema.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "` + collectionName + `", - "type": "object", - "properties": { - "_id": { - "type": "string", - "description": "The instance's id." - }, - "firstName": { - "type": "string", - "description": "The person's first name." - }, - "lastName": { - "type": "string", - "description": "The person's last name." - }, - "age": { - "description": "Age in years which must be equal to or greater than zero.", - "type": "integer", - "minimum": 0 + "$id": "https://example.com/person.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "` + collectionName + `", + "type": "object", + "properties": { + "_id": { + "type": "string", + "description": "The instance's id." + }, + "firstName": { + "type": "string", + "description": "The person's first name." + }, + "lastName": { + "type": "string", + "description": "The person's last name." + }, + "age": { + "description": "Age in years which must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + } } - } -}` + }` + + schema2 = `{ + "$id": "https://example.com/person.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "` + collectionName + `", + "type": "object", + "properties": { + "_id": { + "type": "string", + "description": "The instance's id." + }, + "fullName": { + "type": "string", + "description": "The person's full name." + }, + "age": { + "description": "Age in years which must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + } + } + }` ) type Person struct { @@ -742,6 +847,12 @@ type Person struct { Age int `json:"age,omitempty"` } +type Person2 struct { + ID string `json:"_id"` + FullName string `json:"fullName,omitempty"` + Age int `json:"age,omitempty"` +} + type PersonWithoutID struct { FirstName string `json:"firstName,omitempty"` LastName string `json:"lastName,omitempty"` diff --git a/api/pb/threads.pb.go b/api/pb/threads.pb.go index 5a72a5e7..8842a431 100644 --- a/api/pb/threads.pb.go +++ b/api/pb/threads.pb.go @@ -52,7 +52,7 @@ func (x ListenRequest_Filter_Action) String() string { } func (ListenRequest_Filter_Action) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{29, 0, 0} + return fileDescriptor_f2ba358bb2150022, []int{36, 0, 0} } type ListenReply_Action int32 @@ -80,7 +80,7 @@ func (x ListenReply_Action) String() string { } func (ListenReply_Action) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{30, 0} + return fileDescriptor_f2ba358bb2150022, []int{37, 0} } type GetTokenRequest struct { @@ -346,12 +346,12 @@ func (m *NewDBFromAddrRequest) GetCollections() []*CollectionConfig { } type CollectionConfig struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Schema []byte `protobuf:"bytes,2,opt,name=schema,proto3" json:"schema,omitempty"` - Indexes []*CollectionConfig_IndexConfig `protobuf:"bytes,3,rep,name=indexes,proto3" json:"indexes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Schema []byte `protobuf:"bytes,2,opt,name=schema,proto3" json:"schema,omitempty"` + Indexes []*Index `protobuf:"bytes,3,rep,name=indexes,proto3" json:"indexes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *CollectionConfig) Reset() { *m = CollectionConfig{} } @@ -393,14 +393,14 @@ func (m *CollectionConfig) GetSchema() []byte { return nil } -func (m *CollectionConfig) GetIndexes() []*CollectionConfig_IndexConfig { +func (m *CollectionConfig) GetIndexes() []*Index { if m != nil { return m.Indexes } return nil } -type CollectionConfig_IndexConfig struct { +type Index struct { Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` Unique bool `protobuf:"varint,2,opt,name=unique,proto3" json:"unique,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -408,39 +408,39 @@ type CollectionConfig_IndexConfig struct { XXX_sizecache int32 `json:"-"` } -func (m *CollectionConfig_IndexConfig) Reset() { *m = CollectionConfig_IndexConfig{} } -func (m *CollectionConfig_IndexConfig) String() string { return proto.CompactTextString(m) } -func (*CollectionConfig_IndexConfig) ProtoMessage() {} -func (*CollectionConfig_IndexConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{4, 0} +func (m *Index) Reset() { *m = Index{} } +func (m *Index) String() string { return proto.CompactTextString(m) } +func (*Index) ProtoMessage() {} +func (*Index) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{5} } -func (m *CollectionConfig_IndexConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CollectionConfig_IndexConfig.Unmarshal(m, b) +func (m *Index) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Index.Unmarshal(m, b) } -func (m *CollectionConfig_IndexConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CollectionConfig_IndexConfig.Marshal(b, m, deterministic) +func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Index.Marshal(b, m, deterministic) } -func (m *CollectionConfig_IndexConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_CollectionConfig_IndexConfig.Merge(m, src) +func (m *Index) XXX_Merge(src proto.Message) { + xxx_messageInfo_Index.Merge(m, src) } -func (m *CollectionConfig_IndexConfig) XXX_Size() int { - return xxx_messageInfo_CollectionConfig_IndexConfig.Size(m) +func (m *Index) XXX_Size() int { + return xxx_messageInfo_Index.Size(m) } -func (m *CollectionConfig_IndexConfig) XXX_DiscardUnknown() { - xxx_messageInfo_CollectionConfig_IndexConfig.DiscardUnknown(m) +func (m *Index) XXX_DiscardUnknown() { + xxx_messageInfo_Index.DiscardUnknown(m) } -var xxx_messageInfo_CollectionConfig_IndexConfig proto.InternalMessageInfo +var xxx_messageInfo_Index proto.InternalMessageInfo -func (m *CollectionConfig_IndexConfig) GetPath() string { +func (m *Index) GetPath() string { if m != nil { return m.Path } return "" } -func (m *CollectionConfig_IndexConfig) GetUnique() bool { +func (m *Index) GetUnique() bool { if m != nil { return m.Unique } @@ -457,7 +457,7 @@ func (m *NewDBReply) Reset() { *m = NewDBReply{} } func (m *NewDBReply) String() string { return proto.CompactTextString(m) } func (*NewDBReply) ProtoMessage() {} func (*NewDBReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{5} + return fileDescriptor_f2ba358bb2150022, []int{6} } func (m *NewDBReply) XXX_Unmarshal(b []byte) error { @@ -489,7 +489,7 @@ func (m *GetDBInfoRequest) Reset() { *m = GetDBInfoRequest{} } func (m *GetDBInfoRequest) String() string { return proto.CompactTextString(m) } func (*GetDBInfoRequest) ProtoMessage() {} func (*GetDBInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{6} + return fileDescriptor_f2ba358bb2150022, []int{7} } func (m *GetDBInfoRequest) XXX_Unmarshal(b []byte) error { @@ -529,7 +529,7 @@ func (m *GetDBInfoReply) Reset() { *m = GetDBInfoReply{} } func (m *GetDBInfoReply) String() string { return proto.CompactTextString(m) } func (*GetDBInfoReply) ProtoMessage() {} func (*GetDBInfoReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{7} + return fileDescriptor_f2ba358bb2150022, []int{8} } func (m *GetDBInfoReply) XXX_Unmarshal(b []byte) error { @@ -575,7 +575,7 @@ func (m *DeleteDBRequest) Reset() { *m = DeleteDBRequest{} } func (m *DeleteDBRequest) String() string { return proto.CompactTextString(m) } func (*DeleteDBRequest) ProtoMessage() {} func (*DeleteDBRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{8} + return fileDescriptor_f2ba358bb2150022, []int{9} } func (m *DeleteDBRequest) XXX_Unmarshal(b []byte) error { @@ -613,7 +613,7 @@ func (m *DeleteDBReply) Reset() { *m = DeleteDBReply{} } func (m *DeleteDBReply) String() string { return proto.CompactTextString(m) } func (*DeleteDBReply) ProtoMessage() {} func (*DeleteDBReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{9} + return fileDescriptor_f2ba358bb2150022, []int{10} } func (m *DeleteDBReply) XXX_Unmarshal(b []byte) error { @@ -646,7 +646,7 @@ func (m *NewCollectionRequest) Reset() { *m = NewCollectionRequest{} } func (m *NewCollectionRequest) String() string { return proto.CompactTextString(m) } func (*NewCollectionRequest) ProtoMessage() {} func (*NewCollectionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{10} + return fileDescriptor_f2ba358bb2150022, []int{11} } func (m *NewCollectionRequest) XXX_Unmarshal(b []byte) error { @@ -691,7 +691,7 @@ func (m *NewCollectionReply) Reset() { *m = NewCollectionReply{} } func (m *NewCollectionReply) String() string { return proto.CompactTextString(m) } func (*NewCollectionReply) ProtoMessage() {} func (*NewCollectionReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{11} + return fileDescriptor_f2ba358bb2150022, []int{12} } func (m *NewCollectionReply) XXX_Unmarshal(b []byte) error { @@ -712,6 +712,248 @@ func (m *NewCollectionReply) XXX_DiscardUnknown() { var xxx_messageInfo_NewCollectionReply proto.InternalMessageInfo +type UpdateCollectionRequest struct { + DbID []byte `protobuf:"bytes,1,opt,name=dbID,proto3" json:"dbID,omitempty"` + Config *CollectionConfig `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UpdateCollectionRequest) Reset() { *m = UpdateCollectionRequest{} } +func (m *UpdateCollectionRequest) String() string { return proto.CompactTextString(m) } +func (*UpdateCollectionRequest) ProtoMessage() {} +func (*UpdateCollectionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{13} +} + +func (m *UpdateCollectionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UpdateCollectionRequest.Unmarshal(m, b) +} +func (m *UpdateCollectionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UpdateCollectionRequest.Marshal(b, m, deterministic) +} +func (m *UpdateCollectionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateCollectionRequest.Merge(m, src) +} +func (m *UpdateCollectionRequest) XXX_Size() int { + return xxx_messageInfo_UpdateCollectionRequest.Size(m) +} +func (m *UpdateCollectionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_UpdateCollectionRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_UpdateCollectionRequest proto.InternalMessageInfo + +func (m *UpdateCollectionRequest) GetDbID() []byte { + if m != nil { + return m.DbID + } + return nil +} + +func (m *UpdateCollectionRequest) GetConfig() *CollectionConfig { + if m != nil { + return m.Config + } + return nil +} + +type UpdateCollectionReply struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UpdateCollectionReply) Reset() { *m = UpdateCollectionReply{} } +func (m *UpdateCollectionReply) String() string { return proto.CompactTextString(m) } +func (*UpdateCollectionReply) ProtoMessage() {} +func (*UpdateCollectionReply) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{14} +} + +func (m *UpdateCollectionReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UpdateCollectionReply.Unmarshal(m, b) +} +func (m *UpdateCollectionReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UpdateCollectionReply.Marshal(b, m, deterministic) +} +func (m *UpdateCollectionReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_UpdateCollectionReply.Merge(m, src) +} +func (m *UpdateCollectionReply) XXX_Size() int { + return xxx_messageInfo_UpdateCollectionReply.Size(m) +} +func (m *UpdateCollectionReply) XXX_DiscardUnknown() { + xxx_messageInfo_UpdateCollectionReply.DiscardUnknown(m) +} + +var xxx_messageInfo_UpdateCollectionReply proto.InternalMessageInfo + +type DeleteCollectionRequest struct { + DbID []byte `protobuf:"bytes,1,opt,name=dbID,proto3" json:"dbID,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteCollectionRequest) Reset() { *m = DeleteCollectionRequest{} } +func (m *DeleteCollectionRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteCollectionRequest) ProtoMessage() {} +func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{15} +} + +func (m *DeleteCollectionRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteCollectionRequest.Unmarshal(m, b) +} +func (m *DeleteCollectionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteCollectionRequest.Marshal(b, m, deterministic) +} +func (m *DeleteCollectionRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteCollectionRequest.Merge(m, src) +} +func (m *DeleteCollectionRequest) XXX_Size() int { + return xxx_messageInfo_DeleteCollectionRequest.Size(m) +} +func (m *DeleteCollectionRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteCollectionRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteCollectionRequest proto.InternalMessageInfo + +func (m *DeleteCollectionRequest) GetDbID() []byte { + if m != nil { + return m.DbID + } + return nil +} + +func (m *DeleteCollectionRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type DeleteCollectionReply struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteCollectionReply) Reset() { *m = DeleteCollectionReply{} } +func (m *DeleteCollectionReply) String() string { return proto.CompactTextString(m) } +func (*DeleteCollectionReply) ProtoMessage() {} +func (*DeleteCollectionReply) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{16} +} + +func (m *DeleteCollectionReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteCollectionReply.Unmarshal(m, b) +} +func (m *DeleteCollectionReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteCollectionReply.Marshal(b, m, deterministic) +} +func (m *DeleteCollectionReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteCollectionReply.Merge(m, src) +} +func (m *DeleteCollectionReply) XXX_Size() int { + return xxx_messageInfo_DeleteCollectionReply.Size(m) +} +func (m *DeleteCollectionReply) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteCollectionReply.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteCollectionReply proto.InternalMessageInfo + +type GetCollectionIndexesRequest struct { + DbID []byte `protobuf:"bytes,1,opt,name=dbID,proto3" json:"dbID,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetCollectionIndexesRequest) Reset() { *m = GetCollectionIndexesRequest{} } +func (m *GetCollectionIndexesRequest) String() string { return proto.CompactTextString(m) } +func (*GetCollectionIndexesRequest) ProtoMessage() {} +func (*GetCollectionIndexesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{17} +} + +func (m *GetCollectionIndexesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetCollectionIndexesRequest.Unmarshal(m, b) +} +func (m *GetCollectionIndexesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetCollectionIndexesRequest.Marshal(b, m, deterministic) +} +func (m *GetCollectionIndexesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCollectionIndexesRequest.Merge(m, src) +} +func (m *GetCollectionIndexesRequest) XXX_Size() int { + return xxx_messageInfo_GetCollectionIndexesRequest.Size(m) +} +func (m *GetCollectionIndexesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetCollectionIndexesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetCollectionIndexesRequest proto.InternalMessageInfo + +func (m *GetCollectionIndexesRequest) GetDbID() []byte { + if m != nil { + return m.DbID + } + return nil +} + +func (m *GetCollectionIndexesRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type GetCollectionIndexesReply struct { + Indexes []*Index `protobuf:"bytes,1,rep,name=indexes,proto3" json:"indexes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetCollectionIndexesReply) Reset() { *m = GetCollectionIndexesReply{} } +func (m *GetCollectionIndexesReply) String() string { return proto.CompactTextString(m) } +func (*GetCollectionIndexesReply) ProtoMessage() {} +func (*GetCollectionIndexesReply) Descriptor() ([]byte, []int) { + return fileDescriptor_f2ba358bb2150022, []int{18} +} + +func (m *GetCollectionIndexesReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetCollectionIndexesReply.Unmarshal(m, b) +} +func (m *GetCollectionIndexesReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetCollectionIndexesReply.Marshal(b, m, deterministic) +} +func (m *GetCollectionIndexesReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetCollectionIndexesReply.Merge(m, src) +} +func (m *GetCollectionIndexesReply) XXX_Size() int { + return xxx_messageInfo_GetCollectionIndexesReply.Size(m) +} +func (m *GetCollectionIndexesReply) XXX_DiscardUnknown() { + xxx_messageInfo_GetCollectionIndexesReply.DiscardUnknown(m) +} + +var xxx_messageInfo_GetCollectionIndexesReply proto.InternalMessageInfo + +func (m *GetCollectionIndexesReply) GetIndexes() []*Index { + if m != nil { + return m.Indexes + } + return nil +} + type CreateRequest struct { DbID []byte `protobuf:"bytes,1,opt,name=dbID,proto3" json:"dbID,omitempty"` CollectionName string `protobuf:"bytes,2,opt,name=collectionName,proto3" json:"collectionName,omitempty"` @@ -725,7 +967,7 @@ func (m *CreateRequest) Reset() { *m = CreateRequest{} } func (m *CreateRequest) String() string { return proto.CompactTextString(m) } func (*CreateRequest) ProtoMessage() {} func (*CreateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{12} + return fileDescriptor_f2ba358bb2150022, []int{19} } func (m *CreateRequest) XXX_Unmarshal(b []byte) error { @@ -778,7 +1020,7 @@ func (m *CreateReply) Reset() { *m = CreateReply{} } func (m *CreateReply) String() string { return proto.CompactTextString(m) } func (*CreateReply) ProtoMessage() {} func (*CreateReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{13} + return fileDescriptor_f2ba358bb2150022, []int{20} } func (m *CreateReply) XXX_Unmarshal(b []byte) error { @@ -819,7 +1061,7 @@ func (m *SaveRequest) Reset() { *m = SaveRequest{} } func (m *SaveRequest) String() string { return proto.CompactTextString(m) } func (*SaveRequest) ProtoMessage() {} func (*SaveRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{14} + return fileDescriptor_f2ba358bb2150022, []int{21} } func (m *SaveRequest) XXX_Unmarshal(b []byte) error { @@ -871,7 +1113,7 @@ func (m *SaveReply) Reset() { *m = SaveReply{} } func (m *SaveReply) String() string { return proto.CompactTextString(m) } func (*SaveReply) ProtoMessage() {} func (*SaveReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{15} + return fileDescriptor_f2ba358bb2150022, []int{22} } func (m *SaveReply) XXX_Unmarshal(b []byte) error { @@ -905,7 +1147,7 @@ func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{16} + return fileDescriptor_f2ba358bb2150022, []int{23} } func (m *DeleteRequest) XXX_Unmarshal(b []byte) error { @@ -957,7 +1199,7 @@ func (m *DeleteReply) Reset() { *m = DeleteReply{} } func (m *DeleteReply) String() string { return proto.CompactTextString(m) } func (*DeleteReply) ProtoMessage() {} func (*DeleteReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{17} + return fileDescriptor_f2ba358bb2150022, []int{24} } func (m *DeleteReply) XXX_Unmarshal(b []byte) error { @@ -991,7 +1233,7 @@ func (m *HasRequest) Reset() { *m = HasRequest{} } func (m *HasRequest) String() string { return proto.CompactTextString(m) } func (*HasRequest) ProtoMessage() {} func (*HasRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{18} + return fileDescriptor_f2ba358bb2150022, []int{25} } func (m *HasRequest) XXX_Unmarshal(b []byte) error { @@ -1044,7 +1286,7 @@ func (m *HasReply) Reset() { *m = HasReply{} } func (m *HasReply) String() string { return proto.CompactTextString(m) } func (*HasReply) ProtoMessage() {} func (*HasReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{19} + return fileDescriptor_f2ba358bb2150022, []int{26} } func (m *HasReply) XXX_Unmarshal(b []byte) error { @@ -1085,7 +1327,7 @@ func (m *FindRequest) Reset() { *m = FindRequest{} } func (m *FindRequest) String() string { return proto.CompactTextString(m) } func (*FindRequest) ProtoMessage() {} func (*FindRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{20} + return fileDescriptor_f2ba358bb2150022, []int{27} } func (m *FindRequest) XXX_Unmarshal(b []byte) error { @@ -1138,7 +1380,7 @@ func (m *FindReply) Reset() { *m = FindReply{} } func (m *FindReply) String() string { return proto.CompactTextString(m) } func (*FindReply) ProtoMessage() {} func (*FindReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{21} + return fileDescriptor_f2ba358bb2150022, []int{28} } func (m *FindReply) XXX_Unmarshal(b []byte) error { @@ -1179,7 +1421,7 @@ func (m *FindByIDRequest) Reset() { *m = FindByIDRequest{} } func (m *FindByIDRequest) String() string { return proto.CompactTextString(m) } func (*FindByIDRequest) ProtoMessage() {} func (*FindByIDRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{22} + return fileDescriptor_f2ba358bb2150022, []int{29} } func (m *FindByIDRequest) XXX_Unmarshal(b []byte) error { @@ -1232,7 +1474,7 @@ func (m *FindByIDReply) Reset() { *m = FindByIDReply{} } func (m *FindByIDReply) String() string { return proto.CompactTextString(m) } func (*FindByIDReply) ProtoMessage() {} func (*FindByIDReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{23} + return fileDescriptor_f2ba358bb2150022, []int{30} } func (m *FindByIDReply) XXX_Unmarshal(b []byte) error { @@ -1272,7 +1514,7 @@ func (m *StartTransactionRequest) Reset() { *m = StartTransactionRequest func (m *StartTransactionRequest) String() string { return proto.CompactTextString(m) } func (*StartTransactionRequest) ProtoMessage() {} func (*StartTransactionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{24} + return fileDescriptor_f2ba358bb2150022, []int{31} } func (m *StartTransactionRequest) XXX_Unmarshal(b []byte) error { @@ -1323,7 +1565,7 @@ func (m *ReadTransactionRequest) Reset() { *m = ReadTransactionRequest{} func (m *ReadTransactionRequest) String() string { return proto.CompactTextString(m) } func (*ReadTransactionRequest) ProtoMessage() {} func (*ReadTransactionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{25} + return fileDescriptor_f2ba358bb2150022, []int{32} } func (m *ReadTransactionRequest) XXX_Unmarshal(b []byte) error { @@ -1432,7 +1674,7 @@ func (m *ReadTransactionReply) Reset() { *m = ReadTransactionReply{} } func (m *ReadTransactionReply) String() string { return proto.CompactTextString(m) } func (*ReadTransactionReply) ProtoMessage() {} func (*ReadTransactionReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{26} + return fileDescriptor_f2ba358bb2150022, []int{33} } func (m *ReadTransactionReply) XXX_Unmarshal(b []byte) error { @@ -1531,7 +1773,7 @@ func (m *WriteTransactionRequest) Reset() { *m = WriteTransactionRequest func (m *WriteTransactionRequest) String() string { return proto.CompactTextString(m) } func (*WriteTransactionRequest) ProtoMessage() {} func (*WriteTransactionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{27} + return fileDescriptor_f2ba358bb2150022, []int{34} } func (m *WriteTransactionRequest) XXX_Unmarshal(b []byte) error { @@ -1685,7 +1927,7 @@ func (m *WriteTransactionReply) Reset() { *m = WriteTransactionReply{} } func (m *WriteTransactionReply) String() string { return proto.CompactTextString(m) } func (*WriteTransactionReply) ProtoMessage() {} func (*WriteTransactionReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{28} + return fileDescriptor_f2ba358bb2150022, []int{35} } func (m *WriteTransactionReply) XXX_Unmarshal(b []byte) error { @@ -1819,7 +2061,7 @@ func (m *ListenRequest) Reset() { *m = ListenRequest{} } func (m *ListenRequest) String() string { return proto.CompactTextString(m) } func (*ListenRequest) ProtoMessage() {} func (*ListenRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{29} + return fileDescriptor_f2ba358bb2150022, []int{36} } func (m *ListenRequest) XXX_Unmarshal(b []byte) error { @@ -1867,7 +2109,7 @@ func (m *ListenRequest_Filter) Reset() { *m = ListenRequest_Filter{} } func (m *ListenRequest_Filter) String() string { return proto.CompactTextString(m) } func (*ListenRequest_Filter) ProtoMessage() {} func (*ListenRequest_Filter) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{29, 0} + return fileDescriptor_f2ba358bb2150022, []int{36, 0} } func (m *ListenRequest_Filter) XXX_Unmarshal(b []byte) error { @@ -1923,7 +2165,7 @@ func (m *ListenReply) Reset() { *m = ListenReply{} } func (m *ListenReply) String() string { return proto.CompactTextString(m) } func (*ListenReply) ProtoMessage() {} func (*ListenReply) Descriptor() ([]byte, []int) { - return fileDescriptor_f2ba358bb2150022, []int{30} + return fileDescriptor_f2ba358bb2150022, []int{37} } func (m *ListenReply) XXX_Unmarshal(b []byte) error { @@ -1980,7 +2222,7 @@ func init() { proto.RegisterType((*NewDBRequest)(nil), "threads.pb.NewDBRequest") proto.RegisterType((*NewDBFromAddrRequest)(nil), "threads.pb.NewDBFromAddrRequest") proto.RegisterType((*CollectionConfig)(nil), "threads.pb.CollectionConfig") - proto.RegisterType((*CollectionConfig_IndexConfig)(nil), "threads.pb.CollectionConfig.IndexConfig") + proto.RegisterType((*Index)(nil), "threads.pb.Index") proto.RegisterType((*NewDBReply)(nil), "threads.pb.NewDBReply") proto.RegisterType((*GetDBInfoRequest)(nil), "threads.pb.GetDBInfoRequest") proto.RegisterType((*GetDBInfoReply)(nil), "threads.pb.GetDBInfoReply") @@ -1988,6 +2230,12 @@ func init() { proto.RegisterType((*DeleteDBReply)(nil), "threads.pb.DeleteDBReply") proto.RegisterType((*NewCollectionRequest)(nil), "threads.pb.NewCollectionRequest") proto.RegisterType((*NewCollectionReply)(nil), "threads.pb.NewCollectionReply") + proto.RegisterType((*UpdateCollectionRequest)(nil), "threads.pb.UpdateCollectionRequest") + proto.RegisterType((*UpdateCollectionReply)(nil), "threads.pb.UpdateCollectionReply") + proto.RegisterType((*DeleteCollectionRequest)(nil), "threads.pb.DeleteCollectionRequest") + proto.RegisterType((*DeleteCollectionReply)(nil), "threads.pb.DeleteCollectionReply") + proto.RegisterType((*GetCollectionIndexesRequest)(nil), "threads.pb.GetCollectionIndexesRequest") + proto.RegisterType((*GetCollectionIndexesReply)(nil), "threads.pb.GetCollectionIndexesReply") proto.RegisterType((*CreateRequest)(nil), "threads.pb.CreateRequest") proto.RegisterType((*CreateReply)(nil), "threads.pb.CreateReply") proto.RegisterType((*SaveRequest)(nil), "threads.pb.SaveRequest") @@ -2013,92 +2261,98 @@ func init() { func init() { proto.RegisterFile("threads.proto", fileDescriptor_f2ba358bb2150022) } var fileDescriptor_f2ba358bb2150022 = []byte{ - // 1347 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x51, 0x8f, 0xdb, 0xc4, - 0x13, 0xb7, 0x93, 0x9c, 0x93, 0x8c, 0x2f, 0x4d, 0xb4, 0xba, 0x26, 0xa9, 0xff, 0xd5, 0x29, 0xff, - 0x45, 0x94, 0x00, 0x52, 0xa8, 0x52, 0x8a, 0x0a, 0x95, 0x0a, 0xc9, 0x25, 0xbd, 0x04, 0x4e, 0xa5, - 0x72, 0x02, 0x3c, 0xa1, 0xd6, 0x17, 0x6f, 0x2e, 0xa6, 0x3e, 0x27, 0xb5, 0x7d, 0xd0, 0x48, 0x3c, - 0xf0, 0xc6, 0xf7, 0xe0, 0x89, 0x57, 0x3e, 0x00, 0x4f, 0x48, 0x3c, 0xf2, 0xc6, 0xf7, 0x41, 0xbb, - 0x6b, 0xc7, 0x6b, 0xc7, 0x4e, 0x4b, 0xef, 0x54, 0xde, 0xb2, 0xbb, 0x33, 0xf3, 0xdb, 0x99, 0xf9, - 0xed, 0x8c, 0x27, 0x50, 0xf1, 0x17, 0x2e, 0x31, 0x4c, 0xaf, 0xb3, 0x72, 0x97, 0xfe, 0x12, 0xc1, - 0x66, 0x79, 0x8a, 0x1f, 0x43, 0xf5, 0x98, 0xf8, 0xd3, 0xe5, 0x33, 0xe2, 0xe8, 0xe4, 0xf9, 0x05, - 0xf1, 0x7c, 0x84, 0x20, 0xff, 0x8c, 0xac, 0x9b, 0x72, 0x4b, 0x6e, 0x97, 0x47, 0x92, 0x4e, 0x17, - 0xe8, 0x10, 0xca, 0x9e, 0x75, 0xe6, 0x18, 0xfe, 0x85, 0x4b, 0x9a, 0xb9, 0x96, 0xdc, 0xde, 0x1f, - 0x49, 0x7a, 0xb4, 0xd5, 0x2f, 0x43, 0x71, 0x65, 0xac, 0xed, 0xa5, 0x61, 0x62, 0x1d, 0x2a, 0x91, - 0xc5, 0x95, 0xcd, 0x74, 0x67, 0x0b, 0xc3, 0xb6, 0x89, 0x73, 0x46, 0x98, 0x55, 0xa6, 0xbb, 0xd9, - 0x42, 0x75, 0xd8, 0xf3, 0xa9, 0x34, 0xb3, 0x4b, 0x11, 0xf9, 0x52, 0xb4, 0x79, 0x0a, 0xfb, 0x8f, - 0xc8, 0x0f, 0x83, 0x7e, 0x74, 0xc5, 0x82, 0x79, 0x3a, 0x1e, 0x70, 0x6b, 0x3a, 0xfb, 0x8d, 0x1e, - 0x80, 0x3a, 0x5b, 0xda, 0x36, 0x99, 0xf9, 0xd6, 0xd2, 0xf1, 0x9a, 0xb9, 0x56, 0xbe, 0xad, 0x76, - 0x6f, 0x76, 0x22, 0x5f, 0x3b, 0x47, 0x9b, 0xe3, 0xa3, 0xa5, 0x33, 0xb7, 0xce, 0x74, 0x51, 0x01, - 0xff, 0x08, 0x07, 0x0c, 0xe3, 0xa1, 0xbb, 0x3c, 0xef, 0x99, 0xa6, 0x2b, 0x60, 0x19, 0xa6, 0xe9, - 0x86, 0x58, 0xf4, 0x37, 0xaa, 0xf1, 0x10, 0xb1, 0x40, 0xf0, 0x00, 0x25, 0xd0, 0xf3, 0xff, 0x16, - 0xfd, 0x77, 0x19, 0x6a, 0x49, 0x09, 0x0a, 0xed, 0x18, 0xe7, 0x3c, 0x68, 0x65, 0x9d, 0xfd, 0x46, - 0x75, 0x50, 0xbc, 0xd9, 0x82, 0x9c, 0x1b, 0x01, 0x7a, 0xb0, 0x42, 0x7d, 0x28, 0x5a, 0x8e, 0x49, - 0x5e, 0x90, 0x10, 0xbc, 0xbd, 0x0b, 0xbc, 0x33, 0xa6, 0xb2, 0xc1, 0x45, 0x42, 0x45, 0xed, 0x63, - 0x50, 0x85, 0x7d, 0x0a, 0xbf, 0x32, 0xfc, 0x45, 0x08, 0x4f, 0x7f, 0x53, 0xf8, 0x0b, 0xc7, 0x7a, - 0x7e, 0xc1, 0x59, 0x50, 0xd2, 0x83, 0x15, 0xde, 0x07, 0x08, 0x32, 0xb4, 0xb2, 0xd7, 0xf8, 0x16, - 0xd4, 0x8e, 0x89, 0x3f, 0xe8, 0x8f, 0x9d, 0xf9, 0x72, 0x47, 0xce, 0xf0, 0x3d, 0xb8, 0x26, 0xc8, - 0x51, 0xb2, 0x1c, 0xc0, 0x1e, 0x8d, 0xb0, 0xd7, 0x94, 0x5b, 0xf9, 0xf6, 0xbe, 0xce, 0x17, 0xdb, - 0xf1, 0xc6, 0x6f, 0x43, 0x75, 0x40, 0x6c, 0xe2, 0x93, 0x9d, 0xa4, 0xc0, 0x55, 0xa8, 0x44, 0x62, - 0xf4, 0x66, 0x4f, 0x59, 0x96, 0xa3, 0x70, 0xec, 0x62, 0xd4, 0x87, 0xa0, 0xcc, 0x58, 0x24, 0x18, - 0xf0, 0xcb, 0xd2, 0x19, 0xc8, 0xe2, 0x03, 0x40, 0x09, 0x04, 0x8a, 0x6b, 0x41, 0xe5, 0xc8, 0x25, - 0x86, 0x4f, 0x76, 0x01, 0xde, 0x82, 0x6b, 0x11, 0x27, 0x1e, 0xd1, 0xcc, 0xb3, 0x27, 0xa1, 0x27, - 0x76, 0xd1, 0x4d, 0x28, 0x5b, 0x8e, 0xe7, 0x1b, 0xce, 0x2c, 0xc8, 0xf6, 0xbe, 0x1e, 0x6d, 0xe0, - 0x0f, 0x40, 0x0d, 0xa1, 0x68, 0x44, 0x5b, 0xa0, 0x86, 0x67, 0xe3, 0x01, 0x8f, 0x6b, 0x59, 0x17, - 0xb7, 0xf0, 0x19, 0xa8, 0x13, 0xe3, 0xfb, 0x37, 0x70, 0x33, 0x15, 0xca, 0x1c, 0x88, 0x46, 0xe4, - 0x3c, 0x4c, 0xcd, 0x55, 0xe0, 0x26, 0x9c, 0xcc, 0x6f, 0x3b, 0x59, 0x01, 0x35, 0x84, 0xa3, 0xe8, - 0xdf, 0x01, 0x8c, 0x0c, 0xef, 0xcd, 0x40, 0x63, 0x28, 0x31, 0x2c, 0x9a, 0x8d, 0x3a, 0x28, 0xe4, - 0x85, 0xe5, 0xf9, 0x1e, 0xc3, 0x2a, 0xe9, 0xc1, 0x8a, 0xe6, 0xe0, 0xa1, 0xe5, 0x98, 0x57, 0x94, - 0x83, 0xe7, 0x17, 0xc4, 0x5d, 0x7f, 0x3e, 0xf9, 0xf2, 0x51, 0x33, 0xcf, 0x0c, 0x44, 0x1b, 0xf8, - 0x5d, 0x28, 0x73, 0x20, 0x7a, 0x9b, 0x58, 0xba, 0xe4, 0x64, 0xba, 0xce, 0xa1, 0x4a, 0x45, 0xfb, - 0xeb, 0xf1, 0xe0, 0x2a, 0xee, 0x75, 0x08, 0x10, 0x45, 0x85, 0x5d, 0xac, 0xac, 0x0b, 0x3b, 0xf8, - 0x7d, 0xa8, 0x44, 0x70, 0xf4, 0x76, 0x1a, 0x94, 0xc2, 0xe3, 0x00, 0x70, 0xb3, 0xc6, 0x5f, 0x41, - 0x63, 0xe2, 0x1b, 0xae, 0x3f, 0x75, 0x0d, 0xc7, 0x33, 0x5e, 0xfa, 0x94, 0x5f, 0xf1, 0x8e, 0xf8, - 0x8f, 0x1c, 0xd4, 0x75, 0x62, 0x98, 0x29, 0x66, 0x9f, 0x40, 0xc3, 0x4b, 0x47, 0x64, 0x48, 0x6a, - 0xf7, 0x2d, 0xb1, 0x3c, 0x64, 0x5c, 0x6e, 0x24, 0xe9, 0x59, 0x56, 0xd0, 0x3d, 0x80, 0xc5, 0x86, - 0x92, 0x41, 0xc9, 0xa9, 0x8b, 0x36, 0x23, 0xc2, 0x8e, 0x24, 0x5d, 0x90, 0x45, 0xf7, 0x41, 0x9d, - 0x47, 0xe4, 0x61, 0xa1, 0x55, 0xbb, 0x0d, 0x51, 0x55, 0xe0, 0xd6, 0x48, 0xd2, 0x45, 0x69, 0x74, - 0x0c, 0xd5, 0x79, 0x3c, 0xcb, 0xcd, 0x02, 0x33, 0xf0, 0xbf, 0xa4, 0x01, 0x41, 0x64, 0x24, 0xe9, - 0x49, 0xad, 0x7e, 0x09, 0x94, 0xe5, 0x8a, 0x3a, 0x84, 0xff, 0x92, 0xe1, 0x60, 0x2b, 0x8a, 0x34, - 0xa3, 0x5d, 0x28, 0x2d, 0x82, 0x97, 0x10, 0x04, 0xed, 0x60, 0xcb, 0xc1, 0x95, 0xbd, 0x1e, 0x49, - 0xfa, 0x46, 0x0e, 0xdd, 0x85, 0xf2, 0x3c, 0x24, 0x6c, 0x10, 0x95, 0xeb, 0xdb, 0xae, 0x71, 0xad, - 0x48, 0x12, 0xf5, 0xa0, 0x32, 0x17, 0xd9, 0x14, 0x44, 0xe5, 0x46, 0xba, 0x53, 0x5c, 0x3d, 0xae, - 0x21, 0x38, 0xf4, 0x73, 0x01, 0x1a, 0xdf, 0xb8, 0x96, 0x4f, 0xfe, 0x0b, 0x5e, 0xf4, 0xa0, 0x32, - 0x13, 0x5b, 0x47, 0x10, 0x84, 0x98, 0x27, 0xb1, 0xde, 0x42, 0x3d, 0x89, 0x69, 0x50, 0x82, 0x78, - 0x51, 0x85, 0x4f, 0x23, 0x88, 0xd0, 0x00, 0x28, 0x41, 0x04, 0x69, 0x8a, 0x6f, 0x8a, 0x85, 0x3a, - 0xa0, 0x47, 0x0c, 0x3f, 0x56, 0xc9, 0x29, 0x7e, 0x4c, 0x23, 0x41, 0xed, 0xbd, 0xd7, 0xa7, 0xb6, - 0x72, 0x59, 0x6a, 0x17, 0x2f, 0x49, 0xed, 0x9f, 0xf2, 0x70, 0x7d, 0x9b, 0x09, 0x94, 0x70, 0xf7, - 0x41, 0x9d, 0x45, 0x6d, 0x37, 0xc8, 0x7d, 0x23, 0x2d, 0x49, 0x9c, 0x6c, 0xa2, 0x34, 0x25, 0xb9, - 0x17, 0x76, 0xc6, 0x34, 0x92, 0x6f, 0xda, 0x26, 0xfb, 0xec, 0x0e, 0x17, 0x14, 0xd3, 0x8c, 0x9a, - 0x5a, 0x5a, 0x5e, 0x85, 0x9e, 0x47, 0x31, 0x05, 0xe9, 0xd8, 0x63, 0x2c, 0xbc, 0xce, 0x63, 0xdc, - 0x7b, 0xfd, 0xc7, 0xa8, 0x5c, 0xe2, 0x31, 0xfe, 0x9a, 0x83, 0xca, 0x89, 0xe5, 0xf9, 0x64, 0x67, - 0xc5, 0xff, 0x04, 0x8a, 0x73, 0xcb, 0xf6, 0x89, 0x1b, 0x8e, 0x02, 0x2d, 0x11, 0x2c, 0xa6, 0xdf, - 0x79, 0xc8, 0x04, 0xf5, 0x50, 0x41, 0xfb, 0x53, 0x06, 0x85, 0xef, 0xa5, 0x34, 0x0e, 0xf9, 0x15, - 0x9a, 0x5b, 0x2e, 0xd9, 0xdc, 0xd0, 0xa7, 0xa0, 0x70, 0xb2, 0xb0, 0x24, 0x5d, 0xeb, 0xbe, 0xf3, - 0xb2, 0xdb, 0x74, 0x7a, 0x9c, 0x5b, 0x81, 0x1a, 0xbe, 0x03, 0x0a, 0xdf, 0x41, 0x45, 0xc8, 0xf7, - 0x4e, 0x4e, 0x6a, 0x12, 0x02, 0x50, 0x8e, 0xf4, 0x61, 0x6f, 0x3a, 0xac, 0xc9, 0xa8, 0x04, 0x85, - 0x49, 0xef, 0xeb, 0x61, 0x2d, 0x47, 0x77, 0x07, 0xc3, 0x93, 0xe1, 0x74, 0x58, 0xcb, 0xe3, 0xbf, - 0x65, 0x50, 0x43, 0xe3, 0x34, 0x0f, 0x57, 0xe5, 0xcd, 0x47, 0x09, 0x6f, 0x0e, 0xd3, 0xbc, 0x59, - 0xd9, 0xeb, 0x84, 0x13, 0xb1, 0x8e, 0x5e, 0x48, 0x74, 0xf4, 0xf7, 0x36, 0x0e, 0x46, 0x7e, 0x49, - 0x1b, 0xbf, 0x64, 0xc1, 0xaf, 0x5c, 0xf7, 0xb7, 0x12, 0xe4, 0x7b, 0x8f, 0xc7, 0x68, 0x04, 0xa5, - 0x70, 0xd6, 0x44, 0xb1, 0x37, 0x9d, 0x98, 0x69, 0xb5, 0x1b, 0xe9, 0x87, 0xf4, 0x4b, 0x50, 0x6a, - 0xcb, 0xb7, 0x65, 0x74, 0x1f, 0xf6, 0xd8, 0xfc, 0x82, 0x9a, 0xa2, 0xa4, 0x38, 0x74, 0x6a, 0xf5, - 0x94, 0x13, 0x66, 0x00, 0x7d, 0x01, 0x95, 0xd8, 0xe8, 0x88, 0x5a, 0x5b, 0xa2, 0x89, 0xa9, 0x72, - 0x87, 0xb1, 0x63, 0x28, 0x6f, 0x66, 0x22, 0x74, 0x33, 0x71, 0xef, 0xd8, 0x48, 0xa5, 0x69, 0x19, - 0xa7, 0xdc, 0xd0, 0x00, 0x4a, 0xe1, 0xec, 0x13, 0x0f, 0x4e, 0x62, 0x70, 0xd2, 0x6e, 0xa4, 0x1f, - 0x72, 0x2b, 0x13, 0xe6, 0x5b, 0x34, 0xce, 0x6c, 0xf9, 0xb6, 0x35, 0x4b, 0x69, 0x87, 0x3b, 0x24, - 0xb8, 0xd1, 0x07, 0xa0, 0xf0, 0x62, 0x88, 0xb2, 0xbb, 0x98, 0x96, 0x55, 0x3b, 0xb1, 0x84, 0xee, - 0x41, 0x81, 0x56, 0x44, 0x94, 0xd5, 0xc2, 0xb4, 0xf4, 0xe2, 0xc9, 0x91, 0xb9, 0x87, 0x28, 0xbb, - 0x7f, 0x69, 0x59, 0x15, 0x14, 0x4b, 0xe8, 0x2e, 0xe4, 0x47, 0x86, 0x87, 0x32, 0x9a, 0x97, 0x96, - 0x5a, 0x41, 0xf9, 0x85, 0x69, 0x7d, 0x43, 0x59, 0x9d, 0x4b, 0x4b, 0xaf, 0xa2, 0x3c, 0x8b, 0x61, - 0x65, 0x44, 0xbb, 0xda, 0x96, 0x96, 0x5d, 0x4c, 0xb1, 0x84, 0xbe, 0x85, 0x6a, 0xe2, 0x83, 0x0c, - 0x61, 0x51, 0x3e, 0xfd, 0x9b, 0x57, 0x6b, 0xed, 0x94, 0x89, 0x5e, 0xcf, 0x53, 0xa8, 0x25, 0x9b, - 0x22, 0x8a, 0x7d, 0xf6, 0x64, 0x7c, 0x3c, 0x69, 0xff, 0xdf, 0x2d, 0x14, 0x21, 0x7c, 0x06, 0x0a, - 0xaf, 0x2b, 0xf1, 0xbc, 0xc5, 0x2a, 0x67, 0x3c, 0x6f, 0x42, 0x19, 0xc2, 0xd2, 0x6d, 0xb9, 0xdf, - 0x81, 0x86, 0xb5, 0xec, 0xf8, 0xe4, 0x85, 0x6f, 0xd9, 0x24, 0x14, 0x7c, 0x72, 0xe6, 0xae, 0x66, - 0xfd, 0xe2, 0x94, 0xaf, 0x1e, 0xcb, 0xbf, 0xe4, 0x8a, 0xd3, 0x91, 0x3e, 0xec, 0x0d, 0x26, 0xa7, - 0x0a, 0xfb, 0xb3, 0xec, 0xce, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x99, 0xba, 0xf9, 0x3d, - 0x13, 0x00, 0x00, + // 1446 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x41, 0x6f, 0xdb, 0x36, + 0x14, 0x96, 0x6c, 0x47, 0xb6, 0x9f, 0xe3, 0xc6, 0x23, 0xd2, 0xd8, 0x51, 0x8b, 0xc0, 0xe3, 0xd0, + 0x36, 0x5b, 0x01, 0xaf, 0x48, 0xd7, 0xa1, 0x58, 0x81, 0x6e, 0x76, 0xec, 0xc6, 0xde, 0x82, 0xae, + 0x50, 0xdc, 0xed, 0xb2, 0xa1, 0x55, 0x2c, 0x3a, 0xd6, 0xaa, 0xc8, 0xae, 0xa4, 0x6c, 0x35, 0xb0, + 0xc3, 0x6e, 0xfb, 0x1f, 0x3b, 0xed, 0x7f, 0x0c, 0xd8, 0x71, 0xb7, 0x61, 0x7f, 0x67, 0x20, 0x29, + 0x59, 0xa4, 0x2c, 0x39, 0x6d, 0x93, 0x75, 0x37, 0x93, 0x7c, 0x7c, 0x1f, 0xdf, 0x7b, 0x1f, 0xf9, + 0xe9, 0x19, 0xaa, 0xc1, 0xc4, 0x23, 0xa6, 0xe5, 0xb7, 0x66, 0xde, 0x34, 0x98, 0x22, 0x58, 0x0c, + 0x8f, 0xf1, 0x13, 0xd8, 0x38, 0x20, 0xc1, 0x70, 0xfa, 0x82, 0xb8, 0x06, 0x79, 0x79, 0x46, 0xfc, + 0x00, 0x21, 0xc8, 0xbf, 0x20, 0xf3, 0x86, 0xda, 0x54, 0x77, 0xcb, 0x7d, 0xc5, 0xa0, 0x03, 0xb4, + 0x03, 0x65, 0xdf, 0x3e, 0x71, 0xcd, 0xe0, 0xcc, 0x23, 0x8d, 0x5c, 0x53, 0xdd, 0x5d, 0xef, 0x2b, + 0x46, 0x3c, 0xd5, 0x29, 0x43, 0x71, 0x66, 0xce, 0x9d, 0xa9, 0x69, 0x61, 0x03, 0xaa, 0xb1, 0xc7, + 0x99, 0xc3, 0xf6, 0x8e, 0x26, 0xa6, 0xe3, 0x10, 0xf7, 0x84, 0x30, 0xaf, 0x6c, 0xef, 0x62, 0x0a, + 0x6d, 0xc1, 0x5a, 0x40, 0xad, 0x99, 0x5f, 0x8a, 0xc8, 0x87, 0xa2, 0xcf, 0x63, 0x58, 0x7f, 0x4c, + 0x7e, 0xea, 0x76, 0xe2, 0x23, 0x16, 0xac, 0xe3, 0x41, 0x97, 0x7b, 0x33, 0xd8, 0x6f, 0xf4, 0x10, + 0x2a, 0xa3, 0xa9, 0xe3, 0x90, 0x51, 0x60, 0x4f, 0x5d, 0xbf, 0x91, 0x6b, 0xe6, 0x77, 0x2b, 0x7b, + 0xd7, 0x5b, 0x71, 0xac, 0xad, 0xfd, 0xc5, 0xf2, 0xfe, 0xd4, 0x1d, 0xdb, 0x27, 0x86, 0xb8, 0x01, + 0xff, 0x0c, 0x9b, 0x0c, 0xe3, 0x91, 0x37, 0x3d, 0x6d, 0x5b, 0x96, 0x27, 0x60, 0x99, 0x96, 0xe5, + 0x45, 0x58, 0xf4, 0x37, 0xaa, 0xf1, 0x14, 0xb1, 0x44, 0xf0, 0x04, 0x25, 0xd0, 0xf3, 0x6f, 0x8a, + 0xfe, 0x02, 0x6a, 0x49, 0x03, 0x8a, 0xec, 0x9a, 0xa7, 0x3c, 0x67, 0x65, 0x83, 0xfd, 0x46, 0x5b, + 0xa0, 0xf9, 0xa3, 0x09, 0x39, 0x35, 0x43, 0xf0, 0x70, 0x84, 0x6e, 0x43, 0xd1, 0x76, 0x2d, 0xf2, + 0x8a, 0x44, 0xd8, 0xef, 0x89, 0xd8, 0x03, 0xba, 0x64, 0x44, 0x16, 0xf8, 0x2e, 0xac, 0xb1, 0x19, + 0x8a, 0x30, 0x33, 0x83, 0x49, 0x84, 0x40, 0x7f, 0x53, 0x84, 0x33, 0xd7, 0x7e, 0x79, 0xc6, 0xeb, + 0x5c, 0x32, 0xc2, 0x11, 0x5e, 0x07, 0x08, 0x6b, 0x30, 0x73, 0xe6, 0xf8, 0x26, 0xd4, 0x0e, 0x48, + 0xd0, 0xed, 0x0c, 0xdc, 0xf1, 0x74, 0x45, 0x55, 0xf0, 0x7d, 0xb8, 0x22, 0xd8, 0x51, 0x3a, 0x6c, + 0xc2, 0x1a, 0xcd, 0xa1, 0xdf, 0x50, 0x9b, 0xf9, 0xdd, 0x75, 0x83, 0x0f, 0x96, 0x33, 0x8a, 0x6f, + 0xc0, 0x46, 0x97, 0x38, 0x24, 0x20, 0x2b, 0xcb, 0x8e, 0x37, 0xa0, 0x1a, 0x9b, 0xd1, 0x93, 0x3d, + 0x67, 0x75, 0x8c, 0x93, 0xb9, 0x8a, 0x33, 0x9f, 0x80, 0x36, 0x62, 0xb9, 0x66, 0xc0, 0xe7, 0x15, + 0x2c, 0xb4, 0xc5, 0x9b, 0x80, 0x12, 0x08, 0x14, 0x77, 0x04, 0xf5, 0xa7, 0x33, 0xcb, 0x0c, 0xc8, + 0x7f, 0x09, 0x5d, 0x87, 0xab, 0xcb, 0x20, 0x14, 0xbd, 0x0d, 0x75, 0x9e, 0x86, 0xd7, 0x43, 0x8f, + 0xa8, 0x95, 0x8b, 0xa9, 0x45, 0x7d, 0x2f, 0xbb, 0xa0, 0xbe, 0x7b, 0x70, 0xed, 0x80, 0x04, 0xf1, + 0xec, 0x80, 0xd3, 0xe8, 0x4d, 0xfd, 0xf7, 0x61, 0x3b, 0xdd, 0x0d, 0x65, 0x85, 0xc0, 0x5f, 0xf5, + 0x5c, 0xfe, 0xda, 0x50, 0xdd, 0xf7, 0x88, 0x19, 0x90, 0x55, 0x47, 0xb8, 0x09, 0x57, 0xe2, 0x0b, + 0xf6, 0x38, 0x3e, 0x4c, 0x62, 0x16, 0x5d, 0x87, 0xb2, 0xed, 0xfa, 0x81, 0xe9, 0x8e, 0xc2, 0xbb, + 0xb3, 0x6e, 0xc4, 0x13, 0xf8, 0x63, 0xa8, 0x44, 0x50, 0xf4, 0x98, 0x4d, 0xa8, 0x44, 0x6b, 0x83, + 0x2e, 0x3f, 0x6a, 0xd9, 0x10, 0xa7, 0xf0, 0x09, 0x54, 0x8e, 0xcc, 0x1f, 0xdf, 0xc1, 0xc9, 0x2a, + 0x50, 0xe6, 0x40, 0xb4, 0x44, 0xa7, 0xd1, 0x2d, 0xb8, 0x0c, 0xdc, 0x44, 0x90, 0xf9, 0xe5, 0x20, + 0xab, 0x50, 0x89, 0xe0, 0x28, 0xfa, 0x0f, 0x00, 0x7d, 0xd3, 0x7f, 0x37, 0xd0, 0x18, 0x4a, 0x0c, + 0x8b, 0x56, 0x63, 0x0b, 0x34, 0xf2, 0xca, 0xf6, 0x03, 0x9f, 0x61, 0x95, 0x8c, 0x70, 0x44, 0x6b, + 0xf0, 0xc8, 0x76, 0xad, 0x4b, 0xaa, 0xc1, 0xcb, 0x33, 0xe2, 0xcd, 0xbf, 0x3c, 0xfa, 0xfa, 0x71, + 0x23, 0xcf, 0x1c, 0xc4, 0x13, 0xf8, 0x43, 0x28, 0x73, 0x20, 0x7a, 0x1a, 0xa9, 0x5c, 0x6a, 0xb2, + 0x5c, 0xa7, 0xb0, 0x41, 0x4d, 0x3b, 0xf3, 0x41, 0xf7, 0x32, 0xce, 0xb5, 0x03, 0x10, 0x67, 0x85, + 0x1d, 0xac, 0x6c, 0x08, 0x33, 0xf8, 0x36, 0x54, 0x63, 0x38, 0x7a, 0x3a, 0x1d, 0x4a, 0xd1, 0x72, + 0x08, 0xb8, 0x18, 0xe3, 0xa7, 0x50, 0x3f, 0x0a, 0x4c, 0x2f, 0x18, 0x7a, 0xa6, 0xeb, 0x9b, 0xe7, + 0x3e, 0x1e, 0xaf, 0x79, 0x46, 0xfc, 0x47, 0x0e, 0xb6, 0x0c, 0x62, 0x5a, 0x29, 0x6e, 0x9f, 0x41, + 0xdd, 0x4f, 0x47, 0x64, 0x48, 0x95, 0xbd, 0x0f, 0xc4, 0xeb, 0x9f, 0x71, 0xb8, 0xbe, 0x62, 0x64, + 0x79, 0x41, 0xf7, 0x01, 0x26, 0x0b, 0x4a, 0x86, 0x4f, 0xec, 0x96, 0xe8, 0x33, 0x26, 0x6c, 0x5f, + 0x31, 0x04, 0x5b, 0xf4, 0x00, 0x2a, 0xe3, 0x98, 0x3c, 0x2c, 0xb5, 0x95, 0xbd, 0xba, 0xb8, 0x55, + 0xe0, 0x56, 0x5f, 0x31, 0x44, 0x6b, 0x74, 0x00, 0x1b, 0x63, 0xb9, 0xca, 0x8d, 0x02, 0x73, 0x70, + 0x2d, 0xe9, 0x40, 0x30, 0xe9, 0x2b, 0x46, 0x72, 0x57, 0xa7, 0x04, 0xda, 0x74, 0x46, 0x03, 0xc2, + 0x7f, 0xa9, 0xb0, 0xb9, 0x94, 0x45, 0x5a, 0xd1, 0x3d, 0x28, 0x4d, 0xc2, 0x9b, 0x10, 0x26, 0x6d, + 0x73, 0x29, 0xc0, 0x99, 0x33, 0xef, 0x2b, 0xc6, 0xc2, 0x0e, 0xdd, 0x83, 0xf2, 0x38, 0x22, 0x6c, + 0x98, 0x95, 0xab, 0xcb, 0xa1, 0xf1, 0x5d, 0xb1, 0x25, 0x6a, 0x43, 0x75, 0x2c, 0xb2, 0x29, 0xcc, + 0xca, 0x76, 0x7a, 0x50, 0x7c, 0xbb, 0xbc, 0x43, 0x08, 0xe8, 0xd7, 0x02, 0xd4, 0xbf, 0xf5, 0xec, + 0x80, 0xfc, 0x1f, 0xbc, 0x68, 0x43, 0x75, 0x24, 0x4a, 0x47, 0x98, 0x04, 0x29, 0x12, 0x49, 0x5b, + 0x68, 0x24, 0xd2, 0x0e, 0x4a, 0x10, 0x3f, 0x7e, 0xe1, 0xd3, 0x08, 0x22, 0x08, 0x00, 0x25, 0x88, + 0x60, 0x4d, 0xf1, 0x2d, 0xf1, 0xa1, 0x0e, 0xe9, 0x21, 0xe1, 0x4b, 0x2f, 0x39, 0xc5, 0x97, 0x76, + 0x24, 0xa8, 0xbd, 0xf6, 0xf6, 0xd4, 0xd6, 0x2e, 0x4a, 0xed, 0xe2, 0x05, 0xa9, 0xfd, 0x4b, 0x1e, + 0xae, 0x2e, 0x33, 0x81, 0x12, 0xee, 0x01, 0x54, 0x46, 0xb1, 0xec, 0x86, 0xb5, 0xaf, 0xa7, 0x15, + 0x89, 0x93, 0x4d, 0xb4, 0xa6, 0x24, 0xf7, 0x23, 0x65, 0x4c, 0x23, 0xf9, 0x42, 0x36, 0x59, 0x0f, + 0x13, 0x0d, 0x28, 0xa6, 0x15, 0x8b, 0x5a, 0x5a, 0x5d, 0x05, 0xcd, 0xa3, 0x98, 0x82, 0xb5, 0x74, + 0x19, 0x0b, 0x6f, 0x73, 0x19, 0xd7, 0xde, 0xfe, 0x32, 0x6a, 0x17, 0xb8, 0x8c, 0xbf, 0xe7, 0xa0, + 0x7a, 0x68, 0xfb, 0x01, 0x59, 0xf9, 0xe2, 0x7f, 0x06, 0xc5, 0xb1, 0xed, 0x04, 0xc4, 0x8b, 0xfa, + 0xaa, 0xa6, 0x08, 0x26, 0xed, 0x6f, 0x3d, 0x62, 0x86, 0x46, 0xb4, 0x41, 0xff, 0x53, 0x05, 0x8d, + 0xcf, 0xa5, 0x08, 0x87, 0xfa, 0x1a, 0xe2, 0x96, 0x4b, 0x8a, 0x1b, 0xfa, 0x1c, 0x34, 0x4e, 0x16, + 0x56, 0xa4, 0x2b, 0x7b, 0xb7, 0xce, 0x3b, 0x4d, 0xab, 0xcd, 0xb9, 0x15, 0x6e, 0xc3, 0x77, 0x41, + 0xe3, 0x33, 0xa8, 0x08, 0xf9, 0xf6, 0xe1, 0x61, 0x4d, 0x41, 0x00, 0xda, 0xbe, 0xd1, 0x6b, 0x0f, + 0x7b, 0x35, 0x15, 0x95, 0xa0, 0x70, 0xd4, 0xfe, 0xa6, 0x57, 0xcb, 0xd1, 0xd9, 0x6e, 0xef, 0xb0, + 0x37, 0xec, 0xd5, 0xf2, 0xf8, 0x6f, 0x15, 0x2a, 0x91, 0x73, 0x5a, 0x87, 0xcb, 0x8a, 0xe6, 0xd3, + 0x44, 0x34, 0x3b, 0x69, 0xd1, 0xcc, 0x9c, 0x79, 0x22, 0x08, 0x49, 0xd1, 0x0b, 0x09, 0x45, 0xff, + 0x68, 0x11, 0x60, 0x1c, 0x97, 0xb2, 0x88, 0x4b, 0x15, 0xe2, 0xca, 0xed, 0xfd, 0x03, 0x90, 0x6f, + 0x3f, 0x19, 0xa0, 0x3e, 0x94, 0xa2, 0xc6, 0x1d, 0x49, 0x77, 0x3a, 0xf1, 0x07, 0x81, 0xbe, 0x9d, + 0xbe, 0x48, 0xbf, 0x04, 0x95, 0x5d, 0xf5, 0x8e, 0x8a, 0x1e, 0xc0, 0x1a, 0x6b, 0x15, 0x51, 0x43, + 0xb4, 0x14, 0x3b, 0x78, 0x7d, 0x2b, 0x65, 0x85, 0x39, 0x40, 0x5f, 0x41, 0x55, 0xea, 0xc3, 0x51, + 0x73, 0xc9, 0x34, 0xd1, 0xa2, 0xaf, 0x70, 0x76, 0x00, 0xe5, 0x45, 0xfb, 0x89, 0xae, 0x27, 0xce, + 0x2d, 0x75, 0xaf, 0xba, 0x9e, 0xb1, 0xca, 0x1d, 0x75, 0xa1, 0x14, 0xb5, 0x99, 0x72, 0x72, 0x12, + 0x3d, 0xaa, 0xbe, 0x9d, 0xbe, 0xc8, 0xbd, 0x1c, 0xb1, 0xd8, 0xe2, 0x16, 0x68, 0x29, 0xb6, 0xa5, + 0xee, 0x4d, 0xdf, 0x59, 0x61, 0xc1, 0x9d, 0x7e, 0x07, 0xb5, 0x64, 0x4f, 0x88, 0x24, 0x99, 0xcc, + 0x68, 0x4b, 0xf5, 0xf7, 0x57, 0x1b, 0x2d, 0xbc, 0x27, 0xbb, 0x42, 0xd9, 0x7b, 0x46, 0xdb, 0x29, + 0x7b, 0x4f, 0x6f, 0x2c, 0x15, 0x34, 0x81, 0xcd, 0xb4, 0x9e, 0x10, 0xdd, 0x4a, 0x14, 0x23, 0xab, + 0xf9, 0xd4, 0x6f, 0x9c, 0x6f, 0xc8, 0x91, 0x1e, 0x82, 0xc6, 0x25, 0x03, 0x65, 0x6b, 0xbd, 0x9e, + 0xa5, 0x30, 0x58, 0x41, 0xf7, 0xa1, 0x40, 0x75, 0x03, 0x65, 0x09, 0xbd, 0x9e, 0x2e, 0x31, 0x1c, + 0x99, 0x87, 0x8f, 0xb2, 0x55, 0x5e, 0xcf, 0xd2, 0x19, 0xac, 0xa0, 0x7b, 0x90, 0xef, 0x9b, 0x3e, + 0xca, 0x90, 0x78, 0x3d, 0x55, 0x67, 0xf8, 0x81, 0xa9, 0x0a, 0xa0, 0x2c, 0x7d, 0xd7, 0xd3, 0xb5, + 0x86, 0x73, 0x3d, 0xd2, 0x0f, 0xb4, 0x4a, 0xdc, 0xf5, 0x6c, 0xc9, 0xc1, 0x0a, 0xfa, 0x1e, 0x36, + 0x12, 0x9f, 0xad, 0x08, 0x8b, 0xf6, 0xe9, 0x9d, 0x81, 0xde, 0x5c, 0x69, 0x13, 0xbf, 0x31, 0xcf, + 0xa1, 0x96, 0xfc, 0x74, 0x90, 0x79, 0x99, 0xf1, 0x89, 0x29, 0xf3, 0x32, 0xf5, 0xeb, 0x23, 0x44, + 0xf8, 0x02, 0x34, 0xfe, 0xfa, 0xca, 0x75, 0x93, 0xf4, 0x45, 0xae, 0x9b, 0xf0, 0x58, 0x63, 0xe5, + 0x8e, 0xda, 0x69, 0x41, 0xdd, 0x9e, 0xb6, 0x02, 0xf2, 0x2a, 0xb0, 0x1d, 0x12, 0x19, 0x3e, 0x3b, + 0xf1, 0x66, 0xa3, 0x4e, 0x71, 0xc8, 0x47, 0x4f, 0xd4, 0xdf, 0x72, 0xc5, 0x61, 0xdf, 0xe8, 0xb5, + 0xbb, 0x47, 0xc7, 0x1a, 0xfb, 0x7f, 0xf6, 0xee, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x99, 0xad, + 0x86, 0x89, 0xb0, 0x15, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -2119,6 +2373,9 @@ type APIClient interface { GetDBInfo(ctx context.Context, in *GetDBInfoRequest, opts ...grpc.CallOption) (*GetDBInfoReply, error) DeleteDB(ctx context.Context, in *DeleteDBRequest, opts ...grpc.CallOption) (*DeleteDBReply, error) NewCollection(ctx context.Context, in *NewCollectionRequest, opts ...grpc.CallOption) (*NewCollectionReply, error) + UpdateCollection(ctx context.Context, in *UpdateCollectionRequest, opts ...grpc.CallOption) (*UpdateCollectionReply, error) + DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionReply, error) + GetCollectionIndexes(ctx context.Context, in *GetCollectionIndexesRequest, opts ...grpc.CallOption) (*GetCollectionIndexesReply, error) Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateReply, error) Save(ctx context.Context, in *SaveRequest, opts ...grpc.CallOption) (*SaveReply, error) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteReply, error) @@ -2214,6 +2471,33 @@ func (c *aPIClient) NewCollection(ctx context.Context, in *NewCollectionRequest, return out, nil } +func (c *aPIClient) UpdateCollection(ctx context.Context, in *UpdateCollectionRequest, opts ...grpc.CallOption) (*UpdateCollectionReply, error) { + out := new(UpdateCollectionReply) + err := c.cc.Invoke(ctx, "/threads.pb.API/UpdateCollection", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aPIClient) DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionReply, error) { + out := new(DeleteCollectionReply) + err := c.cc.Invoke(ctx, "/threads.pb.API/DeleteCollection", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *aPIClient) GetCollectionIndexes(ctx context.Context, in *GetCollectionIndexesRequest, opts ...grpc.CallOption) (*GetCollectionIndexesReply, error) { + out := new(GetCollectionIndexesReply) + err := c.cc.Invoke(ctx, "/threads.pb.API/GetCollectionIndexes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *aPIClient) Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateReply, error) { out := new(CreateReply) err := c.cc.Invoke(ctx, "/threads.pb.API/Create", in, out, opts...) @@ -2370,6 +2654,9 @@ type APIServer interface { GetDBInfo(context.Context, *GetDBInfoRequest) (*GetDBInfoReply, error) DeleteDB(context.Context, *DeleteDBRequest) (*DeleteDBReply, error) NewCollection(context.Context, *NewCollectionRequest) (*NewCollectionReply, error) + UpdateCollection(context.Context, *UpdateCollectionRequest) (*UpdateCollectionReply, error) + DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionReply, error) + GetCollectionIndexes(context.Context, *GetCollectionIndexesRequest) (*GetCollectionIndexesReply, error) Create(context.Context, *CreateRequest) (*CreateReply, error) Save(context.Context, *SaveRequest) (*SaveReply, error) Delete(context.Context, *DeleteRequest) (*DeleteReply, error) @@ -2403,6 +2690,15 @@ func (*UnimplementedAPIServer) DeleteDB(ctx context.Context, req *DeleteDBReques func (*UnimplementedAPIServer) NewCollection(ctx context.Context, req *NewCollectionRequest) (*NewCollectionReply, error) { return nil, status.Errorf(codes.Unimplemented, "method NewCollection not implemented") } +func (*UnimplementedAPIServer) UpdateCollection(ctx context.Context, req *UpdateCollectionRequest) (*UpdateCollectionReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateCollection not implemented") +} +func (*UnimplementedAPIServer) DeleteCollection(ctx context.Context, req *DeleteCollectionRequest) (*DeleteCollectionReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented") +} +func (*UnimplementedAPIServer) GetCollectionIndexes(ctx context.Context, req *GetCollectionIndexesRequest) (*GetCollectionIndexesReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetCollectionIndexes not implemented") +} func (*UnimplementedAPIServer) Create(ctx context.Context, req *CreateRequest) (*CreateReply, error) { return nil, status.Errorf(codes.Unimplemented, "method Create not implemented") } @@ -2551,6 +2847,60 @@ func _API_NewCollection_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _API_UpdateCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateCollectionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(APIServer).UpdateCollection(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/threads.pb.API/UpdateCollection", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(APIServer).UpdateCollection(ctx, req.(*UpdateCollectionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _API_DeleteCollection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteCollectionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(APIServer).DeleteCollection(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/threads.pb.API/DeleteCollection", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(APIServer).DeleteCollection(ctx, req.(*DeleteCollectionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _API_GetCollectionIndexes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetCollectionIndexesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(APIServer).GetCollectionIndexes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/threads.pb.API/GetCollectionIndexes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(APIServer).GetCollectionIndexes(ctx, req.(*GetCollectionIndexesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _API_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateRequest) if err := dec(in); err != nil { @@ -2756,6 +3106,18 @@ var _API_serviceDesc = grpc.ServiceDesc{ MethodName: "NewCollection", Handler: _API_NewCollection_Handler, }, + { + MethodName: "UpdateCollection", + Handler: _API_UpdateCollection_Handler, + }, + { + MethodName: "DeleteCollection", + Handler: _API_DeleteCollection_Handler, + }, + { + MethodName: "GetCollectionIndexes", + Handler: _API_GetCollectionIndexes_Handler, + }, { MethodName: "Create", Handler: _API_Create_Handler, diff --git a/api/pb/threads.proto b/api/pb/threads.proto index 3c7f5ffe..b3b3628a 100644 --- a/api/pb/threads.proto +++ b/api/pb/threads.proto @@ -34,12 +34,12 @@ message NewDBFromAddrRequest { message CollectionConfig { string name = 1; bytes schema = 2; - repeated IndexConfig indexes = 3; + repeated Index indexes = 3; +} - message IndexConfig { - string path = 1; - bool unique = 2; - } +message Index { + string path = 1; + bool unique = 2; } message NewDBReply {} @@ -66,6 +66,29 @@ message NewCollectionRequest { message NewCollectionReply {} +message UpdateCollectionRequest { + bytes dbID = 1; + CollectionConfig config = 2; +} + +message UpdateCollectionReply {} + +message DeleteCollectionRequest { + bytes dbID = 1; + string name = 2; +} + +message DeleteCollectionReply {} + +message GetCollectionIndexesRequest { + bytes dbID = 1; + string name = 2; +} + +message GetCollectionIndexesReply { + repeated Index indexes = 1; +} + message CreateRequest { bytes dbID = 1; string collectionName = 2; @@ -204,6 +227,9 @@ service API { rpc GetDBInfo(GetDBInfoRequest) returns (GetDBInfoReply) {} rpc DeleteDB(DeleteDBRequest) returns (DeleteDBReply) {} rpc NewCollection(NewCollectionRequest) returns (NewCollectionReply) {} + rpc UpdateCollection(UpdateCollectionRequest) returns (UpdateCollectionReply) {} + rpc DeleteCollection(DeleteCollectionRequest) returns (DeleteCollectionReply) {} + rpc GetCollectionIndexes(GetCollectionIndexesRequest) returns (GetCollectionIndexesReply) {} rpc Create(CreateRequest) returns (CreateReply) {} rpc Save(SaveRequest) returns (SaveReply) {} rpc Delete(DeleteRequest) returns (DeleteReply) {} diff --git a/api/service.go b/api/service.go index 180b2f00..bd01bc25 100644 --- a/api/service.go +++ b/api/service.go @@ -49,7 +49,7 @@ func NewService(network app.Net, conf Config) (*Service, error) { } } - manager, err := db.NewManager(network, db.WithNewDBRepoPath(conf.RepoPath), db.WithNewDBDebug(conf.Debug)) + manager, err := db.NewManager(network, db.WithNewRepoPath(conf.RepoPath), db.WithNewDebug(conf.Debug)) if err != nil { return nil, err } @@ -160,7 +160,7 @@ func (s *Service) NewDB(ctx context.Context, req *pb.NewDBRequest) (*pb.NewDBRep if err != nil { return nil, err } - if _, err = s.manager.NewDB(ctx, id, db.WithNewManagedDBToken(token), db.WithNewManagedDBCollections(collections...)); err != nil { + if _, err = s.manager.NewDB(ctx, id, db.WithNewManagedToken(token), db.WithNewManagedCollections(collections...)); err != nil { return nil, err } return &pb.NewDBReply{}, nil @@ -189,16 +189,16 @@ func (s *Service) NewDBFromAddr(ctx context.Context, req *pb.NewDBFromAddrReques if err != nil { return nil, err } - if _, err = s.manager.NewDBFromAddr(ctx, addr, key, db.WithNewManagedDBToken(token), db.WithNewManagedDBCollections(collections...)); err != nil { + if _, err = s.manager.NewDBFromAddr(ctx, addr, key, db.WithNewManagedToken(token), db.WithNewManagedCollections(collections...)); err != nil { return nil, err } return &pb.NewDBReply{}, nil } func collectionConfigFromPb(pbc *pb.CollectionConfig) (db.CollectionConfig, error) { - indexes := make([]db.IndexConfig, len(pbc.Indexes)) + indexes := make([]db.Index, len(pbc.Indexes)) for i, index := range pbc.Indexes { - indexes[i] = db.IndexConfig{ + indexes[i] = db.Index{ Path: index.Path, Unique: index.Unique, } @@ -229,7 +229,7 @@ func (s *Service) GetDBInfo(ctx context.Context, req *pb.GetDBInfoRequest) (*pb. return nil, err } - addrs, key, err := d.GetDBInfo(db.WithInviteInfoToken(token)) + addrs, key, err := d.GetDBInfo(db.WithToken(token)) if err != nil { return nil, err } @@ -255,9 +255,9 @@ func (s *Service) DeleteDB(ctx context.Context, req *pb.DeleteDBRequest) (*pb.De return nil, err } - if err = s.manager.DeleteDB(ctx, id, db.WithManagedDBToken(token)); err != nil { + if err = s.manager.DeleteDB(ctx, id, db.WithManagedToken(token)); err != nil { if errors.Is(err, lstore.ErrThreadNotFound) { - return nil, status.Error(codes.NotFound, "db not found") + return nil, status.Error(codes.NotFound, db.ErrDBNotFound.Error()) } else { return nil, err } @@ -288,6 +288,74 @@ func (s *Service) NewCollection(ctx context.Context, req *pb.NewCollectionReques return &pb.NewCollectionReply{}, nil } +func (s *Service) UpdateCollection(ctx context.Context, req *pb.UpdateCollectionRequest) (*pb.UpdateCollectionReply, error) { + id, err := thread.Cast(req.DbID) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + token, err := thread.NewTokenFromMD(ctx) + if err != nil { + return nil, err + } + d, err := s.getDB(ctx, id, token) + if err != nil { + return nil, err + } + cc, err := collectionConfigFromPb(req.Config) + if err != nil { + return nil, err + } + if _, err = d.UpdateCollection(cc); err != nil { + return nil, err + } + return &pb.UpdateCollectionReply{}, nil +} + +func (s *Service) DeleteCollection(ctx context.Context, req *pb.DeleteCollectionRequest) (*pb.DeleteCollectionReply, error) { + id, err := thread.Cast(req.DbID) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + token, err := thread.NewTokenFromMD(ctx) + if err != nil { + return nil, err + } + d, err := s.getDB(ctx, id, token) + if err != nil { + return nil, err + } + if err = d.DeleteCollection(req.Name); err != nil { + return nil, err + } + return &pb.DeleteCollectionReply{}, nil +} + +func (s *Service) GetCollectionIndexes(ctx context.Context, req *pb.GetCollectionIndexesRequest) (*pb.GetCollectionIndexesReply, error) { + id, err := thread.Cast(req.DbID) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + token, err := thread.NewTokenFromMD(ctx) + if err != nil { + return nil, err + } + collection, err := s.getCollection(ctx, req.Name, id, token) + if err != nil { + return nil, err + } + indexes := collection.GetIndexes() + pbindexes := make([]*pb.Index, len(indexes)) + for i, index := range indexes { + pbindexes[i] = &pb.Index{ + Path: index.Path, + Unique: index.Unique, + } + } + return &pb.GetCollectionIndexesReply{ + Indexes: pbindexes, + }, nil +} + func (s *Service) Create(ctx context.Context, req *pb.CreateRequest) (*pb.CreateReply, error) { id, err := thread.Cast(req.DbID) if err != nil { @@ -663,10 +731,10 @@ func (s *Service) Listen(req *pb.ListenRequest, server pb.API_ListenServer) erro } } -func (s *Service) instanceForAction(db *db.DB, action db.Action) ([]byte, error) { - collection := db.GetCollection(action.Collection) +func (s *Service) instanceForAction(d *db.DB, action db.Action) ([]byte, error) { + collection := d.GetCollection(action.Collection) if collection == nil { - return nil, status.Error(codes.NotFound, "collection not found") + return nil, status.Error(codes.NotFound, db.ErrCollectionNotFound.Error()) } res, err := collection.FindByID(action.ID) if err != nil { @@ -742,10 +810,10 @@ func (s *Service) processFindRequest(req *pb.FindRequest, token thread.Token, fi } func (s *Service) getDB(ctx context.Context, id thread.ID, token thread.Token) (*db.DB, error) { - d, err := s.manager.GetDB(ctx, id, db.WithManagedDBToken(token)) + d, err := s.manager.GetDB(ctx, id, db.WithManagedToken(token)) if err != nil { if errors.Is(err, lstore.ErrThreadNotFound) { - return nil, status.Error(codes.NotFound, "db not found") + return nil, status.Error(codes.NotFound, db.ErrDBNotFound.Error()) } else { return nil, err } @@ -760,7 +828,7 @@ func (s *Service) getCollection(ctx context.Context, collectionName string, id t } collection := d.GetCollection(collectionName) if collection == nil { - return nil, status.Error(codes.NotFound, "collection not found") + return nil, status.Error(codes.NotFound, db.ErrCollectionNotFound.Error()) } return collection, nil } diff --git a/core/db/db.go b/core/db/db.go index 041fa698..0d7f8da0 100644 --- a/core/db/db.go +++ b/core/db/db.go @@ -27,16 +27,14 @@ func (e InstanceID) String() string { return string(e) } -// IsValidInstanceID checks if an id is valid -func IsValidInstanceID(instanceID string) bool { - return len(instanceID) > 0 -} - // Event is a local or remote event generated in collection and dispatcher // by Dispatcher. type Event interface { + // Time (wall-clock) the event was created. Time() []byte + // InstanceID is the associated instance's unique identifier. InstanceID() InstanceID + // Collection is the associated instance's collection name. Collection() string } @@ -75,19 +73,16 @@ type ReduceAction struct { InstanceID InstanceID } +// IndexFunc handles index updates. +type IndexFunc func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error + // EventCodec transforms actions generated in collections to // events dispatched to thread logs, and viceversa. type EventCodec interface { // Reduce applies generated events into state. - Reduce( - events []Event, - datastore ds.TxnDatastore, - baseKey ds.Key, - indexFunc func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error, - ) ([]ReduceAction, error) + Reduce(events []Event, datastore ds.TxnDatastore, baseKey ds.Key, indexFunc IndexFunc) ([]ReduceAction, error) // Create corresponding events to be dispatched. Create(ops []Action) ([]Event, format.Node, error) - // EventsFromBytes deserializes a format.Node bytes payload into - // Events. + // EventsFromBytes deserializes a format.Node bytes payload into Events. EventsFromBytes(data []byte) ([]Event, error) } diff --git a/db/bench_test.go b/db/bench_test.go index 709b5505..8408c7b2 100644 --- a/db/bench_test.go +++ b/db/bench_test.go @@ -58,12 +58,12 @@ func checkBenchErr(b *testing.B, err error) { } } -func createBenchDB(b *testing.B, opts ...NewDBOption) (*DB, func()) { +func createBenchDB(b *testing.B, opts ...NewOption) (*DB, func()) { dir, err := ioutil.TempDir("", "") checkBenchErr(b, err) n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkBenchErr(b, err) - opts = append(opts, WithNewDBRepoPath(dir)) + opts = append(opts, WithNewRepoPath(dir)) d, err := NewDB(context.Background(), n, thread.NewIDV1(thread.Raw, 32), opts...) checkBenchErr(b, err) return d, func() { @@ -97,7 +97,7 @@ func BenchmarkIndexCreate(b *testing.B) { collection, err := db.NewCollection(CollectionConfig{ Name: "Dog", Schema: util.SchemaFromSchemaString(testBenchSchema), - Indexes: []IndexConfig{{ + Indexes: []Index{{ Path: "Name", Unique: false, }}, @@ -149,7 +149,7 @@ func BenchmarkIndexSave(b *testing.B) { collection, err := db.NewCollection(CollectionConfig{ Name: "Dog", Schema: util.SchemaFromSchemaString(testBenchSchema), - Indexes: []IndexConfig{{ + Indexes: []Index{{ Path: "Age", Unique: false, }}, @@ -217,7 +217,7 @@ func BenchmarkIndexFind(b *testing.B) { collection, err := db.NewCollection(CollectionConfig{ Name: "Dog", Schema: util.SchemaFromSchemaString(testBenchSchema), - Indexes: []IndexConfig{{ + Indexes: []Index{{ Path: "Name", Unique: false, }}, diff --git a/db/collection.go b/db/collection.go index 06a3eca4..00693169 100644 --- a/db/collection.go +++ b/db/collection.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "reflect" "strings" "github.com/alecthomas/jsonschema" @@ -12,14 +11,18 @@ import ( ds "github.com/ipfs/go-datastore" core "github.com/textileio/go-threads/core/db" "github.com/textileio/go-threads/core/thread" - "github.com/tidwall/gjson" "github.com/xeipuuv/gojsonschema" ) var ( - // ErrNotFound indicates that the specified instance doesn't - // exist in the collection. - ErrNotFound = errors.New("instance not found") + // ErrInvalidCollectionSchemaPath indicates path does not resolve to a schema type. + ErrInvalidCollectionSchemaPath = errors.New("collection schema does not contain path") + // ErrCollectionNotFound indicates that the specified collection doesn't exist in the db. + ErrCollectionNotFound = errors.New("collection not found") + // ErrCollectionAlreadyRegistered indicates a collection with the given name is already registered. + ErrCollectionAlreadyRegistered = errors.New("collection already registered") + // ErrInstanceNotFound indicates that the specified instance doesn't exist in the collection. + ErrInstanceNotFound = errors.New("instance not found") // ErrReadonlyTx indicates that no write operations can be done since // the current transaction is readonly. ErrReadonlyTx = errors.New("read only transaction") @@ -35,122 +38,48 @@ var ( baseKey = dsDBPrefix.ChildString("collection") ) -// Collection contains instances of a schema, and provides operations -// for creating, updating, deleting, and quering them. +// Collection is a group of instances sharing a schema. +// Collections are like RDBMS tables. They can only exist in a single database. type Collection struct { name string schemaLoader gojsonschema.JSONLoader - valueType reflect.Type db *DB indexes map[string]Index } +// newCollection returns a new Collection from schema. func newCollection(name string, schema *jsonschema.Schema, d *DB) (*Collection, error) { if name != "" && !collectionNameRx.MatchString(name) { return nil, ErrInvalidCollectionName } - - // by default, use top level properties to validate ID string property exists - properties := schema.Properties - if schema.Ref != "" { - // the schema specifies a ref to a nested type, use it instead - parts := strings.Split(schema.Ref, "/") - if len(parts) < 1 { - return nil, ErrInvalidCollectionSchema - } - typeName := parts[len(parts)-1] - refDefinition := schema.Definitions[typeName] - if refDefinition == nil { + idType, err := getSchemaTypeAtPath(schema, idFieldName) + if err != nil { + if errors.Is(err, ErrInvalidCollectionSchemaPath) { return nil, ErrInvalidCollectionSchema + } else { + return nil, err } - properties = refDefinition.Properties } - if !hasIDProperty(properties) { + if idType.Type != "string" { return nil, ErrInvalidCollectionSchema } - - schemaBytes, err := json.Marshal(schema) + sb, err := json.Marshal(schema) if err != nil { return nil, err } - schemaLoader := gojsonschema.NewBytesLoader(schemaBytes) - c := &Collection{ + return &Collection{ name: name, - schemaLoader: schemaLoader, - valueType: nil, + schemaLoader: gojsonschema.NewBytesLoader(sb), db: d, indexes: make(map[string]Index), - } - return c, nil + }, nil } -func (c *Collection) BaseKey() ds.Key { +// baseKey returns the collections base key. +func (c *Collection) baseKey() ds.Key { return baseKey.ChildString(c.name) } -// Indexes is a map of collection properties to Indexes -func (c *Collection) Indexes() map[string]Index { - return c.indexes -} - -// AddIndex creates a new index based on the given path string. -// Set unique to true if you want a unique constraint on the given path. -// See https://github.com/tidwall/gjson for documentation on the supported path structure. -// Adding an index will override any overlapping index values if they already exist. -// @note: This does NOT currently build the index. If items have been added prior to adding -// a new index, they will NOT be indexed a posteriori. -func (c *Collection) AddIndex(config IndexConfig) error { - indexKey := dsDBIndexes.ChildString(c.name) - exists, err := c.db.datastore.Has(indexKey) - if err != nil { - return err - } - - indexes := map[string]IndexConfig{} - - if exists { - indexesBytes, err := c.db.datastore.Get(indexKey) - if err != nil { - return err - } - if err = json.Unmarshal(indexesBytes, &indexes); err != nil { - return err - } - } - - // if the index being added is for path ID - if config.Path == idFieldName { - // and there already is and index on ID - if _, exists := indexes[idFieldName]; exists { - // just return gracefully - return nil - } - } - - indexes[config.Path] = config - - indexBytes, err := json.Marshal(indexes) - if err != nil { - return err - } - - if err := c.db.datastore.Put(indexKey, indexBytes); err != nil { - return err - } - - c.indexes[config.Path] = Index{ - IndexFunc: func(field string, value []byte) (ds.Key, error) { - result := gjson.GetBytes(value, field) - if !result.Exists() { - return ds.Key{}, ErrNotIndexable - } - return ds.NewKey(result.String()), nil - }, - Unique: config.Unique, - } - return nil -} - // ReadTxn creates an explicit readonly transaction. Any operation // that tries to mutate an instance of the collection will ErrReadonlyTx. // Provides serializable isolation gurantees. @@ -165,7 +94,7 @@ func (c *Collection) WriteTxn(f func(txn *Txn) error, opts ...TxnOption) error { } // FindByID finds an instance by its ID. -// If doesn't exists returns ErrNotFound. +// If doesn't exists returns ErrInstanceNotFound. func (c *Collection) FindByID(id core.InstanceID, opts ...TxnOption) (instance []byte, err error) { _ = c.ReadTxn(func(txn *Txn) error { instance, err = txn.FindByID(id) @@ -258,20 +187,6 @@ func (c *Collection) Find(q *Query, opts ...TxnOption) (instances [][]byte, err return } -// validInstance validates the json object against the collection schema -func (c *Collection) validInstance(v []byte) (bool, error) { - var vLoader gojsonschema.JSONLoader - vLoader = gojsonschema.NewBytesLoader(v) - r, err := gojsonschema.Validate(c.schemaLoader, vLoader) - if err != nil { - return false, err - } - return r.Valid(), nil -} - -// Sanity check -var _ Indexer = (*Collection)(nil) - // Txn represents a read/write transaction in the db. It allows for // serializable isolation level within the db. type Txn struct { @@ -297,14 +212,6 @@ func (t *Txn) Create(new ...[]byte) ([]core.InstanceID, error) { updated := make([]byte, len(new[i])) copy(updated, new[i]) - valid, err := t.collection.validInstance(updated) - if err != nil { - return nil, err - } - if !valid { - return nil, ErrInvalidSchemaInstance - } - id, err := getInstanceID(updated) if err != nil && !errors.Is(err, errMissingInstanceID) { return nil, err @@ -312,6 +219,11 @@ func (t *Txn) Create(new ...[]byte) ([]core.InstanceID, error) { if id == core.EmptyInstanceID { id, updated = setNewInstanceID(updated) } + + if err := t.collection.validInstance(updated); err != nil { + return nil, err + } + results[i] = id key := baseKey.ChildString(t.collection.name).ChildString(id.String()) exists, err := t.collection.db.datastore.Has(key) @@ -334,8 +246,7 @@ func (t *Txn) Create(new ...[]byte) ([]core.InstanceID, error) { return results, nil } -// Save saves an instance changes to be commited when the -// current transaction commits. +// Save saves an instance changes to be commited when the current transaction commits. func (t *Txn) Save(updated ...[]byte) error { for i := range updated { if t.readonly { @@ -345,13 +256,9 @@ func (t *Txn) Save(updated ...[]byte) error { item := make([]byte, len(updated[i])) copy(item, updated[i]) - valid, err := t.collection.validInstance(item) - if err != nil { + if err := t.collection.validInstance(item); err != nil { return err } - if !valid { - return ErrInvalidSchemaInstance - } id, err := getInstanceID(item) if err != nil { @@ -377,8 +284,27 @@ func (t *Txn) Save(updated ...[]byte) error { return nil } -// Delete deletes instances by ID when the current -// transaction commits. +// validInstance validates the json object against the collection schema. +func (c *Collection) validInstance(v []byte) error { + r, err := gojsonschema.Validate(c.schemaLoader, gojsonschema.NewBytesLoader(v)) + if err != nil { + return err + } + errs := r.Errors() + if len(errs) == 0 { + return nil + } + var msg string + for i, e := range errs { + msg += e.Field() + ": " + e.Description() + if i != len(errs)-1 { + msg += "; " + } + } + return fmt.Errorf("%w: %s", ErrInvalidSchemaInstance, msg) +} + +// Delete deletes instances by ID when the current transaction commits. func (t *Txn) Delete(ids ...core.InstanceID) error { for i := range ids { if t.readonly { @@ -390,7 +316,7 @@ func (t *Txn) Delete(ids ...core.InstanceID) error { return err } if !exists { - return ErrNotFound + return ErrInstanceNotFound } a := core.Action{ Type: core.Delete, @@ -404,8 +330,7 @@ func (t *Txn) Delete(ids ...core.InstanceID) error { return nil } -// Has returns true if all IDs exists in the collection, false -// otherwise. +// Has returns true if all IDs exists in the collection, false otherwise. func (t *Txn) Has(ids ...core.InstanceID) (bool, error) { for i := range ids { key := baseKey.ChildString(t.collection.name).ChildString(ids[i].String()) @@ -425,7 +350,7 @@ func (t *Txn) FindByID(id core.InstanceID) ([]byte, error) { key := baseKey.ChildString(t.collection.name).ChildString(id.String()) bytes, err := t.collection.db.datastore.Get(key) if errors.Is(err, ds.ErrNotFound) { - return nil, ErrNotFound + return nil, ErrInstanceNotFound } if err != nil { return nil, err @@ -456,12 +381,43 @@ func (t *Txn) Commit() error { return t.collection.db.notifyTxnEvents(node, t.token) } -// Discard discards all changes done in the current -// transaction. +// Discard discards all changes done in the current transaction. func (t *Txn) Discard() { t.discarded = true } +func getSchemaTypeAtPath(schema *jsonschema.Schema, pth string) (*jsonschema.Type, error) { + parts := strings.Split(pth, ".") + jt := schema.Type + for _, n := range parts { + props, err := getSchemaTypeProperties(jt, schema.Definitions) + if err != nil { + return nil, err + } + jt = props[n] + if jt == nil { + return nil, ErrInvalidCollectionSchemaPath + } + } + return jt, nil +} + +func getSchemaTypeProperties(jt *jsonschema.Type, defs jsonschema.Definitions) (map[string]*jsonschema.Type, error) { + properties := jt.Properties + if jt.Ref != "" { + parts := strings.Split(jt.Ref, "/") + if len(parts) < 1 { + return nil, ErrInvalidCollectionSchema + } + def := defs[parts[len(parts)-1]] + if def == nil { + return nil, ErrInvalidCollectionSchema + } + properties = def.Properties + } + return properties, nil +} + func getInstanceID(t []byte) (core.InstanceID, error) { partial := &struct { ID *string `json:"_id"` @@ -483,11 +439,3 @@ func setNewInstanceID(t []byte) (core.InstanceID, []byte) { } return newID, patchedValue } - -func hasIDProperty(properties map[string]*jsonschema.Type) bool { - idProperty := properties[idFieldName] - if idProperty == nil || idProperty.Type != "string" { - return false - } - return true -} diff --git a/db/collection_test.go b/db/collection_test.go index c6d994fb..eedea674 100644 --- a/db/collection_test.go +++ b/db/collection_test.go @@ -1,6 +1,7 @@ package db import ( + "encoding/json" "errors" "os" "reflect" @@ -21,11 +22,33 @@ type Person struct { Age int } +type Person2 struct { + ID core.InstanceID `json:"_id"` + Name string + Age int + Toys Toys + Comments []Comment +} + type Dog struct { ID core.InstanceID `json:"_id"` Name string Comments []Comment } + +type Dog2 struct { + ID core.InstanceID `json:"_id"` + FullName string + Breed string + Toys Toys + Comments []Comment +} + +type Toys struct { + Favorite string + Names []string +} + type Comment struct { Body string } @@ -47,6 +70,36 @@ func TestNewCollection(t *testing.T) { }) checkErr(t, err) }) + t.Run("WithIndexes", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + c, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), + Indexes: []Index{{Path: "Name", Unique: false}}, + }) + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 2 { + t.Fatalf("expected %d indexes, got %d", 2, len(indexes)) + } + }) + t.Run("WithNestedIndexes", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + c, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog2{}, false), + Indexes: []Index{{Path: "FullName", Unique: true}, {Path: "Toys.Favorite"}}, + }) + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 3 { + t.Fatalf("expected %d indexes, got %d", 3, len(indexes)) + } + }) t.Run("SingleExpandedSchemaStruct", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) @@ -111,32 +164,251 @@ func TestNewCollection(t *testing.T) { t.Fatal("the collection name should be invalid") } }) + t.Run("Fail/BadIndexPath", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + _, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), + Indexes: []Index{{Path: "Missing", Unique: false}}, + }) + if err == nil { + t.Fatal("index path should not be valid") + } + }) } -func TestAddIndex(t *testing.T) { +func TestUpdateCollection(t *testing.T) { t.Parallel() - t.Run("CreateDBAndCollection", func(t *testing.T) { + t.Run("AddFields", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ - Name: "Person", - Schema: util.SchemaFromInstance(&Person{}, false), + c, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), }) checkErr(t, err) + _, err = c.Create([]byte(`{"Name": "Fido", "Comments": []}`)) + checkErr(t, err) - t.Run("AddNameUniqueIndex", func(t *testing.T) { - err := collection.AddIndex(IndexConfig{Path: "Name", Unique: true}) - checkErr(t, err) + c, err = db.UpdateCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog2{}, false), }) - t.Run("AddAgeNonUniqueIndex", func(t *testing.T) { - err := collection.AddIndex(IndexConfig{Path: "Age", Unique: false}) - checkErr(t, err) + checkErr(t, err) + _, err = c.Create([]byte(`{"Name": "Fido", "Comments": []}`)) + if err == nil { + t.Fatal("instance should not be valid") + } + _, err = c.Create([]byte(`{"FullName": "Lassie", "Breed": "Collie", "Toys": {"Favorite": "Ball", "Names": ["Ball", "Frisbee"]}, "Comments": []}`)) + checkErr(t, err) + + dogs, err := c.Find(&Query{}) + checkErr(t, err) + if len(dogs) != 2 { + t.Fatalf("expected %d indexes, got %d", 2, len(dogs)) + } + dog1 := &Dog2{} + err = json.Unmarshal(dogs[0], dog1) + checkErr(t, err) + dog2 := &Dog2{} + err = json.Unmarshal(dogs[1], dog2) + checkErr(t, err) + }) + t.Run("AddFieldsAndIndexes", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + _, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), + Indexes: []Index{{Path: "Name", Unique: true}}, }) - t.Run("AddIDIndex", func(t *testing.T) { - err := collection.AddIndex(IndexConfig{Path: "ID", Unique: true}) - checkErr(t, err) + checkErr(t, err) + c, err := db.UpdateCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog2{}, false), + Indexes: []Index{{Path: "FullName", Unique: true}, {Path: "Toys.Favorite"}}, + }) + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 3 { + t.Fatalf("expected %d indexes, got %d", 3, len(indexes)) + } + }) + t.Run("RemoveFieldsAndIndexes", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + _, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog2{}, false), + Indexes: []Index{{Path: "FullName", Unique: true}, {Path: "Toys.Favorite"}}, + }) + checkErr(t, err) + c, err := db.UpdateCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), + Indexes: []Index{{Path: "Name", Unique: true}}, + }) + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 2 { + t.Fatalf("expected %d indexes, got %d", 2, len(indexes)) + } + }) + t.Run("Fail/BadIndexPath", func(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + _, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), }) + checkErr(t, err) + _, err = db.UpdateCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog{}, false), + Indexes: []Index{{Path: "Missing", Unique: false}}, + }) + if err == nil { + t.Fatal("index path should not be valid") + } + }) +} + +func TestDeleteCollection(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + _, err := db.NewCollection(CollectionConfig{ + Name: "Dog", + Schema: util.SchemaFromInstance(&Dog2{}, false), + Indexes: []Index{{Path: "FullName", Unique: true}}, + }) + checkErr(t, err) + err = db.DeleteCollection("Dog") + checkErr(t, err) + if db.GetCollection("Dog") != nil { + t.Fatal("collection should be deleted") + } +} + +func TestAddIndex(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + schema := util.SchemaFromInstance(&Person2{}, false) + c, err := db.NewCollection(CollectionConfig{ + Name: "Person", + Schema: schema, + }) + checkErr(t, err) + + t.Run("AddNameUniqueIndex", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Name", Unique: false}) + checkErr(t, err) + }) + t.Run("AddAgeNonUniqueIndex", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Age", Unique: false}) + checkErr(t, err) + }) + t.Run("AddNestedIndex", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Toys.Favorite", Unique: false}) + checkErr(t, err) + }) + t.Run("Fail/AddIndexWithBadPath", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Bad.Path", Unique: false}) + if err == nil { + t.Fatal("index path should not be valid") + } + }) + t.Run("Fail/AddIndexOnRef", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Toys", Unique: false}) + if err == nil { + t.Fatal("index path should not be valid") + } + }) + t.Run("Fail/AddIndexOnBadType", func(t *testing.T) { + err := c.addIndex(schema, Index{Path: "Comments", Unique: false}) + if err == nil { + t.Fatal("index path should not be valid") + } + }) + t.Run("Fail/AddUniqueIndexOnNonUniquePath", func(t *testing.T) { + _, err := c.Create(util.JSONFromInstance(Person2{Name: "Foo", Age: 42, Toys: Toys{Favorite: "", Names: []string{}}, Comments: []Comment{}})) + checkErr(t, err) + _, err = c.Create(util.JSONFromInstance(Person2{Name: "Foo", Age: 42, Toys: Toys{Favorite: "", Names: []string{}}, Comments: []Comment{}})) + checkErr(t, err) + err = c.addIndex(schema, Index{Path: "Name", Unique: true}) + if err == nil { + t.Fatal("index path should not be valid") + } + }) +} + +func TestGetIndexes(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + schema := util.SchemaFromInstance(&Person2{}, false) + c, err := db.NewCollection(CollectionConfig{ + Name: "Person", + Schema: schema, + }) + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 1 { + t.Fatalf("expected %d indexes, got %d", 1, len(indexes)) + } + if !indexes[0].Unique { + t.Fatal("index on _id should be unique") + } + err = c.addIndex(schema, Index{Path: "Name", Unique: false}) + checkErr(t, err) + indexes = c.GetIndexes() + if len(indexes) != 2 { + t.Fatalf("expected %d indexes, got %d", 2, len(indexes)) + } + if indexes[1].Unique { + t.Fatal("index on Name should not be unique") + } +} + +func TestDropIndex(t *testing.T) { + t.Parallel() + db, clean := createTestDB(t) + defer clean() + schema := util.SchemaFromInstance(&Person2{}, false) + c, err := db.NewCollection(CollectionConfig{ + Name: "Person", + Schema: schema, + }) + checkErr(t, err) + err = c.addIndex(schema, Index{Path: "Name", Unique: true}) + checkErr(t, err) + err = c.addIndex(schema, Index{Path: "Age", Unique: false}) + checkErr(t, err) + err = c.addIndex(schema, Index{Path: "Toys.Favorite", Unique: false}) + checkErr(t, err) + + t.Run("DropIndex", func(t *testing.T) { + err := c.dropIndex("Age") + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 3 { + t.Fatalf("expected %d indexes, got %d", 3, len(indexes)) + } + }) + t.Run("DropNestedIndex", func(t *testing.T) { + err := c.dropIndex("Toys.Favorite") + checkErr(t, err) + indexes := c.GetIndexes() + if len(indexes) != 2 { + t.Fatalf("expected %d indexes, got %d", 2, len(indexes)) + } }) } @@ -146,7 +418,7 @@ func TestCreateInstance(t *testing.T) { t.Parallel() db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -154,28 +426,28 @@ func TestCreateInstance(t *testing.T) { t.Run("WithImplicitTx", func(t *testing.T) { newPerson := util.JSONFromInstance(Person{Name: "Foo", Age: 42}) - res, err := collection.Create(newPerson) + res, err := c.Create(newPerson) checkErr(t, err) newPerson = util.SetJSONID(res, newPerson) - assertPersonInCollection(t, collection, newPerson) + assertPersonInCollection(t, c, newPerson) }) t.Run("WithTx", func(t *testing.T) { newPerson := util.JSONFromInstance(Person{Name: "Foo", Age: 42}) var res []core.InstanceID - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { res, err = txn.Create(newPerson) return }) checkErr(t, err) newPerson = util.SetJSONID(res[0], newPerson) - assertPersonInCollection(t, collection, newPerson) + assertPersonInCollection(t, c, newPerson) }) }) t.Run("Multiple", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -185,7 +457,7 @@ func TestCreateInstance(t *testing.T) { newPerson2 := util.JSONFromInstance(Person{Name: "Foo2", Age: 43}) var res1 []core.InstanceID var res2 []core.InstanceID - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { res1, err = txn.Create(newPerson1) if err != nil { return @@ -196,14 +468,14 @@ func TestCreateInstance(t *testing.T) { checkErr(t, err) newPerson1 = util.SetJSONID(res1[0], newPerson1) newPerson2 = util.SetJSONID(res2[0], newPerson2) - assertPersonInCollection(t, collection, newPerson1) - assertPersonInCollection(t, collection, newPerson2) + assertPersonInCollection(t, c, newPerson1) + assertPersonInCollection(t, c, newPerson2) }) t.Run("WithDefinedID", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -211,15 +483,15 @@ func TestCreateInstance(t *testing.T) { definedID := core.NewInstanceID() newPerson := util.JSONFromInstance(&Person{ID: definedID, Name: "Foo1", Age: 42}) - _, err = collection.Create(newPerson) + _, err = c.Create(newPerson) checkErr(t, err) - exists, err := collection.Has(definedID) + exists, err := c.Has(definedID) checkErr(t, err) if !exists { t.Fatal("manually defined instance ID should exist") } - assertPersonInCollection(t, collection, newPerson) + assertPersonInCollection(t, c, newPerson) }) t.Run("Re-Create", func(t *testing.T) { t.Parallel() @@ -244,7 +516,6 @@ func TestCreateInstance(t *testing.T) { func TestReadTxnValidation(t *testing.T) { t.Parallel() - t.Run("TryCreate", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) @@ -355,7 +626,7 @@ func TestGetInstance(t *testing.T) { db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -363,7 +634,7 @@ func TestGetInstance(t *testing.T) { newPerson := util.JSONFromInstance(Person{Name: "Foo", Age: 42}) var res []core.InstanceID - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { res, err = txn.Create(newPerson) return }) @@ -373,7 +644,7 @@ func TestGetInstance(t *testing.T) { util.InstanceFromJSON(newPerson, newPersonInstance) t.Run("WithImplicitTx", func(t *testing.T) { - found, err := collection.FindByID(res[0]) + found, err := c.FindByID(res[0]) checkErr(t, err) foundInstance := &Person{} @@ -385,7 +656,7 @@ func TestGetInstance(t *testing.T) { }) t.Run("WithReadTx", func(t *testing.T) { var found []byte - err = collection.ReadTxn(func(txn *Txn) (err error) { + err = c.ReadTxn(func(txn *Txn) (err error) { found, err = txn.FindByID(res[0]) return }) @@ -400,7 +671,7 @@ func TestGetInstance(t *testing.T) { }) t.Run("WithWriteTx", func(t *testing.T) { var found []byte - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { found, err = txn.FindByID(res[0]) return }) @@ -417,12 +688,11 @@ func TestGetInstance(t *testing.T) { func TestSaveInstance(t *testing.T) { t.Parallel() - t.Run("Simple", func(t *testing.T) { t.Parallel() db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -430,13 +700,13 @@ func TestSaveInstance(t *testing.T) { newPerson := util.JSONFromInstance(Person{Name: "Alice", Age: 42}) var res []core.InstanceID - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { res, err = txn.Create(newPerson) return }) checkErr(t, err) - err = collection.WriteTxn(func(txn *Txn) error { + err = c.WriteTxn(func(txn *Txn) error { instance, err := txn.FindByID(res[0]) checkErr(t, err) @@ -448,7 +718,7 @@ func TestSaveInstance(t *testing.T) { }) checkErr(t, err) - instance, err := collection.FindByID(res[0]) + instance, err := c.FindByID(res[0]) checkErr(t, err) person := &Person{} util.InstanceFromJSON(instance, person) @@ -475,10 +745,9 @@ func TestSaveInstance(t *testing.T) { func TestDeleteInstance(t *testing.T) { t.Parallel() - db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) @@ -486,25 +755,25 @@ func TestDeleteInstance(t *testing.T) { newPerson := util.JSONFromInstance(Person{Name: "Alice", Age: 42}) var res []core.InstanceID - err = collection.WriteTxn(func(txn *Txn) (err error) { + err = c.WriteTxn(func(txn *Txn) (err error) { res, err = txn.Create(newPerson) return }) checkErr(t, err) - err = collection.Delete(res[0]) + err = c.Delete(res[0]) checkErr(t, err) - _, err = collection.FindByID(res[0]) - if err != ErrNotFound { + _, err = c.FindByID(res[0]) + if err != ErrInstanceNotFound { t.Fatalf("FindByID: instance shouldn't exist") } - if exist, err := collection.Has(res[0]); exist || err != nil { + if exist, err := c.Has(res[0]); exist || err != nil { t.Fatalf("Has: instance shouldn't exist") } // Try to delete again - if err = collection.Delete(res[0]); err != ErrNotFound { + if err = c.Delete(res[0]); err != ErrInstanceNotFound { t.Fatalf("cant't delete non-existent instance") } } @@ -516,36 +785,35 @@ type PersonFake struct { func TestInvalidActions(t *testing.T) { t.Parallel() - db, clean := createTestDB(t) defer clean() - collection, err := db.NewCollection(CollectionConfig{ + c, err := db.NewCollection(CollectionConfig{ Name: "Person", Schema: util.SchemaFromInstance(&Person{}, false), }) checkErr(t, err) t.Run("Create", func(t *testing.T) { f := util.JSONFromInstance(PersonFake{Name: "fake"}) - if _, err := collection.Create(f); !errors.Is(err, ErrInvalidSchemaInstance) { + if _, err := c.Create(f); !errors.Is(err, ErrInvalidSchemaInstance) { t.Fatalf("instance should be invalid compared to schema, got: %v", err) } }) t.Run("Save", func(t *testing.T) { r := util.JSONFromInstance(Person{Name: "real"}) - _, err := collection.Create(r) + _, err := c.Create(r) checkErr(t, err) f := util.JSONFromInstance(PersonFake{Name: "fake"}) - if err := collection.Save(f); !errors.Is(err, ErrInvalidSchemaInstance) { + if err := c.Save(f); !errors.Is(err, ErrInvalidSchemaInstance) { t.Fatalf("instance should be invalid compared to schema, got: %v", err) } }) } -func assertPersonInCollection(t *testing.T, collection *Collection, personBytes []byte) { +func assertPersonInCollection(t *testing.T, c *Collection, personBytes []byte) { t.Helper() person := &Person{} util.InstanceFromJSON(personBytes, person) - res, err := collection.FindByID(person.ID) + res, err := c.FindByID(person.ID) checkErr(t, err) p := &Person{} util.InstanceFromJSON(res, p) diff --git a/db/common.go b/db/common.go index 7234c0ba..54ca98b3 100644 --- a/db/common.go +++ b/db/common.go @@ -33,7 +33,7 @@ type op struct { } // SimpleTx implements the transaction interface for datastores who do -// not have any sort of underlying transactional support +// not have any sort of underlying transactional support. type SimpleTx struct { ops map[datastore.Key]op lock sync.RWMutex diff --git a/db/db.go b/db/db.go index 56a39828..f795afb0 100644 --- a/db/db.go +++ b/db/db.go @@ -39,9 +39,10 @@ var ( // ErrInvalidCollectionSchema indicates the provided schema isn't valid for a Collection. ErrInvalidCollectionSchema = errors.New("the collection schema should specify an _id string property") - // ErrInvalidCollectionName indicates the provided name isn't valid for a Collection. ErrInvalidCollectionName = errors.New("collection name may only contain alphanumeric characters or non-consecutive hyphens, and cannot begin or end with a hyphen") + // ErrCannotIndexIDField indicates a custom index was specified on the ID field. + ErrCannotIndexIDField = errors.New("cannot create custom index on " + idFieldName) collectionNameRx *regexp.Regexp @@ -67,9 +68,10 @@ type DB struct { dispatcher *dispatcher eventcodec core.EventCodec - lock sync.RWMutex - collectionNames map[string]*Collection - closed bool + lock sync.RWMutex + txnlock sync.RWMutex + collections map[string]*Collection + closed bool localEventsBus *app.LocalEventsBus stateChangedNotifee *stateChangedNotifee @@ -77,8 +79,8 @@ type DB struct { // NewDB creates a new DB, which will *own* ds and dispatcher for internal use. // Saying it differently, ds and dispatcher shouldn't be used externally. -func NewDB(ctx context.Context, network app.Net, id thread.ID, opts ...NewDBOption) (*DB, error) { - options := &NewDBOptions{} +func NewDB(ctx context.Context, network app.Net, id thread.ID, opts ...NewOption) (*DB, error) { + options := &NewOptions{} for _, opt := range opts { if err := opt(options); err != nil { return nil, err @@ -96,8 +98,8 @@ func NewDB(ctx context.Context, network app.Net, id thread.ID, opts ...NewDBOpti // NewDBFromAddr creates a new DB from a thread hosted by another peer at address, // which will *own* ds and dispatcher for internal use. // Saying it differently, ds and dispatcher shouldn't be used externally. -func NewDBFromAddr(ctx context.Context, network app.Net, addr ma.Multiaddr, key thread.Key, opts ...NewDBOption) (*DB, error) { - options := &NewDBOptions{} +func NewDBFromAddr(ctx context.Context, network app.Net, addr ma.Multiaddr, key thread.Key, opts ...NewOption) (*DB, error) { + options := &NewOptions{} for _, opt := range opts { if err := opt(options); err != nil { return nil, err @@ -121,9 +123,8 @@ func NewDBFromAddr(ctx context.Context, network app.Net, addr ma.Multiaddr, key return d, nil } -// newDB is used directly by a db manager to create new dbs -// with the same config. -func newDB(n app.Net, id thread.ID, options *NewDBOptions) (*DB, error) { +// newDB is used directly by a db manager to create new dbs with the same config. +func newDB(n app.Net, id thread.ID, options *NewOptions) (*DB, error) { if options.Datastore == nil { datastore, err := newDefaultDatastore(options.RepoPath, options.LowMem) if err != nil { @@ -148,7 +149,7 @@ func newDB(n app.Net, id thread.ID, options *NewDBOptions) (*DB, error) { datastore: options.Datastore, dispatcher: newDispatcher(options.Datastore), eventcodec: options.EventCodec, - collectionNames: make(map[string]*Collection), + collections: make(map[string]*Collection), localEventsBus: app.NewLocalEventsBus(), stateChangedNotifee: &stateChangedNotifee{}, } @@ -168,12 +169,21 @@ func newDB(n app.Net, id thread.ID, options *NewDBOptions) (*DB, error) { log.Fatalf("unable to connect app: %s", err) } d.connector = connector - return d, nil } +// managedDatastore returns whether or not the datastore is +// being wrapped by an external datastore. +func managedDatastore(ds ds.Datastore) bool { + _, ok := ds.(kt.KeyTransform) + return ok +} + // reCreateCollections loads and registers schemas from the datastore. func (d *DB) reCreateCollections() error { + d.lock.Lock() + defer d.lock.Unlock() + results, err := d.datastore.Query(query.Query{ Prefix: dsDBSchemas.String(), }) @@ -181,141 +191,180 @@ func (d *DB) reCreateCollections() error { return err } defer results.Close() - for res := range results.Next() { name := ds.RawKey(res.Key).Name() - schema := &jsonschema.Schema{} if err := json.Unmarshal(res.Value, schema); err != nil { return err } - - var indexes map[string]IndexConfig + c, err := newCollection(name, schema, d) + if err != nil { + return err + } + var indexes map[string]Index index, err := d.datastore.Get(dsDBIndexes.ChildString(name)) if err == nil && index != nil { - _ = json.Unmarshal(index, &indexes) - } - - indexValues := make([]IndexConfig, len(indexes)) - for _, value := range indexes { - indexValues = append(indexValues, value) + if err := json.Unmarshal(index, &indexes); err != nil { + return err + } } - - if _, err := d.NewCollection(CollectionConfig{ - Name: name, - Schema: schema, - Indexes: indexValues, - }); err != nil { - return err + for _, index := range indexes { + if index.Path != "" { // Catch bad indexes from an old bug + c.indexes[index.Path] = index + } } + d.collections[c.name] = c } return nil } +// GetDBInfo returns the addresses and key that can be used to join the DB thread. +func (d *DB) GetDBInfo(opts ...Option) ([]ma.Multiaddr, thread.Key, error) { + options := &Options{} + for _, opt := range opts { + opt(options) + } + + tinfo, err := d.connector.Net.GetThread(context.Background(), d.connector.ThreadID(), net.WithThreadToken(options.Token)) + if err != nil { + return nil, thread.Key{}, err + } + return tinfo.Addrs, tinfo.Key, nil +} + // CollectionConfig describes a new Collection. type CollectionConfig struct { Name string Schema *jsonschema.Schema - Indexes []IndexConfig + Indexes []Index } -// NewCollection creates a new collection in the db with a JSON schema. -func (d *DB) NewCollection(config CollectionConfig) (*Collection, error) { +// NewCollection creates a new db collection with config. +// @todo: Handle token auth +func (d *DB) NewCollection(config CollectionConfig, opts ...Option) (*Collection, error) { d.lock.Lock() defer d.lock.Unlock() - if _, ok := d.collectionNames[config.Name]; ok { - return nil, fmt.Errorf("already registered collection") + if _, ok := d.collections[config.Name]; ok { + return nil, ErrCollectionAlreadyRegistered } - c, err := newCollection(config.Name, config.Schema, d) if err != nil { return nil, err } - key := dsDBSchemas.ChildString(config.Name) - exists, err := d.datastore.Has(key) - if err != nil { + if err := d.addIndexes(c, config.Schema, config.Indexes, opts...); err != nil { return nil, err } - if !exists { - schemaBytes, err := json.Marshal(config.Schema) - if err != nil { - return nil, err - } - if err := d.datastore.Put(key, schemaBytes); err != nil { - return nil, err - } + if err := d.saveCollection(c); err != nil { + return nil, err } + return c, nil +} + +// UpdateCollection updates an existing db collection with a new config. +// Indexes to new paths will be created. +// Indexes to removed paths will be dropped. +// @todo: Handle token auth +func (d *DB) UpdateCollection(config CollectionConfig, opts ...Option) (*Collection, error) { + d.lock.Lock() + defer d.lock.Unlock() - if err := c.AddIndex(IndexConfig{Path: idFieldName, Unique: true}); err != nil { + xc, ok := d.collections[config.Name] + if !ok { + return nil, ErrCollectionNotFound + } + c, err := newCollection(config.Name, config.Schema, d) + if err != nil { + return nil, err + } + if err := d.addIndexes(c, config.Schema, config.Indexes, opts...); err != nil { return nil, err } - for _, cfg := range config.Indexes { - // @todo: Should check to make sure this is a valid field path for this schema - if err := c.AddIndex(cfg); err != nil { - return nil, err + // Drop indexes that are no longer requested + for _, index := range xc.indexes { + if _, ok := c.indexes[index.Path]; !ok { + if err := c.dropIndex(index.Path, opts...); err != nil { + return nil, err + } } } - d.collectionNames[config.Name] = c + if err := d.saveCollection(c); err != nil { + return nil, err + } return c, nil } -// GetCollection returns a collection by name. -func (d *DB) GetCollection(name string) *Collection { - return d.collectionNames[name] +func (d *DB) addIndexes(c *Collection, schema *jsonschema.Schema, indexes []Index, opts ...Option) error { + for _, index := range indexes { + if index.Path == idFieldName { + return ErrCannotIndexIDField + } + if err := c.addIndex(schema, index, opts...); err != nil { + return err + } + } + return c.addIndex(schema, Index{Path: idFieldName, Unique: true}, opts...) } -// Reduce processes txn events into the collections. -func (d *DB) Reduce(events []core.Event) error { - codecActions, err := d.eventcodec.Reduce( - events, - d.datastore, - baseKey, - defaultIndexFunc(d), - ) - if err != nil { +func (d *DB) saveCollection(c *Collection) error { + schema := c.schemaLoader.JsonSource().([]byte) + if err := d.datastore.Put(dsDBSchemas.ChildString(c.name), schema); err != nil { return err } - actions := make([]Action, len(codecActions)) - for i, ca := range codecActions { - var actionType ActionType - switch codecActions[i].Type { - case core.Create: - actionType = ActionCreate - case core.Save: - actionType = ActionSave - case core.Delete: - actionType = ActionDelete - default: - panic("eventcodec action not recognized") - } - actions[i] = Action{Collection: ca.Collection, Type: actionType, ID: ca.InstanceID} - } - d.notifyStateChanged(actions) - + d.collections[c.name] = c return nil } -// GetDBInfo returns the addresses and key that can be used to join the DB thread -func (d *DB) GetDBInfo(opts ...InviteInfoOption) ([]ma.Multiaddr, thread.Key, error) { - options := &InviteInfoOptions{} +// GetCollection returns a collection by name. +// @todo: Handle token auth +func (d *DB) GetCollection(name string, opts ...Option) *Collection { + d.lock.Lock() + defer d.lock.Unlock() + options := &Options{} for _, opt := range opts { opt(options) } + return d.collections[name] +} - tinfo, err := d.connector.Net.GetThread(context.Background(), d.connector.ThreadID(), net.WithThreadToken(options.Token)) +// DeleteCollection deletes collection by name and drops all indexes. +// @todo: Handle token auth +func (d *DB) DeleteCollection(name string, opts ...Option) error { + d.lock.Lock() + defer d.lock.Unlock() + options := &Options{} + for _, opt := range opts { + opt(options) + } + + c, ok := d.collections[name] + if !ok { + return ErrCollectionNotFound + } + txn, err := d.datastore.NewTransaction(false) if err != nil { - return nil, thread.Key{}, err + return err } - return tinfo.Addrs, tinfo.Key, nil + if err := txn.Delete(dsDBIndexes.ChildString(c.name)); err != nil { + return err + } + if err := txn.Delete(dsDBSchemas.ChildString(c.name)); err != nil { + return err + } + if err := txn.Commit(); err != nil { + return err + } + delete(d.collections, c.name) + return nil } -// Close closes the db. func (d *DB) Close() error { d.lock.Lock() defer d.lock.Unlock() + d.txnlock.Lock() + defer d.txnlock.Unlock() if d.closed { return nil @@ -337,6 +386,43 @@ func (d *DB) Close() error { return nil } +func (d *DB) Reduce(events []core.Event) error { + codecActions, err := d.eventcodec.Reduce(events, d.datastore, baseKey, defaultIndexFunc(d)) + if err != nil { + return err + } + actions := make([]Action, len(codecActions)) + for i, ca := range codecActions { + var actionType ActionType + switch codecActions[i].Type { + case core.Create: + actionType = ActionCreate + case core.Save: + actionType = ActionSave + case core.Delete: + actionType = ActionDelete + default: + panic("eventcodec action not recognized") + } + actions[i] = Action{Collection: ca.Collection, Type: actionType, ID: ca.InstanceID} + } + d.notifyStateChanged(actions) + return nil +} + +func defaultIndexFunc(d *DB) func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error { + return func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error { + c := d.GetCollection(collection) + if err := c.indexDelete(txn, key, oldData); err != nil { + return err + } + if newData == nil { + return nil + } + return c.indexAdd(txn, key, newData) + } +} + func (d *DB) HandleNetRecord(rec net.ThreadRecord, key thread.Key, lid peer.ID, timeout time.Duration) error { if rec.LogID() == lid { return nil // Ignore our own events since DB already dispatches to DB reducers @@ -385,8 +471,8 @@ func (d *DB) getBlockWithRetry(ctx context.Context, rec net.Record) (format.Node // dispatch applies external events to the db. This function guarantee // no interference with registered collection states, and viceversa. func (d *DB) dispatch(events []core.Event) error { - d.lock.Lock() - defer d.lock.Unlock() + d.txnlock.Lock() + defer d.txnlock.Unlock() return d.dispatcher.Dispatch(events) } @@ -396,16 +482,9 @@ func (d *DB) eventsFromBytes(data []byte) ([]core.Event, error) { return d.eventcodec.EventsFromBytes(data) } -// managedDatastore returns whether or not the datastore is -// being wrapped by an external datastore. -func managedDatastore(ds ds.Datastore) bool { - _, ok := ds.(kt.KeyTransform) - return ok -} - func (d *DB) readTxn(c *Collection, f func(txn *Txn) error, opts ...TxnOption) error { - d.lock.RLock() - defer d.lock.RUnlock() + d.txnlock.RLock() + defer d.txnlock.RUnlock() args := &TxnOptions{} for _, opt := range opts { @@ -420,8 +499,8 @@ func (d *DB) readTxn(c *Collection, f func(txn *Txn) error, opts ...TxnOption) e } func (d *DB) writeTxn(c *Collection, f func(txn *Txn) error, opts ...TxnOption) error { - d.lock.Lock() - defer d.lock.Unlock() + d.txnlock.Lock() + defer d.txnlock.Unlock() args := &TxnOptions{} for _, opt := range opts { @@ -434,18 +513,3 @@ func (d *DB) writeTxn(c *Collection, f func(txn *Txn) error, opts ...TxnOption) } return txn.Commit() } - -func defaultIndexFunc(s *DB) func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error { - return func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error { - indexer := s.GetCollection(collection) - if err := indexDelete(indexer, txn, key, oldData); err != nil { - return err - } - if newData != nil { - if err := indexAdd(indexer, txn, key, newData); err != nil { - return err - } - } - return nil - } -} diff --git a/db/db_test.go b/db/db_test.go index f1f735f0..2ddcdc6b 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -31,7 +31,7 @@ func TestE2EWithThreads(t *testing.T) { defer n1.Close() id1 := thread.NewIDV1(thread.Raw, 32) - d1, err := NewDB(context.Background(), n1, id1, WithNewDBRepoPath(tmpDir1)) + d1, err := NewDB(context.Background(), n1, id1, WithNewRepoPath(tmpDir1)) checkErr(t, err) defer d1.Close() c1, err := d1.NewCollection(CollectionConfig{ @@ -70,7 +70,7 @@ func TestE2EWithThreads(t *testing.T) { Name: "dummy", Schema: util.SchemaFromInstance(&dummy{}, false), } - d2, err := NewDBFromAddr(context.Background(), n2, addr, ti.Key, WithNewDBRepoPath(tmpDir2), WithNewDBCollections(cc)) + d2, err := NewDBFromAddr(context.Background(), n2, addr, ti.Key, WithNewRepoPath(tmpDir2), WithNewCollections(cc)) checkErr(t, err) defer d2.Close() c2 := d1.GetCollection("dummy") @@ -102,7 +102,7 @@ func TestOptions(t *testing.T) { ec := &mockEventCodec{} id := thread.NewIDV1(thread.Raw, 32) - d, err := NewDB(context.Background(), n, id, WithNewDBRepoPath(tmpDir), WithNewDBEventCodec(ec)) + d, err := NewDB(context.Background(), n, id, WithNewRepoPath(tmpDir), WithNewEventCodec(ec)) checkErr(t, err) m, err := d.NewCollection(CollectionConfig{ @@ -125,7 +125,7 @@ func TestOptions(t *testing.T) { n, err = common.DefaultNetwork(tmpDir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(t, err) defer n.Close() - d, err = NewDB(context.Background(), n, id, WithNewDBRepoPath(tmpDir), WithNewDBEventCodec(ec)) + d, err = NewDB(context.Background(), n, id, WithNewRepoPath(tmpDir), WithNewEventCodec(ec)) checkErr(t, err) checkErr(t, d.Close()) } @@ -352,7 +352,7 @@ type mockEventCodec struct { var _ core.EventCodec = (*mockEventCodec)(nil) -func (dec *mockEventCodec) Reduce([]core.Event, ds.TxnDatastore, ds.Key, func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error) ([]core.ReduceAction, error) { +func (dec *mockEventCodec) Reduce([]core.Event, ds.TxnDatastore, ds.Key, core.IndexFunc) ([]core.ReduceAction, error) { dec.called = true return nil, nil } diff --git a/db/dispatcher.go b/db/dispatcher.go index 8bc99030..946d5eab 100644 --- a/db/dispatcher.go +++ b/db/dispatcher.go @@ -35,7 +35,7 @@ type dispatcher struct { lastID int } -// NewDispatcher creates a new EventDispatcher +// NewDispatcher creates a new EventDispatcher. func newDispatcher(store datastore.TxnDatastore) *dispatcher { return &dispatcher{ store: store, @@ -47,7 +47,7 @@ func (d *dispatcher) Store() datastore.Datastore { return d.store } -// Register takes a reducer to be invoked with each dispatched event +// Register takes a reducer to be invoked with each dispatched event. func (d *dispatcher) Register(reducer Reducer) { d.lock.Lock() defer d.lock.Unlock() @@ -103,7 +103,7 @@ func (d *dispatcher) Dispatch(events []core.Event) error { } // Query searches the internal event store and returns a query result. -// This is a syncronouse version of github.com/ipfs/go-datastore's Query method +// This is a syncronouse version of github.com/ipfs/go-datastore's Query method. func (d *dispatcher) Query(query query.Query) ([]query.Entry, error) { result, err := d.store.Query(query) if err != nil { diff --git a/db/encode.go b/db/encode.go index 21c60640..ba8e4dab 100644 --- a/db/encode.go +++ b/db/encode.go @@ -9,13 +9,13 @@ import ( "encoding/gob" ) -// EncodeFunc is a function for encoding a value into bytes +// EncodeFunc is a function for encoding a value into bytes. type EncodeFunc func(value interface{}) ([]byte, error) -// DecodeFunc is a function for decoding a value from bytes +// DecodeFunc is a function for decoding a value from bytes. type DecodeFunc func(data []byte, value interface{}) error -// DefaultEncode is the default encoding func from badgerhold (Gob) +// DefaultEncode is the default encoding func from badgerhold (Gob). func DefaultEncode(value interface{}) ([]byte, error) { var buff bytes.Buffer @@ -29,7 +29,7 @@ func DefaultEncode(value interface{}) ([]byte, error) { return buff.Bytes(), nil } -// DefaultDecode is the default decoding func from badgerhold (Gob) +// DefaultDecode is the default decoding func from badgerhold (Gob). func DefaultDecode(data []byte, value interface{}) error { var buff bytes.Buffer de := gob.NewDecoder(&buff) diff --git a/db/index.go b/db/index.go index 1f868d09..b7d22620 100644 --- a/db/index.go +++ b/db/index.go @@ -11,120 +11,215 @@ import ( "fmt" "sort" + "github.com/alecthomas/jsonschema" ds "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) -// size of iterator keys stored in memory before more are fetched const ( + // iteratorKeyMinCacheSize is the size of iterator keys stored in memory before more are fetched. iteratorKeyMinCacheSize = 100 ) -// ErrUniqueExists is the error thrown when data is being inserted for a unique constraint value that already exists var ( - indexPrefix = ds.NewKey("_index") + // ErrUniqueExists indicates an insert resulted in a unique constraint violation. ErrUniqueExists = errors.New("unique constraint violation") + // ErrNotIndexable indicates an index path does not resolve to a value. ErrNotIndexable = errors.New("value not indexable") - ErrNoIndexFound = errors.New("no index found") -) + // ErrCantCreateUniqueIndex indicates a unique index can't be created because multiple instances share a value at path. + ErrCantCreateUniqueIndex = errors.New("can't create unique index (duplicate instances exist)") + // ErrIndexNotFound indicates a requested index was not found. + ErrIndexNotFound = errors.New("index not found") -// Indexer is the interface to implement to support Collection indexes -type Indexer interface { - BaseKey() ds.Key - Indexes() map[string]Index //[indexname]indexFunc -} + indexPrefix = ds.NewKey("_index") + indexTypes = []string{"string", "number", "integer", "boolean"} +) -// Index is a function that returns the indexable, encoded bytes of the passed in bytes +// Index defines an index. type Index struct { - IndexFunc func(name string, value []byte) (ds.Key, error) - Unique bool + // Path to the field to index in dot syntax, e.g., "name.last" or "age". + Path string `json:"path"` + // Unique indicates that only one instance should exist per field value. + Unique bool `json:"unique,omitempty"` } -// IndexConfig stores the configuration for a given Index. -type IndexConfig struct { - Path string `json:"path"` - Unique bool `json:"unique,omitempty"` +// GetIndexes returns the current indexes. +func (c *Collection) GetIndexes() []Index { + indexes := make([]Index, len(c.indexes)) + var i int + for _, index := range c.indexes { + indexes[i] = index + i++ + } + return indexes } -// adds an item to the index -func indexAdd(indexer Indexer, tx ds.Txn, key ds.Key, data []byte) error { - indexes := indexer.Indexes() - for path, index := range indexes { - err := indexUpdate(indexer.BaseKey(), path, index, tx, key, data, false) +// addIndex creates a new index based on path. +// Use dot syntax to reach nested fields, e.g., "name.last". +// The field at path must be one of the supported JSON Schema types: string, number, integer, or boolean +// Set unique to true if you want a unique constraint on path. +// Adding an index will override any overlapping index values if they already exist. +// @note: This does NOT currently build the index. If items have been added prior to adding +// a new index, they will NOT be indexed a posteriori. +// @todo: Handle token auth +func (c *Collection) addIndex(schema *jsonschema.Schema, index Index, opts ...Option) error { + options := &Options{} + for _, opt := range opts { + opt(options) + } + + // Don't allow the default index to be overwritten + if index.Path == idFieldName { + if _, ok := c.indexes[idFieldName]; ok { + return nil + } + } + + // Validate path and type. + jt, err := getSchemaTypeAtPath(schema, index.Path) + if err != nil { + return err + } + var valid bool + for _, t := range indexTypes { + if jt.Type == t { + valid = true + break + } + } + if !valid { + return ErrNotIndexable + } + + // Skip if nothing to do + if x, ok := c.indexes[index.Path]; ok && index.Unique == x.Unique { + return nil + } + + // Ensure collection does not contain multiple instances with the same value at path + if index.Unique && index.Path != idFieldName { + vals := make(map[interface{}]struct{}) + all, err := c.Find(&Query{}, WithTxnToken(options.Token)) if err != nil { return err } + for _, i := range all { + res := gjson.GetBytes(i, index.Path) + if !res.Exists() { + continue + } + if _, ok := vals[res.Value()]; ok { + return ErrCantCreateUniqueIndex + } else { + vals[res.Value()] = struct{}{} + } + } } - return nil + c.indexes[index.Path] = index + return c.saveIndexes() } -// removes an item from the index -// be sure to pass the data from the old record, not the new one -func indexDelete(indexer Indexer, tx ds.Txn, key ds.Key, originalData []byte) error { - indexes := indexer.Indexes() +// dropIndex drops the index at path. +// @todo: Handle token auth +func (c *Collection) dropIndex(pth string, opts ...Option) error { + options := &Options{} + for _, opt := range opts { + opt(options) + } - for path, index := range indexes { - err := indexUpdate(indexer.BaseKey(), path, index, tx, key, originalData, true) + // Don't allow the default index to be dropped + if pth == idFieldName { + return errors.New(idFieldName + " index cannot be dropped") + } + delete(c.indexes, pth) + return c.saveIndexes() +} + +// saveIndexes persists the current indexes. +func (c *Collection) saveIndexes() error { + ib, err := json.Marshal(c.indexes) + if err != nil { + return err + } + return c.db.datastore.Put(dsDBIndexes.ChildString(c.name), ib) +} + +// indexAdd adds an item to the index. +func (c *Collection) indexAdd(tx ds.Txn, key ds.Key, data []byte) error { + for path, index := range c.indexes { + err := c.indexUpdate(path, index, tx, key, data, false) if err != nil { return err } } - return nil } -// adds or removes a specific index on an item -func indexUpdate(baseKey ds.Key, path string, index Index, tx ds.Txn, key ds.Key, value []byte, - delete bool) error { +// indexDelete removes an item from the index. +// Be sure to pass the data from the old record, not the new one. +func (c *Collection) indexDelete(tx ds.Txn, key ds.Key, originalData []byte) error { + for path, index := range c.indexes { + err := c.indexUpdate(path, index, tx, key, originalData, true) + if err != nil { + return err + } + } + return nil +} - valueKey, err := index.IndexFunc(path, value) - if err != nil && !errors.Is(err, ErrNotIndexable) { +// indexUpdate adds or removes a specific index on an item. +func (c *Collection) indexUpdate(field string, index Index, tx ds.Txn, key ds.Key, input []byte, delete bool) error { + valueKey, err := getIndexValue(field, input) + if err != nil { + if errors.Is(err, ErrNotIndexable) { + return nil + } return err } - if valueKey.String() == "" { - return nil - } - - indexKey := indexPrefix.Child(baseKey).ChildString(path).ChildString(valueKey.String()[1:]) + indexKey := indexPrefix.Child(c.baseKey()).ChildString(field).ChildString(valueKey.String()[1:]) data, err := tx.Get(indexKey) if err != nil && err != ds.ErrNotFound { return err } - - indexValue := make(keyList, 0) - if err != ds.ErrNotFound { if index.Unique && !delete { return ErrUniqueExists } } + indexValue := make(keyList, 0) if data != nil { err = DefaultDecode(data, &indexValue) if err != nil { return err } } - if delete { indexValue.remove(key) } else { indexValue.add(key) } - if len(indexValue) == 0 { return tx.Delete(indexKey) } - - iVal, err := DefaultEncode(indexValue) + val, err := DefaultEncode(indexValue) if err != nil { return err } - return tx.Put(indexKey, iVal) + return tx.Put(indexKey, val) +} + +// getIndexValue returns the result of a field search on input. +func getIndexValue(field string, input []byte) (ds.Key, error) { + result := gjson.GetBytes(input, field) + if !result.Exists() { + return ds.Key{}, ErrNotIndexable + } + return ds.NewKey(result.String()), nil } // keyList is a slice of unique, sorted keys([]byte) such as what an index points to @@ -164,7 +259,6 @@ func (v *keyList) in(key ds.Key) bool { i := sort.Search(len(*v), func(i int) bool { return bytes.Compare((*v)[i], b) >= 0 }) - return i < len(*v) && bytes.Equal((*v)[i], b) } @@ -213,7 +307,7 @@ func newIterator(txn ds.Txn, baseKey ds.Key, q *Query) *iterator { result, ok := i.iter.NextSync() if !ok { if first { - return nil, ErrNoIndexFound + return nil, ErrIndexNotFound } return nKeys, result.Error } @@ -250,7 +344,6 @@ func newIterator(txn ds.Txn, baseKey ds.Key, q *Query) *iterator { } return nKeys, nil } - return i } @@ -297,7 +390,6 @@ func (i *iterator) NextSync() (MarshaledResult, bool) { }, }, false } - i.keyCache = append(i.keyCache, newKeys...) } diff --git a/db/listeners.go b/db/listeners.go index a78d87fb..1802522b 100644 --- a/db/listeners.go +++ b/db/listeners.go @@ -14,8 +14,8 @@ import ( // defined filters. The DB *won't* wait for slow receivers, so if the // channel is full, the action will be dropped. func (d *DB) Listen(los ...ListenOption) (Listener, error) { - d.lock.Lock() - defer d.lock.Unlock() + d.txnlock.Lock() + defer d.txnlock.Unlock() if d.closed { return nil, fmt.Errorf("can't listen on closed DB") } diff --git a/db/manager.go b/db/manager.go index 9fe31bca..9ca06e13 100644 --- a/db/manager.go +++ b/db/manager.go @@ -2,7 +2,7 @@ package db import ( "context" - "fmt" + "errors" "io" "strings" @@ -18,21 +18,27 @@ import ( ) var ( + // ErrDBNotFound indicates that the specified db doesn't exist in the manager. + ErrDBNotFound = errors.New("db not found") + + // ErrDBExists indicates that the specified db alrady exists in the manager. + ErrDBExists = errors.New("db already exists") + dsDBManagerBaseKey = ds.NewKey("/manager") ) type Manager struct { io.Closer - newDBOptions *NewDBOptions + newDBOptions *NewOptions network app.Net dbs map[thread.ID]*DB } // NewManager hydrates and starts dbs from prefixes. -func NewManager(network app.Net, opts ...NewDBOption) (*Manager, error) { - options := &NewDBOptions{} +func NewManager(network app.Net, opts ...NewOption) (*Manager, error) { + options := &NewOptions{} for _, opt := range opts { if err := opt(options); err != nil { return nil, err @@ -99,11 +105,11 @@ func (m *Manager) GetToken(ctx context.Context, identity thread.Identity) (threa } // NewDB creates a new db and prefixes its datastore with base key. -func (m *Manager) NewDB(ctx context.Context, id thread.ID, opts ...NewManagedDBOption) (*DB, error) { +func (m *Manager) NewDB(ctx context.Context, id thread.ID, opts ...NewManagedOption) (*DB, error) { if _, ok := m.dbs[id]; ok { - return nil, fmt.Errorf("db %s already exists", id) + return nil, ErrDBExists } - args := &NewManagedDBOptions{} + args := &NewManagedOptions{} for _, opt := range opts { opt(args) } @@ -126,15 +132,15 @@ func (m *Manager) NewDB(ctx context.Context, id thread.ID, opts ...NewManagedDBO // NewDBFromAddr creates a new db from address and prefixes its datastore with base key. // Unlike NewDB, this method takes a list of collections added to the original db that // should also be added to this host. -func (m *Manager) NewDBFromAddr(ctx context.Context, addr ma.Multiaddr, key thread.Key, opts ...NewManagedDBOption) (*DB, error) { +func (m *Manager) NewDBFromAddr(ctx context.Context, addr ma.Multiaddr, key thread.Key, opts ...NewManagedOption) (*DB, error) { id, err := thread.FromAddr(addr) if err != nil { return nil, err } if _, ok := m.dbs[id]; ok { - return nil, fmt.Errorf("db %s already exists", id) + return nil, ErrDBExists } - args := &NewManagedDBOptions{} + args := &NewManagedOptions{} for _, opt := range opts { opt(args) } @@ -162,8 +168,8 @@ func (m *Manager) NewDBFromAddr(ctx context.Context, addr ma.Multiaddr, key thre } // GetDB returns a db by id. -func (m *Manager) GetDB(ctx context.Context, id thread.ID, opts ...ManagedDBOption) (*DB, error) { - args := &ManagedDBOptions{} +func (m *Manager) GetDB(ctx context.Context, id thread.ID, opts ...ManagedOption) (*DB, error) { + args := &ManagedOptions{} for _, opt := range opts { opt(args) } @@ -174,8 +180,8 @@ func (m *Manager) GetDB(ctx context.Context, id thread.ID, opts ...ManagedDBOpti } // DeleteDB deletes a db by id. -func (m *Manager) DeleteDB(ctx context.Context, id thread.ID, opts ...ManagedDBOption) error { - args := &ManagedDBOptions{} +func (m *Manager) DeleteDB(ctx context.Context, id thread.ID, opts ...ManagedOption) error { + args := &ManagedOptions{} for _, opt := range opts { opt(args) } @@ -233,11 +239,11 @@ func (m *Manager) Close() error { // getDBOptions copies the manager's base config, // wraps the datastore with an id prefix, // and merges specified collection configs with those from base -func getDBOptions(id thread.ID, base *NewDBOptions, collections ...CollectionConfig) (*NewDBOptions, error) { +func getDBOptions(id thread.ID, base *NewOptions, collections ...CollectionConfig) (*NewOptions, error) { if err := id.Validate(); err != nil { return nil, err } - return &NewDBOptions{ + return &NewOptions{ RepoPath: base.RepoPath, Datastore: wrapTxnDatastore(base.Datastore, kt.PrefixTransform{ Prefix: dsDBManagerBaseKey.ChildString(id.String()), diff --git a/db/manager_test.go b/db/manager_test.go index 8dc4013f..15859730 100644 --- a/db/manager_test.go +++ b/db/manager_test.go @@ -81,7 +81,7 @@ func TestManager_NewDB(t *testing.T) { checkErr(t, err) tok, err := man.GetToken(ctx, thread.NewLibp2pIdentity(sk)) checkErr(t, err) - _, err = man.NewDB(ctx, thread.NewIDV1(thread.Raw, 32), WithNewManagedDBToken(tok)) + _, err = man.NewDB(ctx, thread.NewIDV1(thread.Raw, 32), WithNewManagedToken(tok)) checkErr(t, err) }) } @@ -94,7 +94,7 @@ func TestManager_GetDB(t *testing.T) { checkErr(t, err) n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(t, err) - man, err := NewManager(n, WithNewDBRepoPath(dir), WithNewDBDebug(true)) + man, err := NewManager(n, WithNewRepoPath(dir), WithNewDebug(true)) checkErr(t, err) defer func() { _ = os.RemoveAll(dir) @@ -127,7 +127,7 @@ func TestManager_GetDB(t *testing.T) { t.Run("test get db after restart", func(t *testing.T) { n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(t, err) - man, err := NewManager(n, WithNewDBRepoPath(dir), WithNewDBDebug(true)) + man, err := NewManager(n, WithNewRepoPath(dir), WithNewDebug(true)) checkErr(t, err) db, err := man.GetDB(ctx, id) @@ -188,7 +188,7 @@ func createTestManager(t *testing.T) (*Manager, func()) { checkErr(t, err) n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(t, err) - m, err := NewManager(n, WithNewDBRepoPath(dir), WithNewDBDebug(true)) + m, err := NewManager(n, WithNewRepoPath(dir), WithNewDebug(true)) checkErr(t, err) return m, func() { if err := n.Close(); err != nil { diff --git a/db/options.go b/db/options.go index 7ad504e6..886895c3 100644 --- a/db/options.go +++ b/db/options.go @@ -16,20 +16,6 @@ const ( defaultDatastorePath = "eventstore" ) -// NewDBOption takes a Config and modifies it. -type NewDBOption func(*NewDBOptions) error - -// NewDBOptions has configuration parameters for a db. -type NewDBOptions struct { - RepoPath string - Datastore ds.TxnDatastore - EventCodec core.EventCodec - Debug bool - LowMem bool - Collections []CollectionConfig - Token thread.Token -} - func newDefaultEventCodec() core.EventCodec { return jsonpatcher.New() } @@ -46,56 +32,85 @@ func newDefaultDatastore(repoPath string, lowMem bool) (ds.TxnDatastore, error) return badger.NewDatastore(path, &opts) } -// WithNewDBLowMem specifies whether or not to use low memory settings. -func WithNewDBLowMem(low bool) NewDBOption { - return func(o *NewDBOptions) error { +// NewOptions defines options for creating a new db. +type NewOptions struct { + RepoPath string + Datastore ds.TxnDatastore + EventCodec core.EventCodec + Debug bool + LowMem bool + Collections []CollectionConfig + Token thread.Token +} + +// NewOption specifies a new db option. +type NewOption func(*NewOptions) error + +// WithNewLowMem specifies whether or not to use low memory settings. +func WithNewLowMem(low bool) NewOption { + return func(o *NewOptions) error { o.LowMem = low return nil } } -// WithNewDBRepoPath sets the repo path. -func WithNewDBRepoPath(path string) NewDBOption { - return func(o *NewDBOptions) error { +// WithNewRepoPath sets the repo path. +func WithNewRepoPath(path string) NewOption { + return func(o *NewOptions) error { o.RepoPath = path return nil } } -// WithNewDBDebug indicate to output debug information. -func WithNewDBDebug(enable bool) NewDBOption { - return func(o *NewDBOptions) error { +// WithNewDebug indicate to output debug information. +func WithNewDebug(enable bool) NewOption { + return func(o *NewOptions) error { o.Debug = enable return nil } } -// WithNewDBEventCodec configure to use ec as the EventCodec +// WithNewEventCodec configure to use ec as the EventCodec // for transforming actions in events, and viceversa. -func WithNewDBEventCodec(ec core.EventCodec) NewDBOption { - return func(o *NewDBOptions) error { +func WithNewEventCodec(ec core.EventCodec) NewOption { + return func(o *NewOptions) error { o.EventCodec = ec return nil } } -// WithNewDBToken provides authorization for interacting with a db. -func WithNewDBToken(t thread.Token) NewDBOption { - return func(o *NewDBOptions) error { +// WithNewToken provides authorization for interacting with a db. +func WithNewToken(t thread.Token) NewOption { + return func(o *NewOptions) error { o.Token = t return nil } } -// WithNewDBCollections is used to specify collections that +// WithNewCollections is used to specify collections that // will be created. -func WithNewDBCollections(cs ...CollectionConfig) NewDBOption { - return func(o *NewDBOptions) error { +func WithNewCollections(cs ...CollectionConfig) NewOption { + return func(o *NewOptions) error { o.Collections = cs return nil } } +// Options defines options for interacting with a db. +type Options struct { + Token thread.Token +} + +// Option specifies a db option. +type Option func(*Options) + +// WithToken provides authorization for interacting with a db. +func WithToken(t thread.Token) Option { + return func(args *Options) { + args.Token = t + } +} + // TxnOptions defines options for a transaction. type TxnOptions struct { Token thread.Token @@ -111,56 +126,41 @@ func WithTxnToken(t thread.Token) TxnOption { } } -// NewManagedDBOptions defines options for creating a new managed db. -type NewManagedDBOptions struct { +// NewManagedOptions defines options for creating a new managed db. +type NewManagedOptions struct { Collections []CollectionConfig Token thread.Token } -// NewManagedDBOption specifies a new managed db option. -type NewManagedDBOption func(*NewManagedDBOptions) +// NewManagedOption specifies a new managed db option. +type NewManagedOption func(*NewManagedOptions) -// WithNewManagedDBCollections is used to specify collections that +// WithNewManagedCollections is used to specify collections that // will be created in a managed db. -func WithNewManagedDBCollections(cs ...CollectionConfig) NewManagedDBOption { - return func(args *NewManagedDBOptions) { +func WithNewManagedCollections(cs ...CollectionConfig) NewManagedOption { + return func(args *NewManagedOptions) { args.Collections = cs } } -// WithNewManagedDBToken provides authorization for creating a new managed db. -func WithNewManagedDBToken(t thread.Token) NewManagedDBOption { - return func(args *NewManagedDBOptions) { - args.Token = t - } -} - -// ManagedDBOptions defines options for interacting with a managed db. -type ManagedDBOptions struct { - Token thread.Token -} - -// ManagedDBOption specifies a managed db option. -type ManagedDBOption func(*ManagedDBOptions) - -// WithManagedDBToken provides authorization for interacting with a managed db. -func WithManagedDBToken(t thread.Token) ManagedDBOption { - return func(args *ManagedDBOptions) { +// WithNewManagedToken provides authorization for creating a new managed db. +func WithNewManagedToken(t thread.Token) NewManagedOption { + return func(args *NewManagedOptions) { args.Token = t } } -// InviteInfoOptions defines options getting DB invite info. -type InviteInfoOptions struct { +// ManagedOptions defines options for interacting with a managed db. +type ManagedOptions struct { Token thread.Token } -// InviteInfoOption specifies a managed db option. -type InviteInfoOption func(*InviteInfoOptions) +// ManagedOption specifies a managed db option. +type ManagedOption func(*ManagedOptions) -// WithInviteInfoToken provides authorization for accessing DB invite info. -func WithInviteInfoToken(t thread.Token) InviteInfoOption { - return func(args *InviteInfoOptions) { +// WithManagedToken provides authorization for interacting with a managed db. +func WithManagedToken(t thread.Token) ManagedOption { + return func(args *ManagedOptions) { args.Token = t } } diff --git a/db/query.go b/db/query.go index b0f3634e..56860c28 100644 --- a/db/query.go +++ b/db/query.go @@ -8,7 +8,7 @@ import ( "strings" ) -// Query is a json-seriable query representation +// Query is a json-seriable query representation. type Query struct { Ands []*Criterion Ors []*Query @@ -16,7 +16,7 @@ type Query struct { Index string } -// Criterion represents a restriction on a field +// Criterion represents a restriction on a field. type Criterion struct { FieldPath string Operation Operation @@ -24,13 +24,14 @@ type Criterion struct { query *Query } -// Value models a single value in JSON +// Value models a single value in JSON. type Value struct { String *string Bool *bool Float *float64 } +// Validate validates en entire query. func (q *Query) Validate() error { if q == nil { return nil @@ -48,6 +49,7 @@ func (q *Query) Validate() error { return nil } +// Validate validates a single query criterion. func (c *Criterion) Validate() error { if c == nil { return nil @@ -68,13 +70,13 @@ func (c *Criterion) Validate() error { return nil } -// Sort represents a sort order on a field +// Sort represents a sort order on a field. type Sort struct { FieldPath string Desc bool } -// Operation models comparison operators +// Operation models comparison operators. type Operation int const ( @@ -98,7 +100,7 @@ var ( ErrInvalidSortingField = errors.New("sorting field doesn't correspond to instance type") ) -// Where starts to create a query condition for a field +// Where starts to create a query condition for a field. func Where(field string) *Criterion { return &Criterion{ FieldPath: field, @@ -129,7 +131,7 @@ func (q *Query) And(field string) *Criterion { } } -// UseIndex specifies the index to use when running this query +// UseIndex specifies the index to use when running this query. func (q *Query) UseIndex(path string) *Query { q.Index = path return q @@ -242,7 +244,7 @@ func (t *Txn) Find(q *Query) ([][]byte, error) { return nil, fmt.Errorf("error building internal query: %v", err) } defer txn.Discard() - iter := newIterator(txn, t.collection.BaseKey(), q) + iter := newIterator(txn, t.collection.baseKey(), q) defer iter.Close() var values []MarshaledResult diff --git a/db/query_test.go b/db/query_test.go index 19fbcdae..7b31c60a 100644 --- a/db/query_test.go +++ b/db/query_test.go @@ -311,7 +311,7 @@ func createCollectionWithJSONData(t *testing.T) (*Collection, []Book, func()) { c, err := s.NewCollection(CollectionConfig{ Name: "Book", Schema: util.SchemaFromInstance(&Book{}, false), - Indexes: []IndexConfig{ + Indexes: []Index{ { Path: "Meta.TotalReads", }, diff --git a/db/testutils_test.go b/db/testutils_test.go index 3c71be76..3c5d2e7b 100644 --- a/db/testutils_test.go +++ b/db/testutils_test.go @@ -19,12 +19,12 @@ func checkErr(t *testing.T, err error) { } } -func createTestDB(t *testing.T, opts ...NewDBOption) (*DB, func()) { +func createTestDB(t *testing.T, opts ...NewOption) (*DB, func()) { dir, err := ioutil.TempDir("", "") checkErr(t, err) n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(t, err) - opts = append(opts, WithNewDBRepoPath(dir)) + opts = append(opts, WithNewRepoPath(dir)) d, err := NewDB(context.Background(), n, thread.NewIDV1(thread.Raw, 32), opts...) checkErr(t, err) return d, func() { diff --git a/examples/e2e_counter/reader.go b/examples/e2e_counter/reader.go index 6b183ac6..01620b27 100644 --- a/examples/e2e_counter/reader.go +++ b/examples/e2e_counter/reader.go @@ -26,7 +26,7 @@ func runReaderPeer(repo string) { Schema: util.SchemaFromInstance(&myCounter{}, false), } - d, err := db.NewDBFromAddr(context.Background(), n, writerAddr, key, db.WithNewDBRepoPath(repo), db.WithNewDBCollections(cc)) + d, err := db.NewDBFromAddr(context.Background(), n, writerAddr, key, db.WithNewRepoPath(repo), db.WithNewCollections(cc)) checkErr(err) defer d.Close() diff --git a/examples/e2e_counter/writer.go b/examples/e2e_counter/writer.go index a824a819..cd7a8243 100644 --- a/examples/e2e_counter/writer.go +++ b/examples/e2e_counter/writer.go @@ -28,7 +28,7 @@ func runWriterPeer(repo string) { checkErr(err) defer n.Close() id := thread.NewIDV1(thread.Raw, 32) - d, err := db.NewDB(context.Background(), n, id, db.WithNewDBRepoPath(repo)) + d, err := db.NewDB(context.Background(), n, id, db.WithNewRepoPath(repo)) checkErr(err) defer d.Close() diff --git a/examples/local_eventstore/books/main.go b/examples/local_eventstore/books/main.go index e0a1d9f7..100c545d 100644 --- a/examples/local_eventstore/books/main.go +++ b/examples/local_eventstore/books/main.go @@ -182,7 +182,7 @@ func createMemDB() (*db.DB, func()) { n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(err) id := thread.NewIDV1(thread.Raw, 32) - d, err := db.NewDB(context.Background(), n, id, db.WithNewDBRepoPath(dir)) + d, err := db.NewDB(context.Background(), n, id, db.WithNewRepoPath(dir)) checkErr(err) return d, func() { time.Sleep(time.Second) // Give threads a chance to finish work diff --git a/examples/local_eventstore/json-books/main.go b/examples/local_eventstore/json-books/main.go index cb4e54ce..ed76e21d 100644 --- a/examples/local_eventstore/json-books/main.go +++ b/examples/local_eventstore/json-books/main.go @@ -123,7 +123,7 @@ func createMemDB() (*db.DB, func()) { n, err := common.DefaultNetwork(dir, common.WithNetDebug(true), common.WithNetHostAddr(util.FreeLocalAddr())) checkErr(err) id := thread.NewIDV1(thread.Raw, 32) - d, err := db.NewDB(context.Background(), n, id, db.WithNewDBRepoPath(dir)) + d, err := db.NewDB(context.Background(), n, id, db.WithNewRepoPath(dir)) checkErr(err) return d, func() { time.Sleep(time.Second) // Give threads a chance to finish work diff --git a/integrationtests/foldersync/client.go b/integrationtests/foldersync/client.go index f889c846..35e514ee 100644 --- a/integrationtests/foldersync/client.go +++ b/integrationtests/foldersync/client.go @@ -77,7 +77,7 @@ func newRootClient(name, folderPath, repoPath string) (*client, error) { return nil, err } - d, err := db.NewDB(context.Background(), network, id, db.WithNewDBRepoPath(repoPath), db.WithNewDBCollections(cc)) + d, err := db.NewDB(context.Background(), network, id, db.WithNewRepoPath(repoPath), db.WithNewCollections(cc)) if err != nil { return nil, err } @@ -99,7 +99,7 @@ func newJoinerClient(name, folderPath, repoPath string, addr ma.Multiaddr, key t return nil, err } - d, err := db.NewDBFromAddr(context.Background(), network, addr, key, db.WithNewDBRepoPath(repoPath), db.WithNewDBCollections(cc)) + d, err := db.NewDBFromAddr(context.Background(), network, addr, key, db.WithNewRepoPath(repoPath), db.WithNewCollections(cc)) if err != nil { return nil, err } diff --git a/integrationtests/foldersync/foldersync_test.go b/integrationtests/foldersync/foldersync_test.go index f798a2bc..37f0ec73 100644 --- a/integrationtests/foldersync/foldersync_test.go +++ b/integrationtests/foldersync/foldersync_test.go @@ -43,7 +43,7 @@ func TestSimple(t *testing.T) { network0, err := newNetwork(repoPath0) checkErr(t, err) - db0, err := db.NewDB(context.Background(), network0, id, db.WithNewDBRepoPath(repoPath0), db.WithNewDBCollections(cc)) + db0, err := db.NewDB(context.Background(), network0, id, db.WithNewRepoPath(repoPath0), db.WithNewCollections(cc)) checkErr(t, err) defer db0.Close() @@ -60,7 +60,7 @@ func TestSimple(t *testing.T) { network1, err := newNetwork(repoPath1) checkErr(t, err) - db1, err := db.NewDBFromAddr(context.Background(), network1, addrs0[0], key0, db.WithNewDBRepoPath(repoPath1), db.WithNewDBCollections(cc)) + db1, err := db.NewDBFromAddr(context.Background(), network1, addrs0[0], key0, db.WithNewRepoPath(repoPath1), db.WithNewCollections(cc)) checkErr(t, err) defer db1.Close() @@ -74,7 +74,7 @@ func TestSimple(t *testing.T) { network2, err := newNetwork(repoPath2) checkErr(t, err) - db2, err := db.NewDBFromAddr(context.Background(), network2, addrs0[0], key0, db.WithNewDBRepoPath(repoPath2), db.WithNewDBCollections(cc)) + db2, err := db.NewDBFromAddr(context.Background(), network2, addrs0[0], key0, db.WithNewRepoPath(repoPath2), db.WithNewCollections(cc)) checkErr(t, err) defer db2.Close() @@ -88,7 +88,7 @@ func TestSimple(t *testing.T) { network3, err := newNetwork(repoPath3) checkErr(t, err) - db3, err := db.NewDBFromAddr(context.Background(), network3, addrs0[0], key0, db.WithNewDBRepoPath(repoPath3), db.WithNewDBCollections(cc)) + db3, err := db.NewDBFromAddr(context.Background(), network3, addrs0[0], key0, db.WithNewRepoPath(repoPath3), db.WithNewCollections(cc)) checkErr(t, err) defer db3.Close() diff --git a/jsonpatcher/jsonpatcher.go b/jsonpatcher/jsonpatcher.go index 0fe5b410..2a52f16c 100644 --- a/jsonpatcher/jsonpatcher.go +++ b/jsonpatcher/jsonpatcher.go @@ -93,12 +93,7 @@ func (jp *jsonPatcher) Create(actions []core.Action) ([]core.Event, format.Node, return events, n, nil } -func (jp *jsonPatcher) Reduce( - events []core.Event, - datastore ds.TxnDatastore, - baseKey ds.Key, - indexFunc func(collection string, key ds.Key, oldData, newData []byte, txn ds.Txn) error, -) ([]core.ReduceAction, error) { +func (jp *jsonPatcher) Reduce(events []core.Event, datastore ds.TxnDatastore, baseKey ds.Key, indexFunc core.IndexFunc) ([]core.ReduceAction, error) { txn, err := datastore.NewTransaction(false) if err != nil { return nil, err