Skip to content

Commit

Permalink
Collection and index updates and fixes (textileio#362)
Browse files Browse the repository at this point in the history 
* db: cleanup indexing, add update collection method

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: adds delete collection, more index cleanup

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: add update collection api method and more tests

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: add get indexes method to api

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: ensure unique indexes can be unique

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: use token opt in collection methods

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: use two locks

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* db: check index path on hydration

Signed-off-by: Sander Pick <sanderpick@gmail.com>

* review: address feedback

Signed-off-by: Sander Pick <sanderpick@gmail.com>
  • Loading branch information
@sanderpick @dgtony
sanderpick authored and dgtony committed Aug 23, 2020
1 parent 84f67c7 commit a556125
Show file tree
Hide file tree
Showing 29 changed files with 1,694 additions and 705 deletions.
78 changes: 65 additions & 13 deletions api/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,8 @@ func (c *Client) GetToken(ctx context.Context, identity thread.Identity) (tok th
}

// NewDB creates a new DB with ID.
func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManagedDBOption) error {
args := &db.NewManagedDBOptions{}
func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManagedOption) error {
args := &db.NewManagedOptions{}
for _, opt := range opts {
opt(args)
}
Expand All @@ -174,8 +174,8 @@ func (c *Client) NewDB(ctx context.Context, dbID thread.ID, opts ...db.NewManage
}

// NewDBFromAddr creates a new DB with address and keys.
func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey thread.Key, opts ...db.NewManagedDBOption) error {
args := &db.NewManagedDBOptions{}
func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey thread.Key, opts ...db.NewManagedOption) error {
args := &db.NewManagedOptions{}
for _, opt := range opts {
opt(args)
}
Expand All @@ -197,9 +197,9 @@ func (c *Client) NewDBFromAddr(ctx context.Context, dbAddr ma.Multiaddr, dbKey t
}

func collectionConfigToPb(c db.CollectionConfig) (*pb.CollectionConfig, error) {
idx := make([]*pb.CollectionConfig_IndexConfig, len(c.Indexes))
idx := make([]*pb.Index, len(c.Indexes))
for i, index := range c.Indexes {
idx[i] = &pb.CollectionConfig_IndexConfig{
idx[i] = &pb.Index{
Path: index.Path,
Unique: index.Unique,
}
Expand All @@ -216,8 +216,8 @@ func collectionConfigToPb(c db.CollectionConfig) (*pb.CollectionConfig, error) {
}

// GetDBInfo retrives db addresses and keys.
func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.ManagedDBOption) ([]ma.Multiaddr, thread.Key, error) {
args := &db.ManagedDBOptions{}
func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.ManagedOption) ([]ma.Multiaddr, thread.Key, error) {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
Expand All @@ -244,8 +244,8 @@ func (c *Client) GetDBInfo(ctx context.Context, dbID thread.ID, opts ...db.Manag
}

// DeleteDB deletes a db.
func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.ManagedDBOption) error {
args := &db.ManagedDBOptions{}
func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.ManagedOption) error {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
Expand All @@ -257,9 +257,8 @@ func (c *Client) DeleteDB(ctx context.Context, dbID thread.ID, opts ...db.Manage
}

// NewCollection creates a new collection.
// @todo: This should take some thread auth, but collections currently do not involve a thread.
func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedDBOption) error {
args := &db.ManagedDBOptions{}
func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedOption) error {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
Expand All @@ -274,6 +273,59 @@ func (c *Client) NewCollection(ctx context.Context, dbID thread.ID, config db.Co
return err
}

// UpdateCollection updates an existing collection.
func (c *Client) UpdateCollection(ctx context.Context, dbID thread.ID, config db.CollectionConfig, opts ...db.ManagedOption) error {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
cc, err := collectionConfigToPb(config)
if err != nil {
return err
}
_, err = c.c.UpdateCollection(ctx, &pb.UpdateCollectionRequest{
DbID: dbID.Bytes(),
Config: cc,
})
return err
}

// DeleteCollection deletes a collection.
func (c *Client) DeleteCollection(ctx context.Context, dbID thread.ID, name string, opts ...db.ManagedOption) error {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
_, err := c.c.DeleteCollection(ctx, &pb.DeleteCollectionRequest{
DbID: dbID.Bytes(),
Name: name,
})
return err
}

// GetCollectionIndexes returns an existing collection's indexes.
func (c *Client) GetCollectionIndexes(ctx context.Context, dbID thread.ID, name string, opts ...db.ManagedOption) ([]db.Index, error) {
args := &db.ManagedOptions{}
for _, opt := range opts {
opt(args)
}
resp, err := c.c.GetCollectionIndexes(ctx, &pb.GetCollectionIndexesRequest{
DbID: dbID.Bytes(),
Name: name,
})
if err != nil {
return nil, err
}
indexes := make([]db.Index, len(resp.Indexes))
for i, index := range resp.Indexes {
indexes[i] = db.Index{
Path: index.Path,
Unique: index.Unique,
}
}
return indexes, nil
}

// Create creates new instances of objects.
func (c *Client) Create(ctx context.Context, dbID thread.ID, collectionName string, instances Instances, opts ...db.TxnOption) ([]string, error) {
args := &db.TxnOptions{}
Expand Down
161 changes: 136 additions & 25 deletions api/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,83 @@ func TestClient_NewCollection(t *testing.T) {
})
}

func TestClient_UpdateCollection(t *testing.T) {
t.Parallel()
client, done := setup(t)
defer done()

id := thread.NewIDV1(thread.Raw, 32)
err := client.NewDB(context.Background(), id)
checkErr(t, err)
err = client.NewCollection(context.Background(), id, db.CollectionConfig{Name: collectionName, Schema: util.SchemaFromSchemaString(schema)})
checkErr(t, err)

t.Run("test update collection", func(t *testing.T) {
err = client.UpdateCollection(context.Background(), id, db.CollectionConfig{
Name: collectionName,
Schema: util.SchemaFromSchemaString(schema2),
Indexes: []db.Index{{
Path: "age",
Unique: false,
}},
})
if err != nil {
t.Fatalf("failed to update collection: %v", err)
}
_, err = client.Create(context.Background(), id, collectionName, Instances{createPerson2()})
checkErr(t, err)
})
}

func TestClient_DeleteCollection(t *testing.T) {
t.Parallel()
client, done := setup(t)
defer done()

t.Run("test delete collection", func(t *testing.T) {
id := thread.NewIDV1(thread.Raw, 32)
err := client.NewDB(context.Background(), id)
checkErr(t, err)
err = client.NewCollection(context.Background(), id, db.CollectionConfig{Name: collectionName, Schema: util.SchemaFromSchemaString(schema)})
checkErr(t, err)

err = client.DeleteCollection(context.Background(), id, collectionName)
if err != nil {
t.Fatalf("failed to delete collection: %v", err)
}
_, err = client.Create(context.Background(), id, collectionName, Instances{createPerson()})
if err == nil {
t.Fatal("failed to delete collection")
}
})
}

func TestClient_GetCollectionIndexes(t *testing.T) {
t.Parallel()
client, done := setup(t)
defer done()

t.Run("test get collection indexes", func(t *testing.T) {
id := thread.NewIDV1(thread.Raw, 32)
err := client.NewDB(context.Background(), id)
checkErr(t, err)
err = client.NewCollection(context.Background(), id, db.CollectionConfig{
Name: collectionName,
Schema: util.SchemaFromSchemaString(schema),
Indexes: []db.Index{{
Path: "lastName",
Unique: true,
}},
})
checkErr(t, err)
indexes, err := client.GetCollectionIndexes(context.Background(), id, collectionName)
checkErr(t, err)
if len(indexes) != 2 {
t.Fatalf("expected 2 indexes, but got %v", len(indexes))
}
})
}

func TestClient_Create(t *testing.T) {
t.Parallel()
client, done := setup(t)
Expand Down Expand Up @@ -302,7 +379,7 @@ func TestClient_FindWithIndex(t *testing.T) {
err = client.NewCollection(context.Background(), id, db.CollectionConfig{
Name: collectionName,
Schema: util.SchemaFromSchemaString(schema),
Indexes: []db.IndexConfig{{
Indexes: []db.Index{{
Path: "lastName",
Unique: true,
}},
Expand Down Expand Up @@ -698,41 +775,69 @@ func createIdentity(t *testing.T) thread.Identity {

func createPerson() *Person {
return &Person{
ID: "",
FirstName: "Adam",
LastName: "Doe",
Age: 21,
}
}

func createPerson2() *Person2 {
return &Person2{
FullName: "Adam Doe",
Age: 21,
}
}

const (
collectionName = "Person"

schema = `{
"$id": "https://example.com/person.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "` + collectionName + `",
"type": "object",
"properties": {
"_id": {
"type": "string",
"description": "The instance's id."
},
"firstName": {
"type": "string",
"description": "The person's first name."
},
"lastName": {
"type": "string",
"description": "The person's last name."
},
"age": {
"description": "Age in years which must be equal to or greater than zero.",
"type": "integer",
"minimum": 0
"$id": "https://example.com/person.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "` + collectionName + `",
"type": "object",
"properties": {
"_id": {
"type": "string",
"description": "The instance's id."
},
"firstName": {
"type": "string",
"description": "The person's first name."
},
"lastName": {
"type": "string",
"description": "The person's last name."
},
"age": {
"description": "Age in years which must be equal to or greater than zero.",
"type": "integer",
"minimum": 0
}
}
}
}`
}`

schema2 = `{
"$id": "https://example.com/person.schema.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "` + collectionName + `",
"type": "object",
"properties": {
"_id": {
"type": "string",
"description": "The instance's id."
},
"fullName": {
"type": "string",
"description": "The person's full name."
},
"age": {
"description": "Age in years which must be equal to or greater than zero.",
"type": "integer",
"minimum": 0
}
}
}`
)

type Person struct {
Expand All @@ -742,6 +847,12 @@ type Person struct {
Age int `json:"age,omitempty"`
}

type Person2 struct {
ID string `json:"_id"`
FullName string `json:"fullName,omitempty"`
Age int `json:"age,omitempty"`
}

type PersonWithoutID struct {
FirstName string `json:"firstName,omitempty"`
LastName string `json:"lastName,omitempty"`
Expand Down
Loading

0 comments on commit a556125

Please sign in to comment.