Data Querying - JSONPath and Evaluation

documentation/README.md

@@ -328,6 +328,11 @@ A collection is a group of records/data of same modality (schema). Collections a
 
 Each collection has a schema that defines its modality and how data in that collection should be structured. In wunderDb schema for a collection is defined using [JSON Schema](https://json-schema.org/) and at the time when collections are created. JSON Schema defines the structure, type and various other standards of the data. Read more on how to define schema using JSON Schema [here](https://json-schema.org/learn/getting-started-step-by-step.html).
 
+Please note
+
+- wunderDB expects that the schema definition contains the primary key as a part of the required array, else it will throw error.
+- wunderDB also expects the [`additionalFields`](https://json-schema.org/understanding-json-schema/reference/object#additionalproperties) schema properties to be set, so as to specify if any additional fields other than the ones in schema should be allowed. If this property is not set, then wunderDB adds it to the schema definition with the default value `false`, i.e no extra fields allowed.
+
 ### Create Collection
 
 To create a collection in a database, use the following endpoint, passing the schema of the data (in JSON Schema notations) in the body. User must have the `createCollection` access granted on the database where the collection is to be created.
@@ -383,6 +388,32 @@ To filter data while reading, updating or deleting, we need to pass the field na
 
 Example, `.../data?key:name&value:John`, will filter all records with `name=John`.
 
+### Querying/Evaluating Data
+
+To query or evaluating data, use the following call, passing the `mode` (either `jsonpath` or `evaluate`) and `query` as the jsonpath or evaluation queries.
+
+```http
+POST /api/databases/{database}/collections/{collection}/data/query HTTP/1.1
+Authorization: Basic 
+Content-Type: application/json
+
+{
+    "mode": "jsonpath",
+    "query": "$..data.age"
+}
+
+OR
+
+{
+    "mode": "evaluate",
+    "query" : "sum($..data.age)"
+}
+```
+
+- Use `jsonpath` mode to get a specific field value or subvalues from json data, eg: get the value of a sub-field.
+- Use `evaluate` mode to evaluate something based on the data, eg: sum of values of a specific field.
+
+
 ### Insert/Add Data
 
 User must have `addData` permission granted on the collection to add data to. Pass the data to add in the body as JSON object.

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/gofiber/fiber/v2 v2.52.1
 	github.com/google/uuid v1.5.0
 	github.com/sirupsen/logrus v1.9.0
+	github.com/spyzhov/ajson v0.9.1
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli/v2 v2.23.7
 	github.com/xeipuuv/gojsonschema v1.2.0
@@ -28,7 +29,7 @@ require (
 	github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect
 	github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rivo/uniseg v0.4.3 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.51.0 // indirect
@@ -37,7 +38,7 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -53,15 +53,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
-github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spyzhov/ajson v0.9.1 h1:izsDkOC9yznlpA2BxJcrkr8Q5Gyjf43yeSc2Wz0g/aU=
+github.com/spyzhov/ajson v0.9.1/go.mod h1:a6oSw0MMb7Z5aD2tPoPO+jq11ETKgXUr2XktHdT8Wt8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -100,8 +102,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

internal/collections/collections.go

@@ -3,6 +3,7 @@ package collections
 import (
 	"github.com/TanmoySG/wunderDB/internal/metadata"
 	"github.com/TanmoySG/wunderDB/model"
+	s "github.com/TanmoySG/wunderDB/pkg/schema"
 	wdbErrors "github.com/TanmoySG/wunderDB/pkg/wdb/errors"
 )
 
@@ -38,6 +39,9 @@ func (c Collections) CreateCollection(collectionID model.Identifier, schema mode
 		return err
 	}
 
+	// standardize schema: add schema fields if not present
+	schema = s.StandardizeSchema(schema)
+
 	c[collectionID] = &model.Collection{
 		Data:       map[model.Identifier]*model.Record{},
 		Schema:     schema,

internal/data/data.go

@@ -1,6 +1,7 @@
 package data
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/TanmoySG/wunderDB/internal/filter"
@@ -9,12 +10,20 @@ import (
 	"github.com/TanmoySG/wunderDB/pkg/schema"
 	"github.com/TanmoySG/wunderDB/pkg/utils/maps"
 	er "github.com/TanmoySG/wunderDB/pkg/wdb/errors"
+	"github.com/spyzhov/ajson"
 )
 
 const (
 	defaultPrimaryKeyField = "recordId"
 )
 
+var (
+	JsonPathQuery QueryType = "jsonpath"
+	EvaluateQuery QueryType = "evaluate"
+)
+
+type QueryType string
+
 type Data struct {
 	Data       map[model.Identifier]*model.Record
 	Schema     model.Schema
@@ -69,6 +78,7 @@ func (d Data) Read(filters interface{}) (map[model.Identifier]*model.Record, *er
 		filteredData := f.Filter(*d.PrimaryKey, d.Data)
 		return filteredData, nil
 	}
+
 	return d.Data, nil
 }
 
@@ -127,6 +137,48 @@ func (d Data) Delete(filters interface{}) *er.WdbError {
 	return &er.FilterMissingError
 }
 
+func (d Data) Query(query string, mode QueryType) (interface{}, *er.WdbError) {
+
+	jsonData, err := json.Marshal(d.Data)
+	if err != nil {
+		return nil, nil
+	}
+
+	var queryResultNodes []*ajson.Node
+	var queryResults []interface{}
+
+	root, err := ajson.Unmarshal(jsonData)
+	if err != nil {
+		return nil, nil
+	}
+
+	switch mode {
+	case JsonPathQuery:
+		jpqResult, err := root.JSONPath(query)
+		if err != nil {
+			return nil, er.JSONPathQueryError.SetMessage(err.Error())
+		}
+		queryResultNodes = jpqResult
+	case EvaluateQuery:
+		evqResult, err := ajson.Eval(root, query)
+		if err != nil {
+			return nil, er.QueryExecutionFailed.SetMessage(err.Error())
+		}
+
+		queryResultNodes = []*ajson.Node{evqResult}
+	}
+
+	for _, node := range queryResultNodes {
+		marshaledNode, err := ajson.Marshal(node)
+		if err != nil {
+			return nil, nil
+		}
+		queryResults = append(queryResults, string(marshaledNode))
+	}
+
+	return queryResults, nil
+}
+
 func (d Data) getPrimaryKey(recordId model.Identifier, data interface{}) model.Identifier {
 	primaryKeyValue := recordId.String()
 

internal/privileges/constants.go

@@ -54,6 +54,7 @@ const (
 
 	AddData    = "addData"
 	ReadData   = "readData"
+	QueryData  = "queryData"
 	UpdateData = "updateData"
 	DeleteData = "deleteData"
 )

internal/privileges/privileges.go

@@ -26,6 +26,7 @@ var PrivilegeScope = map[string]string{
 	ReadData:         CollectionPrivileges,
 	UpdateData:       CollectionPrivileges,
 	DeleteData:       CollectionPrivileges,
+	QueryData:        CollectionPrivileges,
 }
 
 var PrivilegeType = map[string]PrivilegeActionType{
@@ -39,6 +40,7 @@ var PrivilegeType = map[string]PrivilegeActionType{
 	ReadDatabase:   ReadPrivilege,
 	ReadCollection: ReadPrivilege,
 	ReadData:       ReadPrivilege,
+	QueryData:      ReadPrivilege,
 
 	CreateDatabase:   WritePrivilege,
 	UpdateDatabase:   WritePrivilege,

internal/server/handlers/common.go

@@ -109,7 +109,7 @@ func (wh wdbHandlers) handleTransactions(c *fiber.Ctx, apiResponse response.ApiR
 			txnActor := txlogs.GetTxnActor(c.Get(AuthorizationHeader))
 			txnAction := apiResponse.Response.Action
 
-			txnHttpDetails := txlogs.GetTxnHttpDetails(*c)
+			txnHttpDetails := txlogs.GetTxnHttpDetails(c) // Pass the pointer to fiber.Ctx instead of dereferencing it
 			txnEntityPath := txlModel.TxlogSchemaJsonEntityPath{
 				Database:   databaseEntity,
 				Collection: entities.Collections,

internal/server/handlers/data.go

@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"github.com/TanmoySG/wunderDB/internal/data"
 	"github.com/TanmoySG/wunderDB/internal/privileges"
 	"github.com/TanmoySG/wunderDB/internal/server/response"
 	"github.com/TanmoySG/wunderDB/model"
@@ -12,6 +13,11 @@ const (
 	emptyFilter = ""
 )
 
+type queryRequest struct {
+	QueryMode   string `json:"mode" xml:"mode" form:"mode" validate:"required"`
+	QueryString string `json:"query" xml:"query" form:"query" validate:"required"`
+}
+
 func (wh wdbHandlers) AddData(c *fiber.Ctx) error {
 	privilege := privileges.AddData
 
@@ -90,6 +96,51 @@ func (wh wdbHandlers) ReadData(c *fiber.Ctx) error {
 	return nil
 }
 
+func (wh wdbHandlers) QueryData(c *fiber.Ctx) error {
+	privilege := privileges.QueryData
+
+	var apiError *er.WdbError
+	var queriedData interface{}
+	databaseName := c.Params("database")
+	collectionName := c.Params("collection")
+
+	entities := model.Entities{
+		Databases:   &databaseName,
+		Collections: &collectionName,
+	}
+
+	query := new(queryRequest)
+	if err := c.BodyParser(query); err != nil {
+		return err
+	}
+
+	if err := validateRequest(query); err != nil {
+		apiError = err
+	} else {
+		isValid, error := wh.handleAuthenticationAndAuthorization(c, entities, privilege)
+		if !isValid {
+			apiError = error
+		} else {
+			queriedData, apiError = wh.wdbClient.QueryData(
+				model.Identifier(databaseName),
+				model.Identifier(collectionName),
+				query.QueryString,
+				data.QueryType(query.QueryMode),
+			)
+		}
+	}
+
+	resp := response.Format(privilege, apiError, queriedData, *wh.notices...)
+
+	if err := sendResponse(c, resp); err != nil {
+		return err
+	}
+
+	wh.handleTransactions(c, resp, entities)
+
+	return nil
+}
+
 func (wh wdbHandlers) DeleteData(c *fiber.Ctx) error {
 	privilege := privileges.DeleteData
 

internal/server/handlers/handlers.go

@@ -36,6 +36,7 @@ type Client interface {
 	// Data Handlers
 	AddData(c *fiber.Ctx) error
 	ReadData(c *fiber.Ctx) error
+	QueryData(c *fiber.Ctx) error
 	DeleteData(c *fiber.Ctx) error
 	UpdateData(c *fiber.Ctx) error
 

internal/server/routes/routes.go

@@ -13,6 +13,7 @@ const (
 	// Data Routes
 	AddData    = "/databases/:database/collections/:collection/data"
 	ReadData   = "/databases/:database/collections/:collection/data"
+	QueryData  = "/databases/:database/collections/:collection/data/query" // route for executing jsonpath queries
 	DeleteData = "/databases/:database/collections/:collection/data"
 	UpdateData = "/databases/:database/collections/:collection/data"
 

internal/server/server.go

@@ -63,6 +63,7 @@ func (ws wdbServer) Start() {
 	// Data Routes
 	api.Post(routes.AddData, ws.handler.AddData)
 	api.Get(routes.ReadData, ws.handler.ReadData)
+	api.Post(routes.QueryData, ws.handler.QueryData)
 	api.Delete(routes.DeleteData, ws.handler.DeleteData)
 	api.Patch(routes.UpdateData, ws.handler.UpdateData)
 

internal/txlogs/txlogs.go

@@ -33,7 +33,7 @@ func CreateTxLog(txnAction, txnActor string, txnRequestStatus string, txnEntitie
 	}
 }
 
-func GetTxnHttpDetails(c fiber.Ctx) txlModel.TxlogSchemaJsonTransactionDetails {
+func GetTxnHttpDetails(c *fiber.Ctx) txlModel.TxlogSchemaJsonTransactionDetails {
 	txnHttpUrl, txnUserAgent, txnRequestIP := c.Path(), c.Get("User-Agent"), c.IP()
 
 	txnRequestHttpMethod := txlModel.TxlogSchemaJsonTransactionDetailsMethod(c.Method())

pkg/schema/schema.go

@@ -12,6 +12,14 @@ type Schema struct {
 	loadedSchema jsonschema.JSONLoader
 }
 
+var (
+	// update default values for required schema fields
+	// TODO: make this configurable at startup
+	requiredSchemaFields = map[string]interface{}{
+		"additionalProperties": false,
+	}
+)
+
 func UseSchema(schema model.Schema) (*Schema, *er.WdbError) {
 	marshaledSchemaJSON, err := json.Marshal(schema)
 	if err != nil {
@@ -29,6 +37,27 @@ func (s Schema) Validate(data interface{}) (bool, *er.WdbError) {
 	}
 	loadedData := jsonschema.NewStringLoader(string(marshaledDataJSON))
 
-	validity, _ := jsonschema.Validate(s.loadedSchema, loadedData)
+	validity, err := jsonschema.Validate(s.loadedSchema, loadedData)
+	if err != nil {
+		return false, er.SchemaValidationFailed.SetMessage(err.Error())
+	}
+
 	return validity.Valid(), nil
 }
+
+// adds default schema fields if not present, eg: additionalProperties [default: false]
+func StandardizeSchema(schema model.Schema) model.Schema {
+
+	// return schema if schema is empty
+	if len(schema) == 0 {
+		return schema
+	}
+
+	for field, fieldDefaultValue := range requiredSchemaFields {
+		if _, ok := schema[field]; !ok {
+			schema[field] = fieldDefaultValue
+		}
+	}
+
+	return schema
+}