Replace data in collection with records #155

Merge pull request #155 from TanmoySG/remove-data-breaking

- #121 
- #150 
- Dependant on #156

.github/workflows/railway-deploy.yaml

@@ -0,0 +1,34 @@
+name: Railway Deploy
+
+on:
+  workflow_run:
+    workflows:
+      - TagDeploy
+    types:
+      - completed
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Use Node 12
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install packages
+        run: yarn install --frozen-lockfile
+
+      - name: Run tests
+        run: yarn test
+
+      - name: Install Railway
+        run: npm i -g @railway/cli
+
+      - name: Deploy
+        run: railway up
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}

README.md

@@ -4,7 +4,7 @@ wunderDb is a JSON-based in-memory Data Store.
 <!--  For persistent data storage wunderDb loads data from and dumps to filesystem at the start and end of it's lifecycle (startup and shutdown). -->
 
 > [!NOTE]
-> We've fixed the issues with the docker image where graceful shut-down and data persistence were not working while deploying to kubernetes. Please use [wunderDb `v1.5.0`](https://github.com/TanmoySG/wunderDB/pkgs/container/wunderdb/195399419?tag=v1.5.0) and above for better kubernetes support. Thanks!
+> We've updated the models for collections, replacing data by records. Please use wunderDb `v1.7.0` with the environment variable `RUN_MODE=RUN_MODE_UPGRADE` to migrate the data to new model, otherwise there will be loss of data.
 
 ## Quickstart
 

cmd/entrypoint/entrypoint.go

@@ -7,6 +7,7 @@ import (
 	"github.com/TanmoySG/wunderDB/internal/runmode"
 	"github.com/TanmoySG/wunderDB/internal/server/lifecycle/shutdown"
 	"github.com/TanmoySG/wunderDB/internal/server/lifecycle/startup"
+	"github.com/TanmoySG/wunderDB/internal/upgrades"
 	"github.com/charmbracelet/lipgloss"
 )
 
@@ -23,17 +24,28 @@ func EntryPoint() error {
 	}
 
 	if runmode.ShouldEnterRunMode(runmode.RUN_MODE_TYPE(c.RunMode)) {
-		// add logic to enter run mode
-		// runmode.IsValidUpgradeInstruction()
-
-		mm, err := runmode.NewMaintenanceMode(*c)
-		if err != nil {
-			return fmt.Errorf("error initializing Maintenance Mode: %s", err)
-		}
-
-		err = mm.EnterMaintenanceMode()
-		if err != nil {
-			return fmt.Errorf("error entering Maintenance Mode: %s", err)
+		if c.RunMode == string(runmode.RUN_MODE_MAINTENANCE) {
+			mm, err := runmode.NewMaintenanceMode(*c)
+			if err != nil {
+				return fmt.Errorf("error initializing Maintenance Mode: %s", err)
+			}
+
+			err = mm.EnterMaintenanceMode()
+			if err != nil {
+				return fmt.Errorf("error entering Maintenance Mode: %s", err)
+			}
+		} else if c.RunMode == string(runmode.RUN_MODE_UPGRADE) {
+			err := upgrades.Upgrade(*c)
+			if err != nil {
+				fmt.Printf("error initializing Upgrade Mode: %s", err)
+			}
+
+			c.RunMode = string(runmode.RUN_MODE_NORMAL)
+
+			err = config.Unload(c)
+			if err != nil {
+				fmt.Printf("error unloading configurations: %s", err)
+			}
 		}
 	}
 

documentation/MIGRATION.md

@@ -1,6 +1,8 @@
-# Migrations
+# Migrations/Upgrades
 
-With recent changes in the docker image we've discontinued the use of `wdb-tools`. Migration documentation and mechanism witll be updated soon.
+Model changes internally would now be managed using the `RUN_MODE_UPGARDE` value for the `RUN_MODE` environment variable. For any changes that require migration, we recommend setting RUN_MODE as RUN_MODE_UPGARDE, that wll run the migration steps while starting wunderDb.
+
+<!-- With recent changes in the docker image we've discontinued the use of `wdb-tools`. Migration documentation and mechanism witll be updated soon. -->
 <!-- 
 This document contains background details and instructions to migrate to new models and breaking changes.
 

documentation/README.md

@@ -322,7 +322,7 @@ Authorization: Basic
 
 ## Collections
 
-A collection is a group of records/data of same modality (schema). Collections are the primary containers of data.
+A collection is a group of records of same modality (schema). Collections are the primary containers of data.
 
 ### Schema
 
@@ -386,14 +386,14 @@ Filters are used to (as the name suggests) filter or to create smaller buckets/v
 
 To filter data while reading, updating or deleting, we need to pass the field name to the `key` and the value (of the field) that needs to be matched to the `value`.
 
-Example, `.../data?key:name&value:John`, will filter all records with `name=John`.
+Example, `.../records?key:name&value:John`, will filter all records with `name=John`.
 
 ### Querying/Evaluating Data
 
 To query or evaluating data, use the following call, passing the `mode` (either `jsonpath` or `evaluate`) and `query` as the jsonpath or evaluation queries.
 
 ```http
-POST /api/databases/{database}/collections/{collection}/data/query HTTP/1.1
+POST /api/databases/{database}/collections/{collection}/records/query HTTP/1.1
 Authorization: Basic 
 Content-Type: application/json
 
@@ -419,7 +419,7 @@ OR
 User must have `addData` permission granted on the collection to add data to. Pass the data to add in the body as JSON object.
 
 ```http
-POST /api/databases/{database}/collections/{collection}/data HTTP/1.1
+POST /api/databases/{database}/collections/{collection}/records HTTP/1.1
 Authorization: Basic 
 Content-Type: application/json
 
@@ -435,23 +435,23 @@ If the data passes schema validation it is added otherwise returns error.
 To fetch/read data user must have `readData` permission granted on the collection.
 
 ```http
-GET /api/databases/{database}/collections/{collection}/data HTTP/1.1
+GET /api/databases/{database}/collections/{collection}/records HTTP/1.1
 Accept: application/json
 Authorization: Basic 
 ```
 
 Use filters to fetch specific records based on some condition.
 
 ```http
-GET /api/databases/{database}/collections/{collection}/data?key={field-name}&value={field-value} HTTP/1.1
+GET /api/databases/{database}/collections/{collection}/records?key={field-name}&value={field-value} HTTP/1.1
 ```
 
 ### Delete Data
 
 Use filters to specify/select the data to be deleted based on the key-value condition. User must have `deleteData` permission granted on the collection to delete data from.
 
 ```http
-DELETE /api/databases/{database}/collections/{collection}/data?key={field-name}&value={field-value} HTTP/1.1
+DELETE /api/databases/{database}/collections/{collection}/records?key={field-name}&value={field-value} HTTP/1.1
 Accept: application/json
 Authorization: Basic 
 ```
@@ -461,7 +461,7 @@ Authorization: Basic
 Updating data requires the user to pass the `filters` to specify the data to update as well as the updated values of the fields to change in the body of the request. The user required `updateData` permission granted on the collection.
 
 ```http
-PATCH /api/databases/{database}/collections/{collection}/data?key={field-name}&value={field-value} HTTP/1.1
+PATCH /api/databases/{database}/collections/{collection}/records?key={field-name}&value={field-value} HTTP/1.1
 Authorization: Basic 
 Content-Type: application/json
 

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/go-playground/validator/v10 v10.11.1
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/google/uuid v1.5.0
+	github.com/hashicorp/go-version v1.7.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spyzhov/ajson v0.9.1
 	github.com/stretchr/testify v1.8.1

go.sum

@@ -20,6 +20,8 @@ github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yG
 github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
 github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

internal/collections/collections.go

@@ -43,7 +43,7 @@ func (c Collections) CreateCollection(collectionID model.Identifier, schema mode
 	schema = s.StandardizeSchema(schema)
 
 	c[collectionID] = &model.Collection{
-		Data:       map[model.Identifier]*model.Record{},
+		Records:    map[model.Identifier]*model.Record{},
 		Schema:     schema,
 		Metadata:   metadata.New().BasicChangeMetadata(),
 		PrimaryKey: &primaryKeyField,

internal/collections/collections_test.go

@@ -19,14 +19,14 @@ var (
 			metadata.CreatedAt: fixedTimestamp,
 			metadata.UpdatedAt: fixedTimestamp,
 		},
-		Data: map[model.Identifier]*model.Record{},
+		Records: map[model.Identifier]*model.Record{},
 	}
 
 	testCollection2Name model.Identifier  = "testCollection2"
 	testCollection2     *model.Collection = &model.Collection{
 		Schema:   model.Schema{},
 		Metadata: model.Metadata{},
-		Data:     map[model.Identifier]*model.Record{},
+		Records:  map[model.Identifier]*model.Record{},
 	}
 )
 
@@ -112,7 +112,7 @@ func Test_DeleteDatabase(t *testing.T) {
 func Test_UpdateMetadata(t *testing.T) {
 	expectedCollectionsChange := Collections{
 		testCollection1Name: &model.Collection{
-			Data: map[model.Identifier]*model.Record{},
+			Records: map[model.Identifier]*model.Record{},
 			Metadata: model.Metadata{
 				metadata.CreatedAt: fixedTimestamp,
 				metadata.UpdatedAt: time.Now().UTC().Unix(),

internal/config/config.go

@@ -153,3 +153,23 @@ func getConfigurationValue(key string) string {
 	}
 	return envVal
 }
+
+// function to unload the config on demand
+func Unload(c *Config) error {
+	hostOS, err := system.GetHostOS()
+	if err != nil {
+		return fmt.Errorf("error unloading config: %s", err)
+	}
+
+	homeDir := system.GetUserHome(hostOS)
+
+	wdbRootDirectory := fmt.Sprintf(WDB_ROOT_PATH_FORMAT, homeDir)
+	wdbConfigDirectory := fmt.Sprintf(WDB_CONFIG_DIR_PATH_FORMAT, wdbRootDirectory)
+	wdbConfigFilePath := fmt.Sprintf(WDB_CONFIG_FILE_PATH_FORMAT, wdbConfigDirectory)
+	err = handleFile(wdbConfigFilePath)
+	if err != nil {
+		return fmt.Errorf("error unloading config: %s", err)
+	}
+
+	return writeConfigFile(wdbConfigFilePath, *c)
+}

internal/privileges/constants.go

@@ -52,9 +52,9 @@ const (
 	UpdateCollection = "updateCollection"
 	DeleteCollection = "deleteCollection"
 
-	AddData    = "addData"
-	ReadData   = "readData"
-	QueryData  = "queryData"
-	UpdateData = "updateData"
-	DeleteData = "deleteData"
+	AddRecords    = "addRecords"
+	ReadRecords   = "readRecords"
+	QueryRecords  = "queryRecords"
+	UpdateRecords = "updateRecords"
+	DeleteRecords = "deleteRecords"
 )

internal/privileges/privileges.go

@@ -22,11 +22,11 @@ var PrivilegeScope = map[string]string{
 	ReadCollection:   CollectionPrivileges,
 	UpdateCollection: CollectionPrivileges,
 	DeleteCollection: CollectionPrivileges,
-	AddData:          CollectionPrivileges,
-	ReadData:         CollectionPrivileges,
-	UpdateData:       CollectionPrivileges,
-	DeleteData:       CollectionPrivileges,
-	QueryData:        CollectionPrivileges,
+	AddRecords:       CollectionPrivileges,
+	ReadRecords:      CollectionPrivileges,
+	UpdateRecords:    CollectionPrivileges,
+	DeleteRecords:    CollectionPrivileges,
+	QueryRecords:     CollectionPrivileges,
 }
 
 var PrivilegeType = map[string]PrivilegeActionType{
@@ -39,18 +39,18 @@ var PrivilegeType = map[string]PrivilegeActionType{
 
 	ReadDatabase:   ReadPrivilege,
 	ReadCollection: ReadPrivilege,
-	ReadData:       ReadPrivilege,
-	QueryData:      ReadPrivilege,
+	ReadRecords:    ReadPrivilege,
+	QueryRecords:   ReadPrivilege,
 
 	CreateDatabase:   WritePrivilege,
 	UpdateDatabase:   WritePrivilege,
 	DeleteDatabase:   WritePrivilege,
 	CreateCollection: WritePrivilege,
 	UpdateCollection: WritePrivilege,
 	DeleteCollection: WritePrivilege,
-	AddData:          WritePrivilege,
-	UpdateData:       WritePrivilege,
-	DeleteData:       WritePrivilege,
+	AddRecords:       WritePrivilege,
+	UpdateRecords:    WritePrivilege,
+	DeleteRecords:    WritePrivilege,
 }
 
 func IsAvailable(privilege string) bool {