Merge pull request #84 from fossyy/staging

Implement Redis caching for S3 list object calls
fossyy · Sep 23, 2024 · 5ef1ede · 5ef1ede
2 parents 5656c5a + 9452d48
commit 5ef1ede
Show file tree

Hide file tree

Showing 14 changed files with 279 additions and 241 deletions.
diff --git a/handler/file/file.go b/handler/file/file.go
@@ -8,7 +8,6 @@ import (
 	"github.com/fossyy/filekeeper/utils"
 	fileView "github.com/fossyy/filekeeper/view/client/file"
 	"net/http"
-	"strconv"
 )
 
 func GET(w http.ResponseWriter, r *http.Request) {
@@ -22,26 +21,14 @@ func GET(w http.ResponseWriter, r *http.Request) {
 	var filesData []types.FileData
 
 	for _, file := range files {
-		prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
-
-		existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
+		userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
 		if err != nil {
-			app.Server.Logger.Error(err.Error())
 			w.WriteHeader(http.StatusInternalServerError)
+			app.Server.Logger.Error(err.Error())
 			return
 		}
 
-		missingChunk := len(existingChunks) != int(file.TotalChunk)
-
-		filesData = append(filesData, types.FileData{
-			ID:         file.ID.String(),
-			Name:       file.Name,
-			Size:       utils.ConvertFileSize(file.Size),
-			IsPrivate:  file.IsPrivate,
-			Type:       file.Type,
-			Done:       !missingChunk,
-			Downloaded: strconv.FormatUint(file.Downloaded, 10),
-		})
+		filesData = append(filesData, *userFile)
 	}
 
 	allowance, err := app.Server.Database.GetAllowance(userSession.UserID)
@@ -50,7 +37,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
 		app.Server.Logger.Error(err.Error())
 		return
 	}
-	usage, err := app.Server.Service.GetUserStorageUsage(userSession.UserID.String())
+	usage, err := app.Server.Service.GetUserStorageUsage(r.Context(), userSession.UserID.String())
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())

diff --git a/handler/file/query/query.go b/handler/file/query/query.go
@@ -1,13 +1,10 @@
 package queryHandler
 
 import (
-	"fmt"
 	"github.com/fossyy/filekeeper/app"
 	"github.com/fossyy/filekeeper/types"
-	"github.com/fossyy/filekeeper/utils"
 	fileView "github.com/fossyy/filekeeper/view/client/file"
 	"net/http"
-	"strconv"
 )
 
 func GET(w http.ResponseWriter, r *http.Request) {
@@ -34,26 +31,14 @@ func GET(w http.ResponseWriter, r *http.Request) {
 	var filesData []types.FileData
 
 	for _, file := range files {
-		prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
-
-		existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
+		userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
 		if err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
 			app.Server.Logger.Error(err.Error())
 			return
 		}
 
-		missingChunk := len(existingChunks) != int(file.TotalChunk)
-
-		filesData = append(filesData, types.FileData{
-			ID:         file.ID.String(),
-			Name:       file.Name,
-			Size:       utils.ConvertFileSize(file.Size),
-			IsPrivate:  file.IsPrivate,
-			Type:       file.Type,
-			Done:       !missingChunk,
-			Downloaded: strconv.FormatUint(file.Downloaded, 10),
-		})
+		filesData = append(filesData, *userFile)
 	}
 
 	if r.Header.Get("hx-request") == "true" {

diff --git a/handler/file/rename/rename.go b/handler/file/rename/rename.go
@@ -1,13 +1,10 @@
 package renameFileHandler
 
 import (
-	"fmt"
 	"github.com/fossyy/filekeeper/app"
 	"github.com/fossyy/filekeeper/types"
-	"github.com/fossyy/filekeeper/utils"
 	fileView "github.com/fossyy/filekeeper/view/client/file"
 	"net/http"
-	"strconv"
 )
 
 func PATCH(w http.ResponseWriter, r *http.Request) {
@@ -39,28 +36,14 @@ func PATCH(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
-
-	existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
+	userFile, err := app.Server.Service.GetUserFile(r.Context(), newFile.Name, newFile.OwnerID.String())
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())
 		return
 	}
 
-	missingChunk := len(existingChunks) != int(file.TotalChunk)
-
-	fileData := types.FileData{
-		ID:         newFile.ID.String(),
-		Name:       newFile.Name,
-		Size:       utils.ConvertFileSize(newFile.Size),
-		IsPrivate:  newFile.IsPrivate,
-		Type:       newFile.Type,
-		Done:       !missingChunk,
-		Downloaded: strconv.FormatUint(newFile.Downloaded, 10),
-	}
-
-	component := fileView.JustFile(fileData)
+	component := fileView.JustFile(*userFile)
 	err = component.Render(r.Context(), w)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)

diff --git a/handler/file/upload/upload.go b/handler/file/upload/upload.go
@@ -16,7 +16,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	file, err := app.Server.Service.GetFile(fileID)
+	file, err := app.Server.Service.GetFile(r.Context(), fileID)
 	if err != nil {
 		app.Server.Logger.Error("error getting upload info: " + err.Error())
 		w.WriteHeader(http.StatusInternalServerError)
@@ -52,7 +52,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
 		app.Server.Logger.Error("error copying byte to file dst: " + err.Error())
 		return
 	}
-
+	app.Server.Service.UpdateFileChunk(r.Context(), file.ID, file.OwnerID, rawIndex, file.TotalChunk)
 	w.WriteHeader(http.StatusAccepted)
 	return
 }
diff --git a/handler/file/visibility/visibility.go b/handler/file/visibility/visibility.go
@@ -1,13 +1,10 @@
 package visibilityHandler
 
 import (
-	"fmt"
 	"github.com/fossyy/filekeeper/app"
 	"github.com/fossyy/filekeeper/types"
-	"github.com/fossyy/filekeeper/utils"
 	fileView "github.com/fossyy/filekeeper/view/client/file"
 	"net/http"
-	"strconv"
 )
 
 func PUT(w http.ResponseWriter, r *http.Request) {
@@ -32,26 +29,13 @@ func PUT(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	prefix := fmt.Sprintf("%s/%s/chunk_", file.OwnerID.String(), file.ID.String())
-
-	existingChunks, err := app.Server.Storage.ListObjects(r.Context(), prefix)
+	userFile, err := app.Server.Service.GetUserFile(r.Context(), file.Name, file.OwnerID.String())
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())
 		return
 	}
-
-	missingChunk := len(existingChunks) != int(file.TotalChunk)
-	fileData := types.FileData{
-		ID:         file.ID.String(),
-		Name:       file.Name,
-		Size:       utils.ConvertFileSize(file.Size),
-		IsPrivate:  !file.IsPrivate,
-		Type:       file.Type,
-		Done:       !missingChunk,
-		Downloaded: strconv.FormatUint(file.Downloaded, 10),
-	}
-	component := fileView.JustFile(fileData)
+	component := fileView.JustFile(*userFile)
 	err = component.Render(r.Context(), w)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)

diff --git a/handler/forgotPassword/verify/verify.go b/handler/forgotPassword/verify/verify.go
@@ -122,7 +122,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = app.Server.Service.DeleteUser(userData.User.Email)
+	err = app.Server.Service.DeleteUser(r.Context(), userData.User.Email)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())

diff --git a/handler/user/ResetPassword/ResetPassword.go b/handler/user/ResetPassword/ResetPassword.go
@@ -51,7 +51,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = app.Server.Service.DeleteUser(userSession.Email)
+	err = app.Server.Service.DeleteUser(r.Context(), userSession.Email)
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())

diff --git a/handler/user/totp/setup.go b/handler/user/totp/setup.go
@@ -87,7 +87,7 @@ func POST(w http.ResponseWriter, r *http.Request) {
 			app.Server.Logger.Error(err.Error())
 			return
 		}
-		err := app.Server.Service.DeleteUser(userSession.Email)
+		err := app.Server.Service.DeleteUser(r.Context(), userSession.Email)
 		if err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
 			app.Server.Logger.Error(err.Error())

diff --git a/handler/user/user.go b/handler/user/user.go
@@ -76,7 +76,7 @@ func GET(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	usage, err := app.Server.Service.GetUserStorageUsage(userSession.UserID.String())
+	usage, err := app.Server.Service.GetUserStorageUsage(r.Context(), userSession.UserID.String())
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		app.Server.Logger.Error(err.Error())
@@ -178,8 +178,9 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
 
 			if err != nil {
 				if errors.Is(err, gorm.ErrRecordNotFound) {
+					fileID := uuid.New()
 					newFile := models.File{
-						ID:         uuid.New(),
+						ID:         fileID,
 						OwnerID:    userSession.UserID,
 						Name:       uploadNewFile.Name,
 						Size:       uploadNewFile.Size,
@@ -189,42 +190,24 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
 						TotalChunk: uploadNewFile.Chunk,
 						Downloaded: 0,
 					}
+
 					err := app.Server.Database.CreateFile(&newFile)
 					if err != nil {
 						sendErrorResponse(conn, action.Action, "Error Creating File")
 						continue
 					}
-					fileData := &types.FileWithDetail{
-						ID:         newFile.ID,
-						OwnerID:    newFile.OwnerID,
-						Name:       newFile.Name,
-						Size:       newFile.Size,
-						Downloaded: newFile.Downloaded,
-						Done:       false,
-					}
-					fileData.Chunk = make(map[string]bool)
 
-					prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), newFile.ID.String())
-
-					existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
+					userFile, err := app.Server.Service.GetUserFile(context.Background(), uploadNewFile.Name, userSession.UserID.String())
 					if err != nil {
 						app.Server.Logger.Error(err.Error())
 						sendErrorResponse(conn, action.Action, "Unknown error")
 						continue
-					} else {
-						for i := 0; i < int(newFile.TotalChunk); i++ {
-							fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
-						}
-
-						for _, chunkFile := range existingChunks {
-							var chunkIndex int
-							fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
-							fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
-						}
 					}
-					sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
+
+					sendSuccessResponseWithID(conn, action.Action, userFile, uploadNewFile.RequestID)
 					continue
 				} else {
+					app.Server.Logger.Error(err.Error())
 					sendErrorResponse(conn, action.Action, "Unknown error")
 					continue
 				}
@@ -233,40 +216,14 @@ func handlerWS(conn *websocket.Conn, userSession types.User) {
 				sendErrorResponse(conn, action.Action, "File Is Different")
 				continue
 			}
-			fileData := &types.FileWithDetail{
-				ID:         file.ID,
-				OwnerID:    file.OwnerID,
-				Name:       file.Name,
-				Size:       file.Size,
-				Downloaded: file.Downloaded,
-				Chunk:      make(map[string]bool),
-				Done:       true,
-			}
-
-			prefix := fmt.Sprintf("%s/%s/chunk_", userSession.UserID.String(), file.ID.String())
-			existingChunks, err := app.Server.Storage.ListObjects(context.TODO(), prefix)
+			userFile, err := app.Server.Service.GetUserFile(context.Background(), file.Name, userSession.UserID.String())
 			if err != nil {
 				app.Server.Logger.Error(err.Error())
-				fileData.Done = false
-			} else {
-				for i := 0; i < int(file.TotalChunk); i++ {
-					fileData.Chunk[fmt.Sprintf("chunk_%d", i)] = false
-				}
-				for _, chunkFile := range existingChunks {
-					var chunkIndex int
-					fmt.Sscanf(chunkFile, "chunk_%d", &chunkIndex)
-					fileData.Chunk[fmt.Sprintf("chunk_%d", chunkIndex)] = true
-				}
-
-				for i := 0; i < int(file.TotalChunk); i++ {
-					if !fileData.Chunk[fmt.Sprintf("chunk_%d", i)] {
-						fileData.Done = false
-						break
-					}
-				}
+				sendErrorResponse(conn, action.Action, "Unknown error")
+				continue
 			}
 
-			sendSuccessResponseWithID(conn, action.Action, fileData, uploadNewFile.RequestID)
+			sendSuccessResponseWithID(conn, action.Action, userFile, uploadNewFile.RequestID)
 			continue
 		case Ping:
 			sendSuccessResponse(conn, action.Action, map[string]string{"message": "received"})