From 45899dc72ff368738fe5159a02c38ca5844f7427 Mon Sep 17 00:00:00 2001 From: jarvis8x7b <157810922+jarvis8x7b@users.noreply.github.com> Date: Mon, 14 Oct 2024 05:23:13 +0800 Subject: [PATCH 1/3] fix: substrate cache time for max uid / runspec --- cmd/seed/main.go | 5 +++-- pkg/blockchain/substrate.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/seed/main.go b/cmd/seed/main.go index 444443c..3f73f1b 100644 --- a/cmd/seed/main.go +++ b/cmd/seed/main.go @@ -2,14 +2,15 @@ package main import ( "context" - "dojo-api/db" - "dojo-api/pkg/orm" "encoding/json" "io" "os" "path/filepath" "time" + "dojo-api/db" + "dojo-api/pkg/orm" + "github.com/rs/zerolog/log" "github.com/steebchen/prisma-client-go/runtime/types" ) diff --git a/pkg/blockchain/substrate.go b/pkg/blockchain/substrate.go index 0771512..99ee544 100644 --- a/pkg/blockchain/substrate.go +++ b/pkg/blockchain/substrate.go @@ -171,7 +171,7 @@ func (s *SubstrateService) GetMaxUID(subnetId int) (int, error) { return 0, err } - s.cache.SetWithExpire(CacheKeyMaxUID, valueStr, 24*7*time.Hour) + s.cache.SetWithExpire(CacheKeyMaxUID, valueStr, 5*BlockTimeInSeconds*time.Second) return maxUID, nil } @@ -419,7 +419,7 @@ func (s *SubstrateService) RuntimeSpec() (*RuntimeSpec, error) { return nil, err } - s.cache.SetWithExpire(CacheKeyRuntimeSpec, string(body), 12*time.Hour) + s.cache.SetWithExpire(CacheKeyRuntimeSpec, string(body), 24*time.Hour) return &runtimeSpec, nil } From 366a26ae3a4009b17eb6719bba65de355433edd1 Mon Sep 17 00:00:00 2001 From: jarvis8x7b <157810922+jarvis8x7b@users.noreply.github.com> Date: Thu, 17 Oct 2024 05:37:32 +0800 Subject: [PATCH 2/3] fix: double multipart max memory, increase deadline & error logging (#30) * perf: increase context timeout, double max multipart memory to 64mb * refactor: proper logging of context deadline, increase to 10s * chore: update docs --- README.md | 2 +- cmd/server/main.go | 15 +++++------ pkg/api/controllers.go | 4 +-- pkg/task/task_service.go | 54 +++++++++++++++++++++++++++++++++------- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 02eeec1..3fc971a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Dojo Subnet API +# Dojo Worker API Repository for our Dojo Subnet APIs. Check request.REST file for developed APIs to test out.(make sure that REST client extension is already installed) Extension ID: humao.rest-client diff --git a/cmd/server/main.go b/cmd/server/main.go index af806bf..ebb606a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,10 +2,6 @@ package main import ( "context" - "dojo-api/pkg/api" - "dojo-api/pkg/cache" - "dojo-api/pkg/orm" - "dojo-api/utils" "net/http" "os" "os/signal" @@ -13,6 +9,11 @@ import ( "syscall" "time" + "dojo-api/pkg/api" + "dojo-api/pkg/cache" + "dojo-api/pkg/orm" + "dojo-api/utils" + _ "dojo-api/docs" "github.com/gin-contrib/cors" @@ -23,9 +24,9 @@ import ( ginSwagger "github.com/swaggo/gin-swagger" ) -// @title Dojo Worker API -// @version 1.0 -// @description This is the worker API for the Dojo project. +// @title Dojo Worker API +// @version 1.0 +// @description This is the worker API for the Dojo project. func main() { loadEnvVars() diff --git a/pkg/api/controllers.go b/pkg/api/controllers.go index 4359a77..12ea940 100644 --- a/pkg/api/controllers.go +++ b/pkg/api/controllers.go @@ -173,14 +173,14 @@ func CreateTasksController(c *gin.Context) { } taskService := task.NewTaskService() - tasks, errors := taskService.CreateTasks(requestBody, minerUser.ID) + tasks, errors := taskService.CreateTasksWithTimeout(requestBody, minerUser.ID, 10*time.Second) - log.Info().Msg("Tasks created successfully") if len(tasks) == 0 { c.AbortWithStatusJSON(http.StatusBadRequest, defaultErrorResponse(errors)) return } + log.Info().Msg("Tasks created successfully") taskIds := make([]string, 0, len(tasks)) for _, task := range tasks { taskIds = append(taskIds, task.ID) diff --git a/pkg/task/task_service.go b/pkg/task/task_service.go index f213971..e135073 100644 --- a/pkg/task/task_service.go +++ b/pkg/task/task_service.go @@ -2,10 +2,6 @@ package task import ( "context" - "dojo-api/db" - "dojo-api/pkg/orm" - "dojo-api/pkg/sandbox" - "dojo-api/utils" "encoding/json" "errors" "fmt" @@ -16,6 +12,11 @@ import ( "strconv" "time" + "dojo-api/db" + "dojo-api/pkg/orm" + "dojo-api/pkg/sandbox" + "dojo-api/utils" + "github.com/gin-gonic/gin" "github.com/rs/zerolog/log" ) @@ -196,11 +197,41 @@ func IsValidCriteriaType(criteriaType CriteriaType) bool { } } -// create task -func (s *TaskService) CreateTasks(request CreateTaskRequest, minerUserId string) ([]*db.TaskModel, []error) { - ctxWithTimeout, cancel := context.WithTimeout(context.Background(), 3*time.Second) +func (s *TaskService) CreateTasksWithTimeout(request CreateTaskRequest, minerUserId string, timeout time.Duration) ([]*db.TaskModel, []error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + type result struct { + tasks []*db.TaskModel + errs []error + } + + resultChan := make(chan result, 1) + + go func() { + tasks, errs := s.CreateTasks(ctx, request, minerUserId) + resultChan <- result{tasks: tasks, errs: errs} + }() + + select { + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + log.Error().Dur("timeout", timeout).Msg("CreateTasks timed out due to deadline") + return nil, []error{fmt.Errorf("operation timed out after %v", timeout)} + } + log.Error().Err(ctx.Err()).Msg("Context canceled while creating tasks") + return nil, []error{ctx.Err()} + case res := <-resultChan: + if len(res.tasks) == 0 && len(res.errs) == 0 { + log.Warn().Msg("No tasks created and no errors reported") + return nil, []error{fmt.Errorf("no tasks were created and no errors were reported")} + } + return res.tasks, res.errs + } +} + +// create task +func (s *TaskService) CreateTasks(ctx context.Context, request CreateTaskRequest, minerUserId string) ([]*db.TaskModel, []error) { tasks := make([]*db.TaskModel, 0) errors := make([]error, 0) @@ -244,7 +275,7 @@ func (s *TaskService) CreateTasks(request CreateTaskRequest, minerUserId string) taskToCreate.TotalReward = &request.TotalRewards } - task, err := taskORM.CreateTask(ctxWithTimeout, taskToCreate, minerUserId) + task, err := taskORM.CreateTask(ctx, taskToCreate, minerUserId) if err != nil { log.Error().Msgf("Error creating task: %v", err) errors = append(errors, err) @@ -505,7 +536,6 @@ func ValidateTaskData(taskData TaskData) error { return fmt.Errorf("invalid completion format: %v", taskresponse.Completion) } } - } if len(taskData.Criteria) == 0 { @@ -679,6 +709,12 @@ func (t *TaskService) GetCompletedTaskMap(ctx context.Context, workerId string) } func ProcessRequestBody(c *gin.Context) (CreateTaskRequest, error) { + // set max memory to 64 MB + if err := c.Request.ParseMultipartForm(64 << 20); err != nil { + log.Error().Err(err).Msg("Failed to parse multipart form") + return CreateTaskRequest{}, fmt.Errorf("failed to parse form: %w", err) + } + var reqbody CreateTaskRequest title := c.PostForm("title") body := c.PostForm("body") From 407a99b0c2f2d2db3e53c13b9826e5e59e4ea4aa Mon Sep 17 00:00:00 2001 From: jarvis8x7b <157810922+jarvis8x7b@users.noreply.github.com> Date: Thu, 17 Oct 2024 05:51:49 +0800 Subject: [PATCH 3/3] fix: increase timeout to 60s --- pkg/api/controllers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/controllers.go b/pkg/api/controllers.go index 12ea940..795f327 100644 --- a/pkg/api/controllers.go +++ b/pkg/api/controllers.go @@ -173,7 +173,7 @@ func CreateTasksController(c *gin.Context) { } taskService := task.NewTaskService() - tasks, errors := taskService.CreateTasksWithTimeout(requestBody, minerUser.ID, 10*time.Second) + tasks, errors := taskService.CreateTasksWithTimeout(requestBody, minerUser.ID, 60*time.Second) if len(tasks) == 0 { c.AbortWithStatusJSON(http.StatusBadRequest, defaultErrorResponse(errors))