Skip to content

Commit

Permalink
feat: add stats to repo view e.g. total size in storage
Browse files Browse the repository at this point in the history
  • Loading branch information
garethgeorge committed Dec 25, 2023
1 parent cc11197 commit adb0e3f
Show file tree
Hide file tree
Showing 20 changed files with 697 additions and 366 deletions.
242 changes: 164 additions & 78 deletions gen/go/v1/operations.pb.go

Large diffs are not rendered by default.

246 changes: 121 additions & 125 deletions gen/go/v1/service.pb.go

Large diffs are not rendered by default.

41 changes: 41 additions & 0 deletions gen/go/v1/service_grpc.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 33 additions & 0 deletions gen/go/v1/v1connect/service.connect.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 4 additions & 5 deletions internal/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ func (s *Server) Forget(ctx context.Context, req *connect.Request[types.StringVa
at := time.Now()

s.orchestrator.ScheduleTask(orchestrator.NewOneofForgetTask(s.orchestrator, plan, "", at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityForget)
s.orchestrator.ScheduleTask(orchestrator.NewOneofForgetTask(s.orchestrator, plan, "", at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots)
s.orchestrator.ScheduleTask(orchestrator.NewOneofIndexSnapshotsTask(s.orchestrator, plan, at), orchestrator.TaskPriorityInteractive+orchestrator.TaskPriorityIndexSnapshots)

return connect.NewResponse(&emptypb.Empty{}), nil
}
Expand All @@ -285,9 +285,9 @@ func (s *Server) Prune(ctx context.Context, req *connect.Request[types.StringVal
}

func (s *Server) Restore(ctx context.Context, req *connect.Request[v1.RestoreSnapshotRequest]) (*connect.Response[emptypb.Empty], error) {
_, err := s.orchestrator.GetRepo(req.Msg.RepoId)
plan, err := s.orchestrator.GetPlan(req.Msg.PlanId)
if err != nil {
return nil, fmt.Errorf("failed to get repo %q: %w", req.Msg.RepoId, err)
return nil, fmt.Errorf("failed to get plan %q: %w", req.Msg.PlanId, err)
}

if req.Msg.Target == "" {
Expand All @@ -303,8 +303,7 @@ func (s *Server) Restore(ctx context.Context, req *connect.Request[v1.RestoreSna
at := time.Now()

s.orchestrator.ScheduleTask(orchestrator.NewOneofRestoreTask(s.orchestrator, orchestrator.RestoreTaskOpts{
PlanId: req.Msg.PlanId,
RepoId: req.Msg.RepoId,
Plan: plan,
SnapshotId: req.Msg.SnapshotId,
Path: req.Msg.Path,
Target: target,
Expand Down
11 changes: 6 additions & 5 deletions internal/orchestrator/orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,12 @@ var ErrRepoInitializationFailed = errors.New("repo initialization failed")
var ErrPlanNotFound = errors.New("plan not found")

const (
TaskPriorityDefault = iota
TaskPriorityIndexSnapshots
TaskPriorityPrune
TaskPriorityForget
TaskPriorityInteractive // highest priority (add other priorities to this value for offsets)
TaskPriorityDefault = 0
TaskPriorityInteractive = 10
TaskPriorityIndexSnapshots = 101
TaskPriorityForget = 102
TaskPriorityPrune = 103
TaskPriorityStats = 104
)

// Orchestrator is responsible for managing repos and backups.
Expand Down
13 changes: 13 additions & 0 deletions internal/orchestrator/repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,19 @@ func (r *RepoOrchestrator) Unlock(ctx context.Context) error {
return nil
}

func (r *RepoOrchestrator) Stats(ctx context.Context) (*v1.RepoStats, error) {
r.mu.Lock()
defer r.mu.Unlock()

r.l.Debug("Get Stats")
stats, err := r.repo.Stats(ctx)
if err != nil {
return nil, fmt.Errorf("stats for repo %v: %w", r.repoConfig.Id, err)
}

return protoutil.RepoStatsToProto(stats), nil
}

func tagForPlan(plan *v1.Plan) string {
return fmt.Sprintf("plan:%s", plan.Id)
}
Expand Down
1 change: 1 addition & 0 deletions internal/orchestrator/taskbackup.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ func backupHelper(ctx context.Context, orchestrator *Orchestrator, plan *v1.Plan
}

orchestrator.ScheduleTask(NewOneofIndexSnapshotsTask(orchestrator, plan, at), TaskPriorityIndexSnapshots)
orchestrator.ScheduleTask(NewOneofStatsTask(orchestrator, plan, op.SnapshotId, at), TaskPriorityStats)

return nil
}
13 changes: 6 additions & 7 deletions internal/orchestrator/taskrestore.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@ import (
)

type RestoreTaskOpts struct {
PlanId string
RepoId string
Plan *v1.Plan
SnapshotId string
Path string
Target string
Expand All @@ -37,16 +36,16 @@ func NewOneofRestoreTask(orchestrator *Orchestrator, opts RestoreTaskOpts, at ti
}

func (t *RestoreTask) Name() string {
return fmt.Sprintf("restore snapshot %v in repo %v", t.restoreOpts.SnapshotId, t.restoreOpts.RepoId)
return fmt.Sprintf("restore snapshot %v in repo %v", t.restoreOpts.SnapshotId, t.restoreOpts.Plan.Repo)
}

func (t *RestoreTask) Next(now time.Time) *time.Time {
ret := t.at
if ret != nil {
t.at = nil
if err := t.setOperation(&v1.Operation{
PlanId: t.restoreOpts.PlanId,
RepoId: t.restoreOpts.RepoId,
PlanId: t.restoreOpts.Plan.Id,
RepoId: t.restoreOpts.Plan.Repo,
SnapshotId: t.restoreOpts.SnapshotId,
UnixTimeStartMs: timeToUnixMillis(*ret),
Status: v1.OperationStatus_STATUS_PENDING,
Expand All @@ -70,9 +69,9 @@ func (t *RestoreTask) Run(ctx context.Context) error {
op.Op = forgetOp
op.UnixTimeStartMs = curTimeMillis()

repo, err := t.orch.GetRepo(t.restoreOpts.RepoId)
repo, err := t.orch.GetRepo(t.restoreOpts.Plan.Repo)
if err != nil {
return fmt.Errorf("couldn't get repo %q: %w", t.restoreOpts.RepoId, err)
return fmt.Errorf("couldn't get repo %q: %w", t.restoreOpts.Plan.Repo, err)
}

lastSent := time.Now() // debounce progress updates, these can endup being very frequent.
Expand Down
Loading

0 comments on commit adb0e3f

Please sign in to comment.