feat: add repo stats to restic package

backrest.go

@@ -73,7 +73,7 @@ func main() {
 	// Create and serve the HTTP gateway
 	apiServer := api.NewServer(
 		configStore,
-		orchestrator, // TODO: eliminate default config
+		orchestrator,
 		oplog,
 	)
 

internal/cache/cache.go

@@ -0,0 +1,34 @@
+package cache
+
+import (
+	"fmt"
+
+	v1 "github.com/garethgeorge/backrest/gen/go/v1"
+	"go.etcd.io/bbolt"
+)
+
+type Cache struct {
+	db *bbolt.DB
+}
+
+func NewCache(databasePath string) (*Cache, error) {
+	db, err := bbolt.Open(databasePath, 0600, nil)
+	if err != nil {
+		return nil, fmt.Errorf("open database: %w", err)
+	}
+	return &Cache{db: db}, nil
+}
+
+func (c *Cache) Close() error {
+	return c.db.Close()
+}
+
+func (c *Cache) SetRepoStats(repo *v1.Repo, stats *v1.RepoStats) error {
+	return c.db.Update(func(tx *bbolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("repo_stats"))
+		if err != nil {
+			return fmt.Errorf("create bucket: %w", err)
+		}
+		return b.Put([]byte(repo.Id), protoutil.MustMarshal(stats))
+	})
+}

internal/oplog/oplog.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"path"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	v1 "github.com/garethgeorge/backrest/gen/go/v1"
@@ -30,12 +29,11 @@ const (
 var ErrNotExist = errors.New("operation does not exist")
 
 var (
-	SystemBucket          = []byte("oplog.system")          // system stores metadata
-	OpLogBucket           = []byte("oplog.log")             // oplog stores existant operations.
-	OpLogSoftDeleteBucket = []byte("oplog.log_soft_delete") // oplog_soft_delete stores soft deleted operations
-	RepoIndexBucket       = []byte("oplog.repo_idx")        // repo_index tracks IDs of operations affecting a given repo
-	PlanIndexBucket       = []byte("oplog.plan_idx")        // plan_index tracks IDs of operations affecting a given plan
-	SnapshotIndexBucket   = []byte("oplog.snapshot_idx")    // snapshot_index tracks IDs of operations affecting a given snapshot
+	SystemBucket        = []byte("oplog.system")       // system stores metadata
+	OpLogBucket         = []byte("oplog.log")          // oplog stores existant operations.
+	RepoIndexBucket     = []byte("oplog.repo_idx")     // repo_index tracks IDs of operations affecting a given repo
+	PlanIndexBucket     = []byte("oplog.plan_idx")     // plan_index tracks IDs of operations affecting a given plan
+	SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot
 )
 
 // OpLog represents a log of operations performed.
@@ -45,7 +43,6 @@ type OpLog struct {
 
 	subscribersMu sync.RWMutex
 	subscribers   []*func(*v1.Operation, *v1.Operation)
-	nextId        atomic.Int64
 }
 
 func NewOpLog(databasePath string) (*OpLog, error) {
@@ -61,12 +58,11 @@ func NewOpLog(databasePath string) (*OpLog, error) {
 	o := &OpLog{
 		db: db,
 	}
-	o.nextId.Store(1)
 
 	if err := db.Update(func(tx *bolt.Tx) error {
 		// Create the buckets if they don't exist
 		for _, bucket := range [][]byte{
-			SystemBucket, OpLogBucket, OpLogSoftDeleteBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
+			SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket,
 		} {
 			if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
 				return fmt.Errorf("creating bucket %s: %s", string(bucket), err)
@@ -194,17 +190,7 @@ func (o *OpLog) Delete(ids ...int64) error {
 				return fmt.Errorf("deleting operation %v: %w", id, err)
 			}
 			removedOps = append(removedOps, removed)
-
-			b := tx.Bucket(OpLogSoftDeleteBucket)
-			val, err := proto.Marshal(removed)
-			if err != nil {
-				return fmt.Errorf("marshalling operation %v: %w", id, err)
-			}
-			if err := b.Put(serializationutil.Itob(id), val); err != nil {
-				return fmt.Errorf("putting operation %v into soft delete bucket: %w", id, err)
-			}
 		}
-
 		return nil
 	})
 	if err == nil {

internal/protoutil/conversion.go

@@ -107,3 +107,13 @@ func RestoreProgressEntryToProto(p *restic.RestoreProgressEntry) *v1.RestoreProg
 		PercentDone:   p.PercentDone,
 	}
 }
+
+func RepoStatsToProto(s *restic.RepoStats) *v1.RepoStats {
+	return &v1.RepoStats{
+		TotalSize:             int64(s.TotalSize),
+		TotalUncompressedSize: int64(s.TotalUncompressedSize),
+		CompressionRatio:      s.CompressionRatio,
+		TotalBlobCount:        int64(s.TotalBlobCount),
+		SnapshotCount:         int64(s.SnapshotsCount),
+	}
+}

internal/protoutil/conversion_test.go

@@ -116,3 +116,45 @@ func TestBackupProgressEntryToProto(t *testing.T) {
 		})
 	}
 }
+
+func TestRepoStatsToProto(t *testing.T) {
+	cases := []struct {
+		name  string
+		stats *restic.RepoStats
+		want  *v1.RepoStats
+	}{
+		{
+			name:  "no stats",
+			stats: &restic.RepoStats{},
+			want:  &v1.RepoStats{},
+		},
+		{
+			name: "with stats",
+			stats: &restic.RepoStats{
+				TotalSize:              1,
+				TotalUncompressedSize:  2,
+				CompressionRatio:       3,
+				TotalBlobCount:         5,
+				CompressionProgress:    6,
+				CompressionSpaceSaving: 7,
+				SnapshotsCount:         8,
+			},
+			want: &v1.RepoStats{
+				TotalSize:             1,
+				TotalUncompressedSize: 2,
+				CompressionRatio:      3,
+				TotalBlobCount:        5,
+				SnapshotCount:         8,
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			got := RepoStatsToProto(c.stats)
+			if !proto.Equal(got, c.want) {
+				t.Errorf("wanted: %+v, got: %+v", c.want, got)
+			}
+		})
+	}
+}

pkg/restic/outputs.go

@@ -304,3 +304,13 @@ func ValidateSnapshotId(id string) error {
 	}
 	return nil
 }
+
+type RepoStats struct {
+	TotalSize              int64   `json:"total_size"`
+	TotalUncompressedSize  int64   `json:"total_uncompressed_size"`
+	CompressionRatio       float64 `json:"compression_ratio"`
+	CompressionProgress    int64   `json:"compression_progress"`
+	CompressionSpaceSaving float64 `json:"compression_space_saving"`
+	TotalBlobCount         int64   `json:"total_blob_count"`
+	SnapshotsCount         int64   `json:"snapshots_count"`
+}

pkg/restic/restic.go

@@ -347,6 +347,30 @@ func (r *Repo) Unlock(ctx context.Context, opts ...GenericOption) error {
 	return nil
 }
 
+func (r *Repo) Stats(ctx context.Context, opts ...GenericOption) (*RepoStats, error) {
+	opt := resolveOpts(opts)
+
+	args := []string{"stats", "--json", "--mode=raw-data"}
+	args = append(args, r.extraArgs...)
+	args = append(args, opt.extraArgs...)
+
+	cmd := exec.CommandContext(ctx, r.cmd, args...)
+	cmd.Env = append(cmd.Env, r.buildEnv()...)
+	cmd.Env = append(cmd.Env, opt.extraEnv...)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, newCmdError(cmd, string(output), err)
+	}
+
+	var stats RepoStats
+	if err := json.Unmarshal(output, &stats); err != nil {
+		return nil, newCmdError(cmd, string(output), fmt.Errorf("command output is not valid JSON: %w", err))
+	}
+
+	return &stats, nil
+}
+
 type RetentionPolicy struct {
 	KeepLastN          int    // keep the last n snapshots.
 	KeepHourly         int    // keep the last n hourly snapshots.