diff --git a/backrest.go b/backrest.go index d416b446..a439c076 100644 --- a/backrest.go +++ b/backrest.go @@ -73,7 +73,7 @@ func main() { // Create and serve the HTTP gateway apiServer := api.NewServer( configStore, - orchestrator, // TODO: eliminate default config + orchestrator, oplog, ) diff --git a/gen/go/v1/restic.pb.go b/gen/go/v1/restic.pb.go index 523ecfdc..bab2171a 100644 --- a/gen/go/v1/restic.pb.go +++ b/gen/go/v1/restic.pb.go @@ -582,6 +582,85 @@ func (x *RestoreProgressEntry) GetPercentDone() float64 { return 0 } +type RepoStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TotalSize int64 `protobuf:"varint,1,opt,name=total_size,json=totalSize,proto3" json:"total_size,omitempty"` + TotalUncompressedSize int64 `protobuf:"varint,2,opt,name=total_uncompressed_size,json=totalUncompressedSize,proto3" json:"total_uncompressed_size,omitempty"` + CompressionRatio float64 `protobuf:"fixed64,3,opt,name=compression_ratio,json=compressionRatio,proto3" json:"compression_ratio,omitempty"` + TotalBlobCount int64 `protobuf:"varint,5,opt,name=total_blob_count,json=totalBlobCount,proto3" json:"total_blob_count,omitempty"` + SnapshotCount int64 `protobuf:"varint,6,opt,name=snapshot_count,json=snapshotCount,proto3" json:"snapshot_count,omitempty"` +} + +func (x *RepoStats) Reset() { + *x = RepoStats{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_restic_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepoStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepoStats) ProtoMessage() {} + +func (x *RepoStats) ProtoReflect() protoreflect.Message { + mi := &file_v1_restic_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepoStats.ProtoReflect.Descriptor instead. +func (*RepoStats) Descriptor() ([]byte, []int) { + return file_v1_restic_proto_rawDescGZIP(), []int{6} +} + +func (x *RepoStats) GetTotalSize() int64 { + if x != nil { + return x.TotalSize + } + return 0 +} + +func (x *RepoStats) GetTotalUncompressedSize() int64 { + if x != nil { + return x.TotalUncompressedSize + } + return 0 +} + +func (x *RepoStats) GetCompressionRatio() float64 { + if x != nil { + return x.CompressionRatio + } + return 0 +} + +func (x *RepoStats) GetTotalBlobCount() int64 { + if x != nil { + return x.TotalBlobCount + } + return 0 +} + +func (x *RepoStats) GetSnapshotCount() int64 { + if x != nil { + return x.SnapshotCount + } + return 0 +} + var File_v1_restic_proto protoreflect.FileDescriptor var file_v1_restic_proto_rawDesc = []byte{ @@ -676,10 +755,25 @@ var file_v1_restic_proto_rawDesc = []byte{ 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x70, 0x65, 0x72, - 0x63, 0x65, 0x6e, 0x74, 0x44, 0x6f, 0x6e, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, 0x67, 0x65, 0x6f, - 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, 0x67, 0x65, 0x6e, - 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x65, 0x6e, 0x74, 0x44, 0x6f, 0x6e, 0x65, 0x22, 0xe0, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x70, + 0x6f, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x75, + 0x6e, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x5f, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, + 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2b, 0x0a, + 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x74, 0x69, 0x6f, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x42, 0x6c, 0x6f, 0x62, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x6e, + 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x42, 0x2c, 0x5a, 0x2a, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x61, 0x72, 0x65, 0x74, 0x68, + 0x67, 0x65, 0x6f, 0x72, 0x67, 0x65, 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x72, 0x65, 0x73, 0x74, 0x2f, + 0x67, 0x65, 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -694,7 +788,7 @@ func file_v1_restic_proto_rawDescGZIP() []byte { return file_v1_restic_proto_rawDescData } -var file_v1_restic_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_v1_restic_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_v1_restic_proto_goTypes = []interface{}{ (*ResticSnapshot)(nil), // 0: v1.ResticSnapshot (*ResticSnapshotList)(nil), // 1: v1.ResticSnapshotList @@ -702,6 +796,7 @@ var file_v1_restic_proto_goTypes = []interface{}{ (*BackupProgressStatusEntry)(nil), // 3: v1.BackupProgressStatusEntry (*BackupProgressSummary)(nil), // 4: v1.BackupProgressSummary (*RestoreProgressEntry)(nil), // 5: v1.RestoreProgressEntry + (*RepoStats)(nil), // 6: v1.RepoStats } var file_v1_restic_proto_depIdxs = []int32{ 0, // 0: v1.ResticSnapshotList.snapshots:type_name -> v1.ResticSnapshot @@ -792,6 +887,18 @@ func file_v1_restic_proto_init() { return nil } } + file_v1_restic_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepoStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_v1_restic_proto_msgTypes[2].OneofWrappers = []interface{}{ (*BackupProgressEntry_Status)(nil), @@ -803,7 +910,7 @@ func file_v1_restic_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_v1_restic_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 00000000..69d97e0e --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,34 @@ +package cache + +import ( + "fmt" + + v1 "github.com/garethgeorge/backrest/gen/go/v1" + "go.etcd.io/bbolt" +) + +type Cache struct { + db *bbolt.DB +} + +func NewCache(databasePath string) (*Cache, error) { + db, err := bbolt.Open(databasePath, 0600, nil) + if err != nil { + return nil, fmt.Errorf("open database: %w", err) + } + return &Cache{db: db}, nil +} + +func (c *Cache) Close() error { + return c.db.Close() +} + +func (c *Cache) SetRepoStats(repo *v1.Repo, stats *v1.RepoStats) error { + return c.db.Update(func(tx *bbolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("repo_stats")) + if err != nil { + return fmt.Errorf("create bucket: %w", err) + } + return b.Put([]byte(repo.Id), protoutil.MustMarshal(stats)) + }) +} diff --git a/internal/oplog/oplog.go b/internal/oplog/oplog.go index f6f675a1..ab8632e2 100644 --- a/internal/oplog/oplog.go +++ b/internal/oplog/oplog.go @@ -6,7 +6,6 @@ import ( "os" "path" "sync" - "sync/atomic" "time" v1 "github.com/garethgeorge/backrest/gen/go/v1" @@ -30,12 +29,11 @@ const ( var ErrNotExist = errors.New("operation does not exist") var ( - SystemBucket = []byte("oplog.system") // system stores metadata - OpLogBucket = []byte("oplog.log") // oplog stores existant operations. - OpLogSoftDeleteBucket = []byte("oplog.log_soft_delete") // oplog_soft_delete stores soft deleted operations - RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo - PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan - SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot + SystemBucket = []byte("oplog.system") // system stores metadata + OpLogBucket = []byte("oplog.log") // oplog stores existant operations. + RepoIndexBucket = []byte("oplog.repo_idx") // repo_index tracks IDs of operations affecting a given repo + PlanIndexBucket = []byte("oplog.plan_idx") // plan_index tracks IDs of operations affecting a given plan + SnapshotIndexBucket = []byte("oplog.snapshot_idx") // snapshot_index tracks IDs of operations affecting a given snapshot ) // OpLog represents a log of operations performed. @@ -45,7 +43,6 @@ type OpLog struct { subscribersMu sync.RWMutex subscribers []*func(*v1.Operation, *v1.Operation) - nextId atomic.Int64 } func NewOpLog(databasePath string) (*OpLog, error) { @@ -61,12 +58,11 @@ func NewOpLog(databasePath string) (*OpLog, error) { o := &OpLog{ db: db, } - o.nextId.Store(1) if err := db.Update(func(tx *bolt.Tx) error { // Create the buckets if they don't exist for _, bucket := range [][]byte{ - SystemBucket, OpLogBucket, OpLogSoftDeleteBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket, + SystemBucket, OpLogBucket, RepoIndexBucket, PlanIndexBucket, SnapshotIndexBucket, } { if _, err := tx.CreateBucketIfNotExists(bucket); err != nil { return fmt.Errorf("creating bucket %s: %s", string(bucket), err) @@ -194,17 +190,7 @@ func (o *OpLog) Delete(ids ...int64) error { return fmt.Errorf("deleting operation %v: %w", id, err) } removedOps = append(removedOps, removed) - - b := tx.Bucket(OpLogSoftDeleteBucket) - val, err := proto.Marshal(removed) - if err != nil { - return fmt.Errorf("marshalling operation %v: %w", id, err) - } - if err := b.Put(serializationutil.Itob(id), val); err != nil { - return fmt.Errorf("putting operation %v into soft delete bucket: %w", id, err) - } } - return nil }) if err == nil { diff --git a/internal/protoutil/conversion.go b/internal/protoutil/conversion.go index 08a99070..7f1c7e24 100644 --- a/internal/protoutil/conversion.go +++ b/internal/protoutil/conversion.go @@ -107,3 +107,13 @@ func RestoreProgressEntryToProto(p *restic.RestoreProgressEntry) *v1.RestoreProg PercentDone: p.PercentDone, } } + +func RepoStatsToProto(s *restic.RepoStats) *v1.RepoStats { + return &v1.RepoStats{ + TotalSize: int64(s.TotalSize), + TotalUncompressedSize: int64(s.TotalUncompressedSize), + CompressionRatio: s.CompressionRatio, + TotalBlobCount: int64(s.TotalBlobCount), + SnapshotCount: int64(s.SnapshotsCount), + } +} diff --git a/internal/protoutil/conversion_test.go b/internal/protoutil/conversion_test.go index 95ec7d06..7adb05b8 100644 --- a/internal/protoutil/conversion_test.go +++ b/internal/protoutil/conversion_test.go @@ -116,3 +116,45 @@ func TestBackupProgressEntryToProto(t *testing.T) { }) } } + +func TestRepoStatsToProto(t *testing.T) { + cases := []struct { + name string + stats *restic.RepoStats + want *v1.RepoStats + }{ + { + name: "no stats", + stats: &restic.RepoStats{}, + want: &v1.RepoStats{}, + }, + { + name: "with stats", + stats: &restic.RepoStats{ + TotalSize: 1, + TotalUncompressedSize: 2, + CompressionRatio: 3, + TotalBlobCount: 5, + CompressionProgress: 6, + CompressionSpaceSaving: 7, + SnapshotsCount: 8, + }, + want: &v1.RepoStats{ + TotalSize: 1, + TotalUncompressedSize: 2, + CompressionRatio: 3, + TotalBlobCount: 5, + SnapshotCount: 8, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := RepoStatsToProto(c.stats) + if !proto.Equal(got, c.want) { + t.Errorf("wanted: %+v, got: %+v", c.want, got) + } + }) + } +} diff --git a/pkg/restic/outputs.go b/pkg/restic/outputs.go index cb7ea3e9..dade304d 100644 --- a/pkg/restic/outputs.go +++ b/pkg/restic/outputs.go @@ -304,3 +304,13 @@ func ValidateSnapshotId(id string) error { } return nil } + +type RepoStats struct { + TotalSize int64 `json:"total_size"` + TotalUncompressedSize int64 `json:"total_uncompressed_size"` + CompressionRatio float64 `json:"compression_ratio"` + CompressionProgress int64 `json:"compression_progress"` + CompressionSpaceSaving float64 `json:"compression_space_saving"` + TotalBlobCount int64 `json:"total_blob_count"` + SnapshotsCount int64 `json:"snapshots_count"` +} diff --git a/pkg/restic/restic.go b/pkg/restic/restic.go index 1e4501c4..6c02b771 100644 --- a/pkg/restic/restic.go +++ b/pkg/restic/restic.go @@ -347,6 +347,30 @@ func (r *Repo) Unlock(ctx context.Context, opts ...GenericOption) error { return nil } +func (r *Repo) Stats(ctx context.Context, opts ...GenericOption) (*RepoStats, error) { + opt := resolveOpts(opts) + + args := []string{"stats", "--json", "--mode=raw-data"} + args = append(args, r.extraArgs...) + args = append(args, opt.extraArgs...) + + cmd := exec.CommandContext(ctx, r.cmd, args...) + cmd.Env = append(cmd.Env, r.buildEnv()...) + cmd.Env = append(cmd.Env, opt.extraEnv...) + + output, err := cmd.CombinedOutput() + if err != nil { + return nil, newCmdError(cmd, string(output), err) + } + + var stats RepoStats + if err := json.Unmarshal(output, &stats); err != nil { + return nil, newCmdError(cmd, string(output), fmt.Errorf("command output is not valid JSON: %w", err)) + } + + return &stats, nil +} + type RetentionPolicy struct { KeepLastN int // keep the last n snapshots. KeepHourly int // keep the last n hourly snapshots. diff --git a/pkg/restic/restic_test.go b/pkg/restic/restic_test.go index c9d3dc59..d619f221 100644 --- a/pkg/restic/restic_test.go +++ b/pkg/restic/restic_test.go @@ -362,3 +362,42 @@ func TestResticRestore(t *testing.T) { t.Errorf("wanted 101 files to be restored, got: %d", summary.TotalFiles) } } + +func TestResticStats(t *testing.T) { + t.Parallel() + + repo := t.TempDir() + r := NewRepo(helpers.ResticBinary(t), &v1.Repo{ + Id: "test", + Uri: repo, + Password: "test", + }, WithFlags("--no-cache")) + if err := r.Init(context.Background()); err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + testData := helpers.CreateTestData(t) + + _, err := r.Backup(context.Background(), nil, WithBackupPaths(testData)) + if err != nil { + t.Fatalf("failed to backup and create new snapshot: %v", err) + } + + // restore all files + stats, err := r.Stats(context.Background()) + if err != nil { + t.Fatalf("failed to get stats: %v", err) + } + if stats.SnapshotsCount != 1 { + t.Errorf("wanted 1 snapshot, got: %d", stats.SnapshotsCount) + } + if stats.TotalSize == 0 { + t.Errorf("wanted non-zero total size, got: %d", stats.TotalSize) + } + if stats.TotalUncompressedSize == 0 { + t.Errorf("wanted non-zero total uncompressed size, got: %d", stats.TotalUncompressedSize) + } + if stats.TotalBlobCount == 0 { + t.Errorf("wanted non-zero total blob count, got: %d", stats.TotalBlobCount) + } +} diff --git a/proto/v1/restic.proto b/proto/v1/restic.proto index d949159b..182e8db4 100644 --- a/proto/v1/restic.proto +++ b/proto/v1/restic.proto @@ -65,4 +65,12 @@ message RestoreProgressEntry { int64 total_files = 5; int64 files_restored = 6; double percent_done = 7; // 0.0 - 1.0 +} + +message RepoStats { + int64 total_size = 1; + int64 total_uncompressed_size = 2; + double compression_ratio = 3; + int64 total_blob_count = 5; + int64 snapshot_count = 6; } \ No newline at end of file diff --git a/webui/gen/ts/v1/restic_pb.ts b/webui/gen/ts/v1/restic_pb.ts index 5b565549..bbdd09fd 100644 --- a/webui/gen/ts/v1/restic_pb.ts +++ b/webui/gen/ts/v1/restic_pb.ts @@ -443,3 +443,64 @@ export class RestoreProgressEntry extends Message { } } +/** + * @generated from message v1.RepoStats + */ +export class RepoStats extends Message { + /** + * @generated from field: int64 total_size = 1; + */ + totalSize = protoInt64.zero; + + /** + * @generated from field: int64 total_uncompressed_size = 2; + */ + totalUncompressedSize = protoInt64.zero; + + /** + * @generated from field: double compression_ratio = 3; + */ + compressionRatio = 0; + + /** + * @generated from field: int64 total_blob_count = 5; + */ + totalBlobCount = protoInt64.zero; + + /** + * @generated from field: int64 snapshot_count = 6; + */ + snapshotCount = protoInt64.zero; + + constructor(data?: PartialMessage) { + super(); + proto3.util.initPartial(data, this); + } + + static readonly runtime: typeof proto3 = proto3; + static readonly typeName = "v1.RepoStats"; + static readonly fields: FieldList = proto3.util.newFieldList(() => [ + { no: 1, name: "total_size", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 2, name: "total_uncompressed_size", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 3, name: "compression_ratio", kind: "scalar", T: 1 /* ScalarType.DOUBLE */ }, + { no: 5, name: "total_blob_count", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 6, name: "snapshot_count", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + ]); + + static fromBinary(bytes: Uint8Array, options?: Partial): RepoStats { + return new RepoStats().fromBinary(bytes, options); + } + + static fromJson(jsonValue: JsonValue, options?: Partial): RepoStats { + return new RepoStats().fromJson(jsonValue, options); + } + + static fromJsonString(jsonString: string, options?: Partial): RepoStats { + return new RepoStats().fromJsonString(jsonString, options); + } + + static equals(a: RepoStats | PlainMessage | undefined, b: RepoStats | PlainMessage | undefined): boolean { + return proto3.util.equals(RepoStats, a, b); + } +} +