From 089c5951ae88f20466d29e0ee109ee4cc92025b3 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 27 Apr 2020 12:33:15 +0800 Subject: [PATCH] services, design: Propose and implement 21-split-segmenter (#270) Signed-off-by: Xuanwo --- docs/design/21-split-segmenter.md | 72 ++++++++++++++++++++++++++++++ segmenter.go | 68 ++++++++++++++++++++++++++++ services/error.go | 4 +- services/qingstor/generated.go | 54 ++++++++-------------- services/qingstor/meta.json | 4 -- services/qingstor/storager.go | 20 ++++----- services/qingstor/storager_test.go | 4 +- services/s3/generated.go | 54 ++++++++-------------- services/s3/meta.json | 4 -- services/s3/storager.go | 20 ++++----- storager.go | 63 -------------------------- 11 files changed, 200 insertions(+), 167 deletions(-) create mode 100644 docs/design/21-split-segmenter.md create mode 100644 segmenter.go diff --git a/docs/design/21-split-segmenter.md b/docs/design/21-split-segmenter.md new file mode 100644 index 000000000..9c808bb3d --- /dev/null +++ b/docs/design/21-split-segmenter.md @@ -0,0 +1,72 @@ +--- +author: Xuanwo +status: candidate +updated_at: 2020-04-27 +--- + +# Proposal: Split Segmenter + +## Background + +In proposal [18-return-segment-interface-instead], we introduced `segment.Segmenter` interface, but also introduce a new problem: user don't know which type of segment that service supported. + +In the `Rationale` section of [18-return-segment-interface-instead], we have already discussed two implementations: + +1. `OffsetBasedSegmenter` vs `IndexBasedSegmenter` +2. `WriteSegmentViaIndex` vs `WriteSegmentViaOffset` + +We said: + +> No.1 implementation's problem is these add too much work for services to support both of them. + +But maybe this is the way, we can build new segmenter upon this idea. + +## Proposal + +Based on the idea of `OffsetBasedSegmenter` and `IndexBasedSegmenter`, I propose following changes: + +Add new interface `IndexSegmenter` that implement following functions: + +- `InitIndexSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error)` +- `WriteIndexSegment(seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error)` +- `CompleteSegment(seg segment.Segment, pairs ...*types.Pair) (err error)` +- `AbortSegment(seg segment.Segment, pairs ...*types.Pair) (err error)` + +Extract `CompleteSegment` and `AbortSegment` into a non-exported interface `segmenter`. + +Embed `segmenter` into `IndexSegmenter`, `DirSegmentsLister` and `PrefixSegmentsLister` + +Remove `Segmenter` + +### Add new interface `IndexSegmenter` + +It's easy to find out that we don't need to implement an entire new segmenter interface. After introducing `segment.Segmenter` interface, we can reuse already implemented `CompleteSegment` and `AbortSegment`. + +After this change, the workload for service implementer reduced as the same. If new segment method needs to implemented, we only need to add `OffsetSegmenter`. + +### Extract non-exported interface and embed into others + +We already added `DirSegmentsLister` and `PrefixSegmentsLister`, but list only interface is not useful, we always need to convert to `Segmenter`. After extracting `CompleteSegment` and `AbortSegment` into a non-exported interface, we can make `DirSegmentsLister` and `PrefixSegmentsLister` more useful. + +### Remove `Segmenter` + +Yes, I know, I introduced an API breaking changes here. + +I'm sorry, but I don't have the time and energy to maintain both v1 and v2 for now. Since [storage] is nearly no one except me used, I think it's safe to introduce this change here and not break others. + +I'm aware the risk, and I will be responsible for this. + +## Rationale + +None. + +## Compatibility + +All API call related `Segmenter` will be borken. + +## Implementation + +Most of the work would be done by the author of this proposal. + +[18-return-segment-interface-instead]: (./18-return-segment-interface-instead.md) +[storage]: https://github.com/Xuanwo/storage \ No newline at end of file diff --git a/segmenter.go b/segmenter.go new file mode 100644 index 000000000..4ead8996c --- /dev/null +++ b/segmenter.go @@ -0,0 +1,68 @@ +package storage + +import ( + "context" + "io" + + "github.com/Xuanwo/storage/pkg/segment" + "github.com/Xuanwo/storage/types" +) + +type segmenter interface { + // CompleteSegment will complete a segment and merge them into a File. + // + // Implementer: + // - SHOULD return error while caller call CompleteSegment without init. + // Caller: + // - SHOULD call InitIndexSegment before CompleteSegment. + CompleteSegment(seg segment.Segment, pairs ...*types.Pair) (err error) + // CompleteSegmentWithContext will complete a segment and merge them into a File. + CompleteSegmentWithContext(ctx context.Context, seg segment.Segment, pairs ...*types.Pair) (err error) + // AbortSegment will abort a segment. + // + // Implementer: + // - SHOULD return error while caller call AbortSegment without init. + // Caller: + // - SHOULD call InitIndexSegment before AbortSegment. + AbortSegment(seg segment.Segment, pairs ...*types.Pair) (err error) + // AbortSegmentWithContext will abort a segment. + AbortSegmentWithContext(ctx context.Context, seg segment.Segment, pairs ...*types.Pair) (err error) +} + +// DirSegmentsLister is used for directory based storage service to list segments under a dir. +type DirSegmentsLister interface { + segmenter + + // ListDirSegments will list segments via dir. + ListDirSegments(path string, pairs ...*types.Pair) (err error) + // ListDirSegmentsWithContext will list segments via dir. + ListDirSegmentsWithContext(ctx context.Context, path string, pairs ...*types.Pair) (err error) +} + +// PrefixSegmentsLister is used for prefix based storage service to list segments under a prefix. +type PrefixSegmentsLister interface { + segmenter + + // ListSegments will list segments. + // + // Implementer: + // - If prefix == "", services should return all segments. + ListPrefixSegments(prefix string, pairs ...*types.Pair) (err error) + // ListSegmentsWithContext will list segments. + ListPrefixSegmentsWithContext(ctx context.Context, prefix string, pairs ...*types.Pair) (err error) +} + +// IndexSegmenter is the interface for index based segment. +type IndexSegmenter interface { + segmenter + + // InitIndexSegment will init an index based segment + InitIndexSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) + // InitIndexSegmentWithContext will init an index based segment + InitIndexSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) + + // WriteIndexSegment will write a part into an index based segment. + WriteIndexSegment(seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) + // WriteIndexSegmentWithContext will write a part into an index based segment. + WriteIndexSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) +} diff --git a/services/error.go b/services/error.go index 86fa235f3..cb32ee219 100644 --- a/services/error.go +++ b/services/error.go @@ -47,8 +47,8 @@ const ( // Segment related op OpListPrefixSegments = "list_prefix_segments" - OpInitSegment = "init_segment" - OpWriteSegment = "write_segment" + OpInitIndexSegment = "init_index_segment" + OpWriteIndexSegment = "write_index_segment" OpCompleteSegment = "complete_segment" OpAbortSegment = "abort_segment" ) diff --git a/services/qingstor/generated.go b/services/qingstor/generated.go index b053171c8..e41b766ba 100644 --- a/services/qingstor/generated.go +++ b/services/qingstor/generated.go @@ -558,25 +558,25 @@ func (s *Storage) parsePairDelete(opts ...*types.Pair) (*pairStorageDelete, erro return result, nil } -var pairStorageInitSegmentMap = map[string]struct{}{ +var pairStorageInitIndexSegmentMap = map[string]struct{}{ // Pre-defined pairs "context": struct{}{}, // Meta-defined pairs } -type pairStorageInitSegment struct { +type pairStorageInitIndexSegment struct { // Pre-defined pairs Context context.Context // Meta-defined pairs } -func (s *Storage) parsePairInitSegment(opts ...*types.Pair) (*pairStorageInitSegment, error) { - result := &pairStorageInitSegment{} +func (s *Storage) parsePairInitIndexSegment(opts ...*types.Pair) (*pairStorageInitIndexSegment, error) { + result := &pairStorageInitIndexSegment{} values := make(map[string]interface{}) for _, v := range opts { - if _, ok := pairStorageInitSegmentMap[v.Key]; !ok { + if _, ok := pairStorageInitIndexSegmentMap[v.Key]; !ok { return nil, services.NewPairUnsupportedError(v) } values[v.Key] = v.Value @@ -1138,32 +1138,28 @@ func (s *Storage) parsePairWrite(opts ...*types.Pair) (*pairStorageWrite, error) return result, nil } -var pairStorageWriteSegmentMap = map[string]struct{}{ +var pairStorageWriteIndexSegmentMap = map[string]struct{}{ // Pre-defined pairs "context": struct{}{}, // Meta-defined pairs - "index": struct{}{}, "read_callback_func": struct{}{}, - "size": struct{}{}, } -type pairStorageWriteSegment struct { +type pairStorageWriteIndexSegment struct { // Pre-defined pairs Context context.Context // Meta-defined pairs - Index int HasReadCallbackFunc bool ReadCallbackFunc func([]byte) - Size int64 } -func (s *Storage) parsePairWriteSegment(opts ...*types.Pair) (*pairStorageWriteSegment, error) { - result := &pairStorageWriteSegment{} +func (s *Storage) parsePairWriteIndexSegment(opts ...*types.Pair) (*pairStorageWriteIndexSegment, error) { + result := &pairStorageWriteIndexSegment{} values := make(map[string]interface{}) for _, v := range opts { - if _, ok := pairStorageWriteSegmentMap[v.Key]; !ok { + if _, ok := pairStorageWriteIndexSegmentMap[v.Key]; !ok { return nil, services.NewPairUnsupportedError(v) } values[v.Key] = v.Value @@ -1181,25 +1177,11 @@ func (s *Storage) parsePairWriteSegment(opts ...*types.Pair) (*pairStorageWriteS } // Parse meta-defined pairs - v, ok = values[ps.Index] - if !ok { - return nil, services.NewPairRequiredError(ps.Index) - } - if ok { - result.Index = v.(int) - } v, ok = values[ps.ReadCallbackFunc] if ok { result.HasReadCallbackFunc = true result.ReadCallbackFunc = v.(func([]byte)) } - v, ok = values[ps.Size] - if !ok { - return nil, services.NewPairRequiredError(ps.Size) - } - if ok { - result.Size = v.(int64) - } return result, nil } @@ -1276,13 +1258,13 @@ func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...* return s.Delete(path, pairs...) } -// InitSegmentWithContext adds context support for InitSegment. -func (s *Storage) InitSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/qingstor.storage.InitSegment") +// InitIndexSegmentWithContext adds context support for InitIndexSegment. +func (s *Storage) InitIndexSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/qingstor.storage.InitIndexSegment") defer span.Finish() pairs = append(pairs, ps.WithContext(ctx)) - return s.InitSegment(path, pairs...) + return s.InitIndexSegment(path, pairs...) } // ListDirWithContext adds context support for ListDir. @@ -1375,11 +1357,11 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.Write(path, r, pairs...) } -// WriteSegmentWithContext adds context support for WriteSegment. -func (s *Storage) WriteSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/qingstor.storage.WriteSegment") +// WriteIndexSegmentWithContext adds context support for WriteIndexSegment. +func (s *Storage) WriteIndexSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/qingstor.storage.WriteIndexSegment") defer span.Finish() pairs = append(pairs, ps.WithContext(ctx)) - return s.WriteSegment(seg, r, pairs...) + return s.WriteIndexSegment(seg, r, index, size, pairs...) } diff --git a/services/qingstor/meta.json b/services/qingstor/meta.json index 3adae7503..55553add6 100644 --- a/services/qingstor/meta.json +++ b/services/qingstor/meta.json @@ -42,10 +42,6 @@ "checksum": false, "size": true, "storage_class": false - }, - "write_segment": { - "index": true, - "size": true } } } \ No newline at end of file diff --git a/services/qingstor/storager.go b/services/qingstor/storager.go index 7054d153f..8092ce95d 100644 --- a/services/qingstor/storager.go +++ b/services/qingstor/storager.go @@ -412,13 +412,13 @@ func (s *Storage) ListPrefixSegments(prefix string, pairs ...*types.Pair) (err e return } -// InitSegment implements Storager.InitSegment -func (s *Storage) InitSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) { +// InitIndexSegment implements Storager.InitIndexSegment +func (s *Storage) InitIndexSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) { defer func() { - err = s.formatError(services.OpInitSegment, err, path) + err = s.formatError(services.OpInitIndexSegment, err, path) }() - _, err = s.parsePairInitSegment(pairs...) + _, err = s.parsePairInitIndexSegment(pairs...) if err != nil { return } @@ -438,18 +438,18 @@ func (s *Storage) InitSegment(path string, pairs ...*types.Pair) (seg segment.Se return seg, nil } -// WriteSegment implements Storager.WriteSegment -func (s *Storage) WriteSegment(seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) { +// WriteIndexSegment implements Storager.WriteIndexSegment +func (s *Storage) WriteIndexSegment(seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) { defer func() { - err = s.formatError(services.OpWriteSegment, err, seg.Path(), seg.ID()) + err = s.formatError(services.OpWriteIndexSegment, err, seg.Path(), seg.ID()) }() - opt, err := s.parsePairWriteSegment(pairs...) + opt, err := s.parsePairWriteIndexSegment(pairs...) if err != nil { return } - p, err := seg.(*segment.IndexBasedSegment).InsertPart(opt.Index, opt.Size) + p, err := seg.(*segment.IndexBasedSegment).InsertPart(index, size) if err != nil { return } @@ -463,7 +463,7 @@ func (s *Storage) WriteSegment(seg segment.Segment, r io.Reader, pairs ...*types _, err = s.bucket.UploadMultipart(rp, &service.UploadMultipartInput{ PartNumber: service.Int(p.Index), UploadID: service.String(seg.ID()), - ContentLength: &opt.Size, + ContentLength: &size, Body: r, }) if err != nil { diff --git a/services/qingstor/storager_test.go b/services/qingstor/storager_test.go index 202405cac..d498d4eaa 100644 --- a/services/qingstor/storager_test.go +++ b/services/qingstor/storager_test.go @@ -288,7 +288,7 @@ func TestStorage_InitSegment(t *testing.T) { bucket: mockBucket, } - _, err := client.InitSegment(v.path) + _, err := client.InitIndexSegment(v.path) if v.hasError { assert.Error(t, err) assert.True(t, errors.Is(err, v.wantErr)) @@ -673,7 +673,7 @@ func TestStorage_WriteSegment(t *testing.T) { assert.Equal(t, id, *input.UploadID) }) - err := client.WriteSegment(seg, nil, pairs.WithSize(100), pairs.WithIndex(0)) + err := client.WriteIndexSegment(seg, nil, 0, 100) assert.NoError(t, err) }) } diff --git a/services/s3/generated.go b/services/s3/generated.go index 44245f13f..a7e26a074 100644 --- a/services/s3/generated.go +++ b/services/s3/generated.go @@ -470,25 +470,25 @@ func (s *Storage) parsePairDelete(opts ...*types.Pair) (*pairStorageDelete, erro return result, nil } -var pairStorageInitSegmentMap = map[string]struct{}{ +var pairStorageInitIndexSegmentMap = map[string]struct{}{ // Pre-defined pairs "context": struct{}{}, // Meta-defined pairs } -type pairStorageInitSegment struct { +type pairStorageInitIndexSegment struct { // Pre-defined pairs Context context.Context // Meta-defined pairs } -func (s *Storage) parsePairInitSegment(opts ...*types.Pair) (*pairStorageInitSegment, error) { - result := &pairStorageInitSegment{} +func (s *Storage) parsePairInitIndexSegment(opts ...*types.Pair) (*pairStorageInitIndexSegment, error) { + result := &pairStorageInitIndexSegment{} values := make(map[string]interface{}) for _, v := range opts { - if _, ok := pairStorageInitSegmentMap[v.Key]; !ok { + if _, ok := pairStorageInitIndexSegmentMap[v.Key]; !ok { return nil, services.NewPairUnsupportedError(v) } values[v.Key] = v.Value @@ -912,32 +912,28 @@ func (s *Storage) parsePairWrite(opts ...*types.Pair) (*pairStorageWrite, error) return result, nil } -var pairStorageWriteSegmentMap = map[string]struct{}{ +var pairStorageWriteIndexSegmentMap = map[string]struct{}{ // Pre-defined pairs "context": struct{}{}, // Meta-defined pairs - "index": struct{}{}, "read_callback_func": struct{}{}, - "size": struct{}{}, } -type pairStorageWriteSegment struct { +type pairStorageWriteIndexSegment struct { // Pre-defined pairs Context context.Context // Meta-defined pairs - Index int HasReadCallbackFunc bool ReadCallbackFunc func([]byte) - Size int64 } -func (s *Storage) parsePairWriteSegment(opts ...*types.Pair) (*pairStorageWriteSegment, error) { - result := &pairStorageWriteSegment{} +func (s *Storage) parsePairWriteIndexSegment(opts ...*types.Pair) (*pairStorageWriteIndexSegment, error) { + result := &pairStorageWriteIndexSegment{} values := make(map[string]interface{}) for _, v := range opts { - if _, ok := pairStorageWriteSegmentMap[v.Key]; !ok { + if _, ok := pairStorageWriteIndexSegmentMap[v.Key]; !ok { return nil, services.NewPairUnsupportedError(v) } values[v.Key] = v.Value @@ -955,25 +951,11 @@ func (s *Storage) parsePairWriteSegment(opts ...*types.Pair) (*pairStorageWriteS } // Parse meta-defined pairs - v, ok = values[ps.Index] - if !ok { - return nil, services.NewPairRequiredError(ps.Index) - } - if ok { - result.Index = v.(int) - } v, ok = values[ps.ReadCallbackFunc] if ok { result.HasReadCallbackFunc = true result.ReadCallbackFunc = v.(func([]byte)) } - v, ok = values[ps.Size] - if !ok { - return nil, services.NewPairRequiredError(ps.Size) - } - if ok { - result.Size = v.(int64) - } return result, nil } @@ -1041,13 +1023,13 @@ func (s *Storage) DeleteWithContext(ctx context.Context, path string, pairs ...* return s.Delete(path, pairs...) } -// InitSegmentWithContext adds context support for InitSegment. -func (s *Storage) InitSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/s3.storage.InitSegment") +// InitIndexSegmentWithContext adds context support for InitIndexSegment. +func (s *Storage) InitIndexSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/s3.storage.InitIndexSegment") defer span.Finish() pairs = append(pairs, ps.WithContext(ctx)) - return s.InitSegment(path, pairs...) + return s.InitIndexSegment(path, pairs...) } // ListDirWithContext adds context support for ListDir. @@ -1113,11 +1095,11 @@ func (s *Storage) WriteWithContext(ctx context.Context, path string, r io.Reader return s.Write(path, r, pairs...) } -// WriteSegmentWithContext adds context support for WriteSegment. -func (s *Storage) WriteSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/s3.storage.WriteSegment") +// WriteIndexSegmentWithContext adds context support for WriteIndexSegment. +func (s *Storage) WriteIndexSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "github.com/Xuanwo/storage/services/s3.storage.WriteIndexSegment") defer span.Finish() pairs = append(pairs, ps.WithContext(ctx)) - return s.WriteSegment(seg, r, pairs...) + return s.WriteIndexSegment(seg, r, index, size, pairs...) } diff --git a/services/s3/meta.json b/services/s3/meta.json index 10831e56c..7ebd419a3 100644 --- a/services/s3/meta.json +++ b/services/s3/meta.json @@ -37,10 +37,6 @@ "checksum": false, "size": true, "storage_class": false - }, - "write_segment": { - "index": true, - "size": true } } } \ No newline at end of file diff --git a/services/s3/storager.go b/services/s3/storager.go index 7fc8011f9..611f2221d 100644 --- a/services/s3/storager.go +++ b/services/s3/storager.go @@ -321,13 +321,13 @@ func (s *Storage) ListPrefixSegments(path string, pairs ...*types.Pair) (err err return } -// InitSegment implements Storager.InitSegment -func (s *Storage) InitSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) { +// InitIndexSegment implements Storager.InitIndexSegment +func (s *Storage) InitIndexSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) { defer func() { - err = s.formatError(services.OpInitSegment, err, path) + err = s.formatError(services.OpInitIndexSegment, err, path) }() - _, err = s.parsePairInitSegment(pairs...) + _, err = s.parsePairInitIndexSegment(pairs...) if err != nil { return } @@ -350,18 +350,18 @@ func (s *Storage) InitSegment(path string, pairs ...*types.Pair) (seg segment.Se return seg, nil } -// WriteSegment implements Storager.WriteSegment -func (s *Storage) WriteSegment(seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) { +// WriteIndexSegment implements Storager.WriteIndexSegment +func (s *Storage) WriteIndexSegment(seg segment.Segment, r io.Reader, index int, size int64, pairs ...*types.Pair) (err error) { defer func() { - err = s.formatError(services.OpWriteSegment, err, seg.Path(), seg.ID()) + err = s.formatError(services.OpWriteIndexSegment, err, seg.Path(), seg.ID()) }() - opt, err := s.parsePairWriteSegment(pairs...) + opt, err := s.parsePairWriteIndexSegment(pairs...) if err != nil { return } - p, err := seg.(*segment.IndexBasedSegment).InsertPart(opt.Index, opt.Size) + p, err := seg.(*segment.IndexBasedSegment).InsertPart(index, size) if err != nil { return } @@ -375,7 +375,7 @@ func (s *Storage) WriteSegment(seg segment.Segment, r io.Reader, pairs ...*types _, err = s.service.UploadPart(&s3.UploadPartInput{ Body: aws.ReadSeekCloser(r), Bucket: aws.String(s.name), - ContentLength: aws.Int64(opt.Size), + ContentLength: aws.Int64(size), Key: aws.String(rp), PartNumber: aws.Int64(int64(p.Index)), UploadId: aws.String(seg.ID()), diff --git a/storager.go b/storager.go index 626a1f64d..1c861ac1b 100644 --- a/storager.go +++ b/storager.go @@ -4,7 +4,6 @@ import ( "context" "io" - "github.com/Xuanwo/storage/pkg/segment" "github.com/Xuanwo/storage/types" "github.com/Xuanwo/storage/types/metadata" ) @@ -91,25 +90,6 @@ type PrefixLister interface { ListPrefixWithContext(ctx context.Context, prefix string, pairs ...*types.Pair) (err error) } -// DirSegmentsLister is used for directory based storage service to list segments under a dir. -type DirSegmentsLister interface { - // ListDirSegments will list segments via dir. - ListDirSegments(path string, pairs ...*types.Pair) (err error) - // ListDirSegmentsWithContext will list segments via dir. - ListDirSegmentsWithContext(ctx context.Context, path string, pairs ...*types.Pair) (err error) -} - -// PrefixSegmentsLister is used for prefix based storage service to list segments under a prefix. -type PrefixSegmentsLister interface { - // ListSegments will list segments. - // - // Implementer: - // - If prefix == "", services should return all segments. - ListPrefixSegments(prefix string, pairs ...*types.Pair) (err error) - // ListSegmentsWithContext will list segments. - ListPrefixSegmentsWithContext(ctx context.Context, prefix string, pairs ...*types.Pair) (err error) -} - // Copier is the interface for Copy. type Copier interface { // Copy will copy an Object or multiple object in the service. @@ -149,46 +129,3 @@ type Statistician interface { // StatisticalWithContext will count service's statistics, such as Size, Count. StatisticalWithContext(ctx context.Context, pairs ...*types.Pair) (metadata.StorageStatistic, error) } - -// Segmenter is the interface for Segment. -type Segmenter interface { - // Segment Operations. - - // InitSegment will init a segment which could be a File after complete. - // - // Implementer: - // - MUST maintain whole segment operation runtime data, including part number and any - // other similar things. - // Caller: - // - SHOULD call InitSegment before Write, Complete or Abort. - InitSegment(path string, pairs ...*types.Pair) (seg segment.Segment, err error) - // InitSegmentWithContext will init a segment which could be a File after complete. - InitSegmentWithContext(ctx context.Context, path string, pairs ...*types.Pair) (seg segment.Segment, err error) - // WriteSegment will read data into segment. - // - // Implementer: - // - SHOULD return error while caller call WriteSegment without init. - // Caller: - // - SHOULD call InitSegment before WriteSegment. - WriteSegment(seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) - // WriteSegmentWithContext will read data into segment. - WriteSegmentWithContext(ctx context.Context, seg segment.Segment, r io.Reader, pairs ...*types.Pair) (err error) - // CompleteSegment will complete a segment and merge them into a File. - // - // Implementer: - // - SHOULD return error while caller call CompleteSegment without init. - // Caller: - // - SHOULD call InitSegment before CompleteSegment. - CompleteSegment(seg segment.Segment, pairs ...*types.Pair) (err error) - // CompleteSegmentWithContext will complete a segment and merge them into a File. - CompleteSegmentWithContext(ctx context.Context, seg segment.Segment, pairs ...*types.Pair) (err error) - // AbortSegment will abort a segment. - // - // Implementer: - // - SHOULD return error while caller call AbortSegment without init. - // Caller: - // - SHOULD call InitSegment before AbortSegment. - AbortSegment(seg segment.Segment, pairs ...*types.Pair) (err error) - // AbortSegmentWithContext will abort a segment. - AbortSegmentWithContext(ctx context.Context, seg segment.Segment, pairs ...*types.Pair) (err error) -}