Skip to content

Commit

Permalink
definitions: Fix statistical's result is incorrect (#367)
Browse files Browse the repository at this point in the history
Signed-off-by: Xuanwo <github@xuanwo.io>
  • Loading branch information
Xuanwo authored May 27, 2020
1 parent 4ba9a02 commit 4370d2c
Show file tree
Hide file tree
Showing 5 changed files with 230 additions and 71 deletions.
2 changes: 1 addition & 1 deletion definitions/operations.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ interface "statistician" {

op "statistical" {
description = "will count service's statistics, such as Size, Count."
results = ["url"]
results = ["statistic"]
}
}
interface "storager" {
Expand Down
2 changes: 1 addition & 1 deletion definitions/services/qingstor.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ namespace "service" {
}
}
namespace "storage" {
implement = ["copier", "dir_lister", "index_segmenter", "mover", "prefix_lister", "prefix_segments_lister", "reacher"]
implement = ["copier", "dir_lister", "index_segmenter", "mover", "prefix_lister", "prefix_segments_lister", "reacher", "segmenter", "statistician"]

new {
required = ["name"]
Expand Down
4 changes: 2 additions & 2 deletions generated.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,9 @@ type Servicer interface {
type Statistician interface {

// Statistical will count service's statistics, such as Size, Count.
Statistical(pairs ...*types.Pair) (url string, err error)
Statistical(pairs ...*types.Pair) (statistic info.StorageStatistic, err error)
// StatisticalWithContext will count service's statistics, such as Size, Count.
StatisticalWithContext(ctx context.Context, pairs ...*types.Pair) (url string, err error)
StatisticalWithContext(ctx context.Context, pairs ...*types.Pair) (statistic info.StorageStatistic, err error)
}

// Storager is the interface for storage service.
Expand Down
177 changes: 177 additions & 0 deletions services/qingstor/generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

116 changes: 49 additions & 67 deletions services/qingstor/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,77 +10,10 @@ import (
"github.com/Xuanwo/storage/pkg/headers"
"github.com/Xuanwo/storage/pkg/iowrap"
"github.com/Xuanwo/storage/pkg/segment"
"github.com/Xuanwo/storage/services"
"github.com/Xuanwo/storage/types"
"github.com/Xuanwo/storage/types/info"
)

// Statistical implements Storager.Statistical
func (s *Storage) Statistical(pairs ...*types.Pair) (m info.StorageStatistic, err error) {
defer func() {
err = s.formatError(services.OpStatistical, err)
}()

m = info.NewStorageStatistic()

output, err := s.bucket.GetStatistics()
if err != nil {
return
}

if output.Size != nil {
m.SetSize(*output.Size)
}
if output.Count != nil {
m.SetCount(*output.Count)
}
return m, nil
}

// CompleteSegment implements Storager.CompleteSegment
func (s *Storage) CompleteSegment(seg segment.Segment, pairs ...*types.Pair) (err error) {
defer func() {
err = s.formatError(services.OpCompleteSegment, err, seg.Path(), seg.ID())
}()

parts := seg.(*segment.IndexBasedSegment).Parts()
objectParts := make([]*service.ObjectPartType, 0, len(parts))
for _, v := range parts {
objectParts = append(objectParts, &service.ObjectPartType{
PartNumber: service.Int(v.Index),
Size: service.Int64(v.Size),
})
}

rp := s.getAbsPath(seg.Path())

_, err = s.bucket.CompleteMultipartUpload(rp, &service.CompleteMultipartUploadInput{
UploadID: service.String(seg.ID()),
ObjectParts: objectParts,
})
if err != nil {
return
}
return
}

// AbortSegment implements Storager.AbortSegment
func (s *Storage) AbortSegment(seg segment.Segment, pairs ...*types.Pair) (err error) {
defer func() {
err = s.formatError(services.OpAbortSegment, err, seg.Path(), seg.ID())
}()

rp := s.getAbsPath(seg.Path())

_, err = s.bucket.AbortMultipartUpload(rp, &service.AbortMultipartUploadInput{
UploadID: service.String(seg.ID()),
})
if err != nil {
return
}
return
}

func (s *Storage) delete(ctx context.Context, path string, opt *pairStorageDelete) (err error) {
rp := s.getAbsPath(path)

Expand Down Expand Up @@ -394,3 +327,52 @@ func (s *Storage) move(ctx context.Context, src string, dst string, opt *pairSto
}
return nil
}

func (s *Storage) abortSegment(ctx context.Context, seg segment.Segment, opt *pairStorageAbortSegment) (err error) {
rp := s.getAbsPath(seg.Path())

_, err = s.bucket.AbortMultipartUpload(rp, &service.AbortMultipartUploadInput{
UploadID: service.String(seg.ID()),
})
if err != nil {
return
}
return
}
func (s *Storage) completeSegment(ctx context.Context, seg segment.Segment, opt *pairStorageCompleteSegment) (err error) {
parts := seg.(*segment.IndexBasedSegment).Parts()
objectParts := make([]*service.ObjectPartType, 0, len(parts))
for _, v := range parts {
objectParts = append(objectParts, &service.ObjectPartType{
PartNumber: service.Int(v.Index),
Size: service.Int64(v.Size),
})
}

rp := s.getAbsPath(seg.Path())

_, err = s.bucket.CompleteMultipartUpload(rp, &service.CompleteMultipartUploadInput{
UploadID: service.String(seg.ID()),
ObjectParts: objectParts,
})
if err != nil {
return
}
return
}
func (s *Storage) statistical(ctx context.Context, opt *pairStorageStatistical) (statistic info.StorageStatistic, err error) {
statistic = info.NewStorageStatistic()

output, err := s.bucket.GetStatistics()
if err != nil {
return
}

if output.Size != nil {
statistic.SetSize(*output.Size)
}
if output.Count != nil {
statistic.SetCount(*output.Count)
}
return statistic, nil
}

1 comment on commit 4370d2c

@vercel
Copy link

@vercel vercel bot commented on 4370d2c May 27, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.