From 0a090e16d947f3b6ff26aebeeeac0c7ff8358366 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 30 Jun 2022 10:58:41 +0200 Subject: [PATCH] use checksum to calculate etag Signed-off-by: jkoberg --- pkg/storage/utils/decomposedfs/node/node.go | 50 ++++++++++++------- pkg/storage/utils/decomposedfs/revisions.go | 2 +- pkg/storage/utils/decomposedfs/spaces.go | 5 +- .../decomposedfs/upload/postprocessing.go | 6 --- .../utils/decomposedfs/upload/upload.go | 2 +- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index be8b5f894e..3a788cb619 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -472,12 +472,27 @@ func (n *Node) LockFilePath() string { } // CalculateEtag returns a hash of fileid + tmtime (or mtime) -func CalculateEtag(nodeID string, tmTime time.Time) (string, error) { - return calculateEtag(nodeID, tmTime) +func (n *Node) CalculateEtag() (string, error) { + tmTime, err := n.GetTMTime() + if err != nil { + // no tmtime, use mtime + var fi os.FileInfo + if fi, err = os.Lstat(n.InternalPath()); err != nil { + return "", err + } + tmTime = fi.ModTime() + } + + var checksum string + if !n.IsProcessing() { + checksum, _ = n.GetChecksum("sha1") + } + + return calculateEtag(n.ID, tmTime, checksum) } // calculateEtag returns a hash of fileid + tmtime (or mtime) -func calculateEtag(nodeID string, tmTime time.Time) (string, error) { +func calculateEtag(nodeID string, tmTime time.Time, checksum string) (string, error) { h := md5.New() if _, err := io.WriteString(h, nodeID); err != nil { return "", err @@ -489,6 +504,9 @@ func calculateEtag(nodeID string, tmTime time.Time) (string, error) { } else { return "", err } + if _, err := h.Write([]byte(checksum)); err != nil { + return "", err + } return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil } @@ -517,17 +535,9 @@ func (n *Node) SetMtime(ctx context.Context, mtime string) error { func (n *Node) SetEtag(ctx context.Context, val string) (err error) { sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() nodePath := n.InternalPath() - var tmTime time.Time - if tmTime, err = n.GetTMTime(); err != nil { - // no tmtime, use mtime - var fi os.FileInfo - if fi, err = os.Lstat(nodePath); err != nil { - return - } - tmTime = fi.ModTime() - } + var etag string - if etag, err = calculateEtag(n.ID, tmTime); err != nil { + if etag, err = n.CalculateEtag(); err != nil { return } @@ -655,8 +665,6 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } } - // TODO make etag of files use fileid and checksum - var tmTime time.Time if tmTime, err = n.GetTMTime(); err != nil { // no tmtime, use mtime @@ -666,8 +674,11 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // use temporary etag if it is set if b, err := xattrs.Get(nodePath, xattrs.TmpEtagAttr); err == nil { ri.Etag = fmt.Sprintf(`"%x"`, b) // TODO why do we convert string(b)? is the temporary etag stored as string? -> should we use bytes? use hex.EncodeToString? - } else if ri.Etag, err = calculateEtag(n.ID, tmTime); err != nil { - sublog.Debug().Err(err).Msg("could not calculate etag") + } else { + ri.Etag, err = n.CalculateEtag() + if err != nil { + sublog.Debug().Err(err).Msg("could not calculate etag") + } } // mtime uses tmtime if present @@ -927,6 +938,11 @@ func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { return n.SetMetadata(xattrs.ChecksumPrefix+csType, string(h.Sum(nil))) } +// GetChecksum writes the checksum with the given checksum type to the extended attributes +func (n *Node) GetChecksum(csType string) (string, error) { + return n.GetMetadata(xattrs.ChecksumPrefix + csType) +} + // UnsetTempEtag removes the temporary etag attribute func (n *Node) UnsetTempEtag() (err error) { err = xattrs.Remove(n.InternalPath(), xattrs.TmpEtagAttr) diff --git a/pkg/storage/utils/decomposedfs/revisions.go b/pkg/storage/utils/decomposedfs/revisions.go index 37cd90fa56..e7e23480eb 100644 --- a/pkg/storage/utils/decomposedfs/revisions.go +++ b/pkg/storage/utils/decomposedfs/revisions.go @@ -83,7 +83,7 @@ func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Referen return nil, errors.Wrapf(err, "error reading blobsize xattr") } rev.Size = uint64(blobSize) - etag, err := node.CalculateEtag(np, mtime) + etag, err := n.CalculateEtag() if err != nil { return nil, errors.Wrapf(err, "error calculating etag") } diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 52a9e4a06c..f26bb3a0bd 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -674,9 +674,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, // we set the space mtime to the root item mtime // override the stat mtime with a tmtime if it is present - var tmtime time.Time if tmt, err := n.GetTMTime(); err == nil { - tmtime = tmt un := tmt.UnixNano() space.Mtime = &types.Timestamp{ Seconds: uint64(un / 1000000000), @@ -684,7 +682,6 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, } } else if fi, err := os.Stat(nodePath); err == nil { // fall back to stat mtime - tmtime = fi.ModTime() un := fi.ModTime().UnixNano() space.Mtime = &types.Timestamp{ Seconds: uint64(un / 1000000000), @@ -692,7 +689,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, } } - etag, err := node.CalculateEtag(n.ID, tmtime) + etag, err := n.CalculateEtag() if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/upload/postprocessing.go b/pkg/storage/utils/decomposedfs/upload/postprocessing.go index 2cdd5c5882..90cb559d7e 100644 --- a/pkg/storage/utils/decomposedfs/upload/postprocessing.go +++ b/pkg/storage/utils/decomposedfs/upload/postprocessing.go @@ -19,11 +19,9 @@ package upload import ( - "fmt" "time" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/cs3org/reva/v2/pkg/utils" "github.com/cs3org/reva/v2/pkg/utils/postprocessing" ) @@ -70,10 +68,6 @@ func configurePostprocessing(upload *Upload, o options.PostprocessingOptions) po if err := upload.node.UnmarkProcessing(); err != nil { upload.log.Info().Str("path", upload.node.InternalPath()).Err(err).Msg("unmarking processing failed") } - now := utils.TSNow() - if err := upload.node.SetMtime(upload.Ctx, fmt.Sprintf("%d.%d", now.Seconds, now.Nanos)); err != nil { - upload.log.Info().Str("path", upload.node.InternalPath()).Err(err).Msg("could not set mtime") - } if err := upload.tp.Propagate(upload.Ctx, upload.node); err != nil { upload.log.Info().Str("path", upload.node.InternalPath()).Err(err).Msg("could not set mtime") diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index 857e08b58d..72a8f5f78a 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -259,7 +259,7 @@ func (upload *Upload) finishUpload() (err error) { // etag still matches before finishing the upload. if ifMatch, ok := upload.Info.MetaData["if-match"]; ok { var targetEtag string - targetEtag, err = node.CalculateEtag(n.ID, fi.ModTime()) + targetEtag, err = n.CalculateEtag() if err != nil { return errtypes.InternalError(err.Error()) }