From 60e4c47416b216616ef936ea51f8ca87f29b9ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 3 Feb 2021 12:41:23 +0000 Subject: [PATCH] implement tree size propagation in ocis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../http/services/owncloud/ocdav/propfind.go | 93 +++++++---- .../ocs/handlers/cloud/users/users.go | 10 +- pkg/storage/fs/ocis/node.go | 152 ++++++++++++++++++ pkg/storage/fs/ocis/ocis.go | 54 ++++++- pkg/storage/fs/ocis/tree.go | 46 +++++- 5 files changed, 309 insertions(+), 46 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/propfind.go b/internal/http/services/owncloud/ocdav/propfind.go index c95c04b5bea..ce0fca5e936 100644 --- a/internal/http/services/owncloud/ocdav/propfind.go +++ b/internal/http/services/owncloud/ocdav/propfind.go @@ -93,22 +93,6 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string) return } - ref := &provider.Reference{ - Spec: &provider.Reference_Path{Path: fn}, - } - req := &provider.StatRequest{Ref: ref} - res, err := client.Stat(ctx, req) - if err != nil { - sublog.Error().Err(err).Msgf("error sending a grpc stat request to ref: %v", ref) - w.WriteHeader(http.StatusInternalServerError) - return - } - - if res.Status.Code != rpc.Code_CODE_OK { - HandleErrorStatus(&sublog, w, res.Status) - return - } - metadataKeys := []string{} if pf.Allprop != nil { // TODO this changes the behavior and returns all properties if allprops has been set, @@ -124,6 +108,24 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string) } } } + ref := &provider.Reference{ + Spec: &provider.Reference_Path{Path: fn}, + } + req := &provider.StatRequest{ + Ref: ref, + ArbitraryMetadataKeys: metadataKeys, + } + res, err := client.Stat(ctx, req) + if err != nil { + sublog.Error().Err(err).Interface("req", req).Msg("error sending a grpc stat request") + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res.Status.Code != rpc.Code_CODE_OK { + HandleErrorStatus(&sublog, w, res.Status) + return + } info := res.Info infos := []*provider.ResourceInfo{info} @@ -221,7 +223,14 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string) func requiresExplicitFetching(n *xml.Name) bool { switch n.Space { case _nsDav: - return false + switch n.Local { + case "quota-available-bytes", "quota-used-bytes": + // A PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes + // from https://www.rfc-editor.org/rfc/rfc4331.html#section-2 + return true + default: + return false + } case _nsOwncloud: switch n.Local { case "favorite", "share-types", "checksums": @@ -338,7 +347,13 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide } var ls *link.PublicShare - var treesize, quota string + + // -1 indicates uncalculated + // -2 indicates unknown (default) + // -3 indicates unlimited + treesize := _propQuotaUnknown + quota := _propQuotaUnknown + size := fmt.Sprintf("%d", md.Size) // TODO refactor helper functions: GetOpaqueJSONEncoded(opaque, key string, *struct) err, GetOpaquePlainEncoded(opaque, key) value, err // or use ok like pattern and return bool? if md.Opaque != nil && md.Opaque.Map != nil { @@ -349,15 +364,16 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide sublog.Error().Err(err).Msg("could not unmarshal link json") } } + // TODO the ResourceInfo should have a dedicated TreeSize property for collections. + // Currently, we are reusing the Size but that cannot indicate an unknown size because it is uint64 if md.Opaque.Map["treesize"] != nil && md.Opaque.Map["treesize"].Decoder == "plain" { treesize = string(md.Opaque.Map["treesize"].Value) - } else { - treesize = _propQuotaUnknown + if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + size = treesize + } } if md.Opaque.Map["quota"] != nil && md.Opaque.Map["quota"].Decoder == "plain" { quota = string(md.Opaque.Map["quota"].Value) - } else { - quota = _propQuotaUnknown } } @@ -411,14 +427,15 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide } // always return size, well nearly always ... public link shares are a little weird - size := fmt.Sprintf("%d", md.Size) if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { propstatOK.Prop = append(propstatOK.Prop, s.newPropRaw("d:resourcetype", "")) if ls == nil { propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size)) } - propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize)) - propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota)) + // A PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes + // from https://www.rfc-editor.org/rfc/rfc4331.html#section-2 + //propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize)) + //propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota)) } else { propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:resourcetype", ""), @@ -482,7 +499,6 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide // TODO return other properties ... but how do we put them in a namespace? } else { // otherwise return only the requested properties - size := fmt.Sprintf("%d", md.Size) for i := range pf.Prop { switch pf.Prop[i].Space { case _nsOwncloud: @@ -567,7 +583,8 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:public-link-expiration", "")) case "size": // phoenix only // TODO we cannot find out if md.Size is set or not because ints in go default to 0 - // oc:size is also available on folders + // TODO what is the difference to d:quota-used-bytes (which only exists for collections)? + // oc:size is available on files and folders and behaves like d:getcontentlength or d:quota-used-bytes respectively if ls == nil { propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size)) } else { @@ -718,17 +735,20 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:getlastmodified", "")) } - case "quota-used-bytes": // desktop client - // oc10 renders the tree size here + case "quota-used-bytes": // RFC 4331 if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + // TODO RFC says if treesize is unknown return it as not found, what does oc10 do? + // always returns the current usage, + // in oc10 there seems to be a bug that makes the size in webdav differ from the one in the user properties, not taking shares into account + // in ocis we plan to always mak the quota a property of the storage space propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize)) } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-used-bytes", "")) } - case "quota-available-bytes": // desktop client - // we can start with hardcoding unknown + case "quota-available-bytes": // RFC 4331 if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", _propQuotaUnknown)) + // oc10 returns -3 for unlimited, -2 for unknown, -1 for uncalculated + propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota)) } else { propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-available-bytes", "")) } @@ -805,7 +825,14 @@ func (c *countingReader) Read(p []byte) (int, error) { } func metadataKeyOf(n *xml.Name) string { - return fmt.Sprintf("%s/%s", n.Space, n.Local) + switch { + case n.Space == _nsDav && n.Local == "quota-used-bytes": + return "treesize" + case n.Space == _nsDav && n.Local == "quota-available-bytes": + return "quota" + default: + return fmt.Sprintf("%s/%s", n.Space, n.Local) + } } // http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind) diff --git a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go index 67c519420db..9922110415e 100644 --- a/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go +++ b/internal/http/services/owncloud/ocs/handlers/cloud/users/users.go @@ -59,7 +59,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } if user != u.Username { - // FIXME allow fetching other users info? + // FIXME allow fetching other users info? only for admins response.WriteOCSError(w, r, http.StatusForbidden, "user id mismatch", fmt.Errorf("%s tried to access %s user info endpoint", u.Id.OpaqueId, user)) return } @@ -147,9 +147,11 @@ func (h *Handler) handleUsers(w http.ResponseWriter, r *http.Request, u *userpb. response.WriteOCSSuccess(w, r, &Users{ // ocs can only return the home storage quota Quota: &Quota{ - Free: int64(getQuotaRes.TotalBytes - getQuotaRes.UsedBytes), - Used: int64(getQuotaRes.UsedBytes), - Total: int64(getQuotaRes.TotalBytes), // -1, -2 have special meaning? + Free: int64(getQuotaRes.TotalBytes - getQuotaRes.UsedBytes), + Used: int64(getQuotaRes.UsedBytes), + // TODO support negative values or flags for the quota to carry special meaning: -1 = uncalculated, -2 = unknown, -3 = unlimited + // for now we can only report total and used + Total: int64(getQuotaRes.TotalBytes), Relative: float32(float64(getQuotaRes.UsedBytes) / float64(getQuotaRes.TotalBytes)), Definition: "default", }, diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go index f79a07e0ed6..4ef7a1f9809 100644 --- a/pkg/storage/fs/ocis/node.go +++ b/pkg/storage/fs/ocis/node.go @@ -27,6 +27,7 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" "time" @@ -50,6 +51,12 @@ const ( _favoriteKey = "http://owncloud.org/ns/favorite" _checksumsKey = "http://owncloud.org/ns/checksums" + _treesizeKey = "treesize" + _quotaKey = "quota" + + _quotaUncalculated = "-1" + _quotaUnknown = "-2" + _quotaUnlimited = "-3" ) // Node represents a node in the tree and provides methods to get a Parent or Child instance @@ -547,6 +554,31 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi readChecksumIntoOpaque(ctx, nodePath, storageprovider.XSAdler32, ri) } + // treesize + if _, ok := mdKeysMap[_treesizeKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok { + readTreesizeIntoOpaque(ctx, nodePath, ri) + } + + // quota + if _, ok := mdKeysMap[_quotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok { + var quotaPath string + if n.lu.Options.EnableHome { + if r, err := n.lu.HomeNode(ctx); err == nil { + quotaPath = n.lu.toInternalPath(r.ID) + readQuotaIntoOpaque(ctx, quotaPath, ri) + } else { + sublog.Error().Err(err).Msg("error determining home node for quota") + } + } else { + if r, err := n.lu.RootNode(ctx); err == nil { + quotaPath = n.lu.toInternalPath(r.ID) + readQuotaIntoOpaque(ctx, quotaPath, ri) + } else { + sublog.Error().Err(err).Msg("error determining root node for quota") + } + } + } + // only read the requested metadata attributes attrs, err := xattr.List(nodePath) if err != nil { @@ -619,6 +651,112 @@ func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *prov appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("could not read checksum") } } +func readTreesizeIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) { + v, err := xattr.Get(nodePath, treesizeAttr) + switch { + case err == nil: + // make sure we have a proper unsigned int + if treesize, err := strconv.ParseUint(string(v), 10, 64); err == nil { + if ri.Opaque == nil { + ri.Opaque = &types.Opaque{ + Map: map[string]*types.OpaqueEntry{}, + } + } + ri.Opaque.Map[_treesizeKey] = &types.OpaqueEntry{ + Decoder: "plain", + Value: v, + } + // when it exists it also overrules the size + ri.Size = treesize + } else { + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("treesize", string(v)).Msg("malformed treesize") + } + case isNoData(err): + appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("treesize not set") + case isNotFound(err): + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("file not found when reading treesize") + default: + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read treesize") + } +} + +// quota is always stored on the root node +func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) { + v, err := xattr.Get(nodePath, quotaAttr) + switch { + case err == nil: + // make sure we have a proper signed int + // we use the same magic numbers to indicate: + // -1 = uncalculated + // -2 = unknown + // -3 = unlimited + if _, err := strconv.ParseInt(string(v), 10, 64); err == nil { + if ri.Opaque == nil { + ri.Opaque = &types.Opaque{ + Map: map[string]*types.OpaqueEntry{}, + } + } + ri.Opaque.Map[_quotaKey] = &types.OpaqueEntry{ + Decoder: "plain", + Value: v, + } + } else { + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("treesize", string(v)).Msg("malformed quota") + } + case isNoData(err): + appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("quota not set") + case isNotFound(err): + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("file not found when reading quota") + default: + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read quota") + } +} + +func (n *Node) CalculateTreeSize(ctx context.Context) (uint64, error) { + var size uint64 + // TODO check if this is a dir? + nodePath := n.lu.toInternalPath(n.ID) + + f, err := os.Open(nodePath) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not open dir") + return 0, err + } + defer f.Close() + + names, err := f.Readdirnames(0) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read dirnames") + return 0, err + } + for i := range names { + cPath := filepath.Join(nodePath, names[i]) + info, err := os.Stat(cPath) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not stat child entry") + continue // continue after an error + } + if !info.IsDir() { + size += uint64(info.Size()) + } else { + // read from attr + var b []byte + // xattr.Get will follow the symlink + if b, err = xattr.Get(cPath, treesizeAttr); err != nil { + // TODO recursively descend and recalculate treesize + continue // continue after an error + } + csize, err := strconv.ParseUint(string(b), 10, 64) + if err != nil { + // TODO recursively descend and recalculate treesize + continue // continue after an error + } + size += csize + } + } + return size, err + +} // HasPropagation checks if the propagation attribute exists and is set to "1" func (n *Node) HasPropagation() (propagation bool) { @@ -642,6 +780,20 @@ func (n *Node) SetTMTime(t time.Time) (err error) { return xattr.Set(n.lu.toInternalPath(n.ID), treeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano))) } +// GetTreeSize reads the treesize from the extended attributes +func (n *Node) GetTreeSize() (treesize uint64, err error) { + var b []byte + if b, err = xattr.Get(n.lu.toInternalPath(n.ID), treesizeAttr); err != nil { + return + } + return strconv.ParseUint(string(b), 10, 64) +} + +// SetTreeSize writes the treesize to the extended attributes +func (n *Node) SetTreeSize(ts uint64) (err error) { + return xattr.Set(n.lu.toInternalPath(n.ID), treesizeAttr, []byte(strconv.FormatUint(ts, 10))) +} + // SetChecksum writes the checksum with the given checksum type to the extended attributes func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { return xattr.Set(n.lu.toInternalPath(n.ID), checksumPrefix+csType, h.Sum(nil)) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index ca83d2aa2ce..1725c9529be 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "syscall" @@ -85,7 +86,11 @@ const ( // the size of the tree below this node, // propagated when treesize_accounting is true and // user.ocis.propagation=1 is set - //treesizeAttr string = ocisPrefix + "treesize" + // stored as uint64, little endian + treesizeAttr string = ocisPrefix + "treesize" + + // the quota for the storage space / tree, regardless who accesses it + quotaAttr string = ocisPrefix + "quota" ) func init() { @@ -184,22 +189,55 @@ func (fs *ocisfs) Shutdown(ctx context.Context) error { return nil } +// TODO Document in the cs3 should we return quota or free space? func (fs *ocisfs) GetQuota(ctx context.Context) (uint64, uint64, error) { - // TODO quota of which storage space? - // we could use the logged in user, but when a user has access to multiple storages this falls short - // for now return quota of root - n, err := fs.lu.NodeFromPath(ctx, "/") + var node *Node + var err error + if node, err = fs.lu.HomeOrRootNode(ctx); err != nil { + return 0, 0, err + } + + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return 0, 0, err + } + + rp, err := fs.p.AssemblePermissions(ctx, node) + switch { + case err != nil: + return 0, 0, errtypes.InternalError(err.Error()) + case !rp.GetQuota: + return 0, 0, errtypes.PermissionDenied(node.ID) + } + + ri, err := node.AsResourceInfo(ctx, rp, []string{"treesize", "quota"}) if err != nil { return 0, 0, err } + + quotaStr := _quotaUnknown + if ri.Opaque != nil && ri.Opaque.Map != nil && ri.Opaque.Map["quota"] != nil && ri.Opaque.Map["quota"].Decoder == "plain" { + quotaStr = string(ri.Opaque.Map["quota"].Value) + } stat := syscall.Statfs_t{} - err = syscall.Statfs(n.lu.toInternalPath(n.ID), &stat) + err = syscall.Statfs(fs.lu.toInternalPath(node.ID), &stat) if err != nil { return 0, 0, err } + total := stat.Blocks * uint64(stat.Bsize) // Total data blocks in filesystem - used := stat.Bavail * uint64(stat.Bsize) // Free blocks available to unprivileged user - return total, used, nil + switch { + case quotaStr == _quotaUncalculated, quotaStr == _quotaUnknown, quotaStr == _quotaUnlimited: + // best we can do is return current total + // TODO indicate unlimited total? + default: + if quota, err := strconv.ParseUint(quotaStr, 10, 64); err == nil { + if total > quota { + total = quota + } + } + } + return total, ri.Size, nil } // CreateHome creates a new root node that has no parent id diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go index 44d27f9d762..7dec8ec1d26 100644 --- a/pkg/storage/fs/ocis/tree.go +++ b/pkg/storage/fs/ocis/tree.go @@ -318,9 +318,11 @@ func (t *Tree) Propagate(ctx context.Context, n *Node) (err error) { // use a sync time and don't rely on the mtime of the current node, as the stat might not change when a rename happened too quickly sTime := time.Now().UTC() + // we loop until we reach the root for err == nil && n.ID != root.ID { log.Debug().Interface("node", n).Msg("propagating") + // make n the parent or break the loop if n, err = n.Parent(); err != nil { break } @@ -376,7 +378,49 @@ func (t *Tree) Propagate(ctx context.Context, n *Node) (err error) { } - // TODO size accounting + // size accounting + if t.lu.Options.TreeSizeAccounting { + // update the treesize if it differs from the current size + updateTreeSize := false + + var treeSize, calculatedTreeSize uint64 + calculatedTreeSize, err = n.CalculateTreeSize(ctx) + if err != nil { + return + } + + treeSize, err = n.GetTreeSize() + switch { + case err != nil: + // missing attribute, or invalid format, overwrite + log.Debug().Err(err). + Interface("node", n). + Msg("could not read treesize attribute, overwriting") + updateTreeSize = true + case treeSize != calculatedTreeSize: + log.Debug(). + Interface("node", n). + Uint64("treesize", treeSize). + Uint64("calculatedTreeSize", calculatedTreeSize). + Msg("parent treesize is different then calculated treesize, updating") + updateTreeSize = true + default: + log.Debug(). + Interface("node", n). + Uint64("treesize", treeSize). + Uint64("calculatedTreeSize", calculatedTreeSize). + Msg("parent size matches calculated size, not updating") + } + + if updateTreeSize { + // update the tree time of the parent node + if err = n.SetTreeSize(calculatedTreeSize); err != nil { + log.Error().Err(err).Interface("node", n).Uint64("calculatedTreeSize", calculatedTreeSize).Msg("could not update treesize of parent node") + return + } + log.Debug().Interface("node", n).Uint64("calculatedTreeSize", calculatedTreeSize).Msg("updated treesize of parent node") + } + } } if err != nil {