Skip to content

Commit

Permalink
implement tree size propagation in ocis
Browse files Browse the repository at this point in the history 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
  • Loading branch information
@butonic
butonic committed Feb 3, 2021
1 parent 5306992 commit 60e4c47
Show file tree
Hide file tree
Showing 5 changed files with 309 additions and 46 deletions.
93 changes: 60 additions & 33 deletions internal/http/services/owncloud/ocdav/propfind.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,22 +93,6 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
return
}

ref := &provider.Reference{
Spec: &provider.Reference_Path{Path: fn},
}
req := &provider.StatRequest{Ref: ref}
res, err := client.Stat(ctx, req)
if err != nil {
sublog.Error().Err(err).Msgf("error sending a grpc stat request to ref: %v", ref)
w.WriteHeader(http.StatusInternalServerError)
return
}

if res.Status.Code != rpc.Code_CODE_OK {
HandleErrorStatus(&sublog, w, res.Status)
return
}

metadataKeys := []string{}
if pf.Allprop != nil {
// TODO this changes the behavior and returns all properties if allprops has been set,
Expand All @@ -124,6 +108,24 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
}
}
}
ref := &provider.Reference{
Spec: &provider.Reference_Path{Path: fn},
}
req := &provider.StatRequest{
Ref: ref,
ArbitraryMetadataKeys: metadataKeys,
}
res, err := client.Stat(ctx, req)
if err != nil {
sublog.Error().Err(err).Interface("req", req).Msg("error sending a grpc stat request")
w.WriteHeader(http.StatusInternalServerError)
return
}

if res.Status.Code != rpc.Code_CODE_OK {
HandleErrorStatus(&sublog, w, res.Status)
return
}

info := res.Info
infos := []*provider.ResourceInfo{info}
Expand Down Expand Up @@ -221,7 +223,14 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
func requiresExplicitFetching(n *xml.Name) bool {
switch n.Space {
case _nsDav:
return false
switch n.Local {
case "quota-available-bytes", "quota-used-bytes":
// A <DAV:allprop> PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes
// from https://www.rfc-editor.org/rfc/rfc4331.html#section-2
return true
default:
return false
}
case _nsOwncloud:
switch n.Local {
case "favorite", "share-types", "checksums":
Expand Down Expand Up @@ -338,7 +347,13 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
}

var ls *link.PublicShare
var treesize, quota string

// -1 indicates uncalculated
// -2 indicates unknown (default)
// -3 indicates unlimited
treesize := _propQuotaUnknown
quota := _propQuotaUnknown
size := fmt.Sprintf("%d", md.Size)
// TODO refactor helper functions: GetOpaqueJSONEncoded(opaque, key string, *struct) err, GetOpaquePlainEncoded(opaque, key) value, err
// or use ok like pattern and return bool?
if md.Opaque != nil && md.Opaque.Map != nil {
Expand All @@ -349,15 +364,16 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
sublog.Error().Err(err).Msg("could not unmarshal link json")
}
}
// TODO the ResourceInfo should have a dedicated TreeSize property for collections.
// Currently, we are reusing the Size but that cannot indicate an unknown size because it is uint64
if md.Opaque.Map["treesize"] != nil && md.Opaque.Map["treesize"].Decoder == "plain" {
treesize = string(md.Opaque.Map["treesize"].Value)
} else {
treesize = _propQuotaUnknown
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
size = treesize
}
}
if md.Opaque.Map["quota"] != nil && md.Opaque.Map["quota"].Decoder == "plain" {
quota = string(md.Opaque.Map["quota"].Value)
} else {
quota = _propQuotaUnknown
}
}

Expand Down Expand Up @@ -411,14 +427,15 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
}

// always return size, well nearly always ... public link shares are a little weird
size := fmt.Sprintf("%d", md.Size)
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
propstatOK.Prop = append(propstatOK.Prop, s.newPropRaw("d:resourcetype", "<d:collection/>"))
if ls == nil {
propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size))
}
propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize))
propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota))
// A <DAV:allprop> PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes
// from https://www.rfc-editor.org/rfc/rfc4331.html#section-2
//propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize))
//propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota))
} else {
propstatOK.Prop = append(propstatOK.Prop,
s.newProp("d:resourcetype", ""),
Expand Down Expand Up @@ -482,7 +499,6 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
// TODO return other properties ... but how do we put them in a namespace?
} else {
// otherwise return only the requested properties
size := fmt.Sprintf("%d", md.Size)
for i := range pf.Prop {
switch pf.Prop[i].Space {
case _nsOwncloud:
Expand Down Expand Up @@ -567,7 +583,8 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:public-link-expiration", ""))
case "size": // phoenix only
// TODO we cannot find out if md.Size is set or not because ints in go default to 0
// oc:size is also available on folders
// TODO what is the difference to d:quota-used-bytes (which only exists for collections)?
// oc:size is available on files and folders and behaves like d:getcontentlength or d:quota-used-bytes respectively
if ls == nil {
propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size))
} else {
Expand Down Expand Up @@ -718,17 +735,20 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
} else {
propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:getlastmodified", ""))
}
case "quota-used-bytes": // desktop client
// oc10 renders the tree size here
case "quota-used-bytes": // RFC 4331
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
// TODO RFC says if treesize is unknown return it as not found, what does oc10 do?
// always returns the current usage,
// in oc10 there seems to be a bug that makes the size in webdav differ from the one in the user properties, not taking shares into account
// in ocis we plan to always mak the quota a property of the storage space
propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", treesize))
} else {
propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-used-bytes", ""))
}
case "quota-available-bytes": // desktop client
// we can start with hardcoding unknown
case "quota-available-bytes": // RFC 4331
if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", _propQuotaUnknown))
// oc10 returns -3 for unlimited, -2 for unknown, -1 for uncalculated
propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota))
} else {
propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-available-bytes", ""))
}
Expand Down Expand Up @@ -805,7 +825,14 @@ func (c *countingReader) Read(p []byte) (int, error) {
}

func metadataKeyOf(n *xml.Name) string {
return fmt.Sprintf("%s/%s", n.Space, n.Local)
switch {
case n.Space == _nsDav && n.Local == "quota-used-bytes":
return "treesize"
case n.Space == _nsDav && n.Local == "quota-available-bytes":
return "quota"
default:
return fmt.Sprintf("%s/%s", n.Space, n.Local)
}
}

// http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind)
Expand Down
10 changes: 6 additions & 4 deletions internal/http/services/owncloud/ocs/handlers/cloud/users/users.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
if user != u.Username {
// FIXME allow fetching other users info?
// FIXME allow fetching other users info? only for admins
response.WriteOCSError(w, r, http.StatusForbidden, "user id mismatch", fmt.Errorf("%s tried to access %s user info endpoint", u.Id.OpaqueId, user))
return
}
Expand Down Expand Up @@ -147,9 +147,11 @@ func (h *Handler) handleUsers(w http.ResponseWriter, r *http.Request, u *userpb.
response.WriteOCSSuccess(w, r, &Users{
// ocs can only return the home storage quota
Quota: &Quota{
Free: int64(getQuotaRes.TotalBytes - getQuotaRes.UsedBytes),
Used: int64(getQuotaRes.UsedBytes),
Total: int64(getQuotaRes.TotalBytes), // -1, -2 have special meaning?
Free: int64(getQuotaRes.TotalBytes - getQuotaRes.UsedBytes),
Used: int64(getQuotaRes.UsedBytes),
// TODO support negative values or flags for the quota to carry special meaning: -1 = uncalculated, -2 = unknown, -3 = unlimited
// for now we can only report total and used
Total: int64(getQuotaRes.TotalBytes),
Relative: float32(float64(getQuotaRes.UsedBytes) / float64(getQuotaRes.TotalBytes)),
Definition: "default",
},
Expand Down
152 changes: 152 additions & 0 deletions pkg/storage/fs/ocis/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"

Expand All @@ -50,6 +51,12 @@ const (

_favoriteKey = "http://owncloud.org/ns/favorite"
_checksumsKey = "http://owncloud.org/ns/checksums"
_treesizeKey = "treesize"
_quotaKey = "quota"

_quotaUncalculated = "-1"
_quotaUnknown = "-2"
_quotaUnlimited = "-3"
)

// Node represents a node in the tree and provides methods to get a Parent or Child instance
Expand Down Expand Up @@ -547,6 +554,31 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi
readChecksumIntoOpaque(ctx, nodePath, storageprovider.XSAdler32, ri)
}

// treesize
if _, ok := mdKeysMap[_treesizeKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok {
readTreesizeIntoOpaque(ctx, nodePath, ri)
}

// quota
if _, ok := mdKeysMap[_quotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllKeys || ok {
var quotaPath string
if n.lu.Options.EnableHome {
if r, err := n.lu.HomeNode(ctx); err == nil {
quotaPath = n.lu.toInternalPath(r.ID)
readQuotaIntoOpaque(ctx, quotaPath, ri)
} else {
sublog.Error().Err(err).Msg("error determining home node for quota")
}
} else {
if r, err := n.lu.RootNode(ctx); err == nil {
quotaPath = n.lu.toInternalPath(r.ID)
readQuotaIntoOpaque(ctx, quotaPath, ri)
} else {
sublog.Error().Err(err).Msg("error determining root node for quota")
}
}
}

// only read the requested metadata attributes
attrs, err := xattr.List(nodePath)
if err != nil {
Expand Down Expand Up @@ -619,6 +651,112 @@ func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *prov
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("could not read checksum")
}
}
func readTreesizeIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) {
v, err := xattr.Get(nodePath, treesizeAttr)
switch {
case err == nil:
// make sure we have a proper unsigned int
if treesize, err := strconv.ParseUint(string(v), 10, 64); err == nil {
if ri.Opaque == nil {
ri.Opaque = &types.Opaque{
Map: map[string]*types.OpaqueEntry{},
}
}
ri.Opaque.Map[_treesizeKey] = &types.OpaqueEntry{
Decoder: "plain",
Value: v,
}
// when it exists it also overrules the size
ri.Size = treesize
} else {
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("treesize", string(v)).Msg("malformed treesize")
}
case isNoData(err):
appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("treesize not set")
case isNotFound(err):
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("file not found when reading treesize")
default:
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read treesize")
}
}

// quota is always stored on the root node
func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) {
v, err := xattr.Get(nodePath, quotaAttr)
switch {
case err == nil:
// make sure we have a proper signed int
// we use the same magic numbers to indicate:
// -1 = uncalculated
// -2 = unknown
// -3 = unlimited
if _, err := strconv.ParseInt(string(v), 10, 64); err == nil {
if ri.Opaque == nil {
ri.Opaque = &types.Opaque{
Map: map[string]*types.OpaqueEntry{},
}
}
ri.Opaque.Map[_quotaKey] = &types.OpaqueEntry{
Decoder: "plain",
Value: v,
}
} else {
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("treesize", string(v)).Msg("malformed quota")
}
case isNoData(err):
appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("quota not set")
case isNotFound(err):
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("file not found when reading quota")
default:
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read quota")
}
}

func (n *Node) CalculateTreeSize(ctx context.Context) (uint64, error) {
var size uint64
// TODO check if this is a dir?
nodePath := n.lu.toInternalPath(n.ID)

f, err := os.Open(nodePath)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not open dir")
return 0, err
}
defer f.Close()

names, err := f.Readdirnames(0)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Msg("could not read dirnames")
return 0, err
}
for i := range names {
cPath := filepath.Join(nodePath, names[i])
info, err := os.Stat(cPath)
if err != nil {
appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not stat child entry")
continue // continue after an error
}
if !info.IsDir() {
size += uint64(info.Size())
} else {
// read from attr
var b []byte
// xattr.Get will follow the symlink
if b, err = xattr.Get(cPath, treesizeAttr); err != nil {
// TODO recursively descend and recalculate treesize
continue // continue after an error
}
csize, err := strconv.ParseUint(string(b), 10, 64)
if err != nil {
// TODO recursively descend and recalculate treesize
continue // continue after an error
}
size += csize
}
}
return size, err

}

// HasPropagation checks if the propagation attribute exists and is set to "1"
func (n *Node) HasPropagation() (propagation bool) {
Expand All @@ -642,6 +780,20 @@ func (n *Node) SetTMTime(t time.Time) (err error) {
return xattr.Set(n.lu.toInternalPath(n.ID), treeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano)))
}

// GetTreeSize reads the treesize from the extended attributes
func (n *Node) GetTreeSize() (treesize uint64, err error) {
var b []byte
if b, err = xattr.Get(n.lu.toInternalPath(n.ID), treesizeAttr); err != nil {
return
}
return strconv.ParseUint(string(b), 10, 64)
}

// SetTreeSize writes the treesize to the extended attributes
func (n *Node) SetTreeSize(ts uint64) (err error) {
return xattr.Set(n.lu.toInternalPath(n.ID), treesizeAttr, []byte(strconv.FormatUint(ts, 10)))
}

// SetChecksum writes the checksum with the given checksum type to the extended attributes
func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) {
return xattr.Set(n.lu.toInternalPath(n.ID), checksumPrefix+csType, h.Sum(nil))
Expand Down
Loading

0 comments on commit 60e4c47

Please sign in to comment.