quota querying and tree accounting

changelog/unreleased/ocis-quota.md

@@ -0,0 +1,5 @@
+Enhancement: quota querying and tree accounting
+
+The ocs api now returns the user quota for the users home storage. Furthermore, the ocis storage driver now reads the quota from the extended attributes of the user home or root node and implements tree size accounting. Finally, ocdav PROPFINDS now handle the `DAV:quota-used-bytes` and `DAV:quote-available-bytes` properties.
+
+https://github.com/cs3org/reva/pull/1405

internal/grpc/services/gateway/storageprovider.go

@@ -1762,9 +1762,20 @@ func (s *svc) PurgeRecycle(ctx context.Context, req *gateway.PurgeRecycleRequest
 	return res, nil
 }
 
-func (s *svc) GetQuota(ctx context.Context, _ *gateway.GetQuotaRequest) (*provider.GetQuotaResponse, error) {
-	res := &provider.GetQuotaResponse{
-		Status: status.NewUnimplemented(ctx, nil, "GetQuota not yet implemented"),
+func (s *svc) GetQuota(ctx context.Context, req *gateway.GetQuotaRequest) (*provider.GetQuotaResponse, error) {
+	c, err := s.find(ctx, req.Ref)
+	if err != nil {
+		return &provider.GetQuotaResponse{
+			Status: status.NewStatusFromErrType(ctx, "GetQuota ref="+req.Ref.String(), err),
+		}, nil
+	}
+
+	res, err := c.GetQuota(ctx, &provider.GetQuotaRequest{
+		Opaque: req.GetOpaque(),
+		//Ref:    req.GetRef(), // TODO send which storage space ... or root
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "gateway: error calling GetQuota")
 	}
 	return res, nil
 }

internal/grpc/services/storageprovider/storageprovider.go

@@ -1062,8 +1062,8 @@ func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (
 
 	res := &provider.GetQuotaResponse{
 		Status:     status.NewOK(ctx),
-		TotalBytes: uint64(total),
-		UsedBytes:  uint64(used),
+		TotalBytes: total,
+		UsedBytes:  used,
 	}
 	return res, nil
 }

internal/http/services/owncloud/ocdav/propfind.go

@@ -52,6 +52,10 @@ const (
 
 	// RFC1123 time that mimics oc10. time.RFC1123 would end in "UTC", see https://github.com/golang/go/issues/13781
 	RFC1123 = "Mon, 02 Jan 2006 15:04:05 GMT"
+
+	//_propQuotaUncalculated = "-1"
+	_propQuotaUnknown = "-2"
+	//_propQuotaUnlimited    = "-3"
 )
 
 // ns is the namespace that is prefixed to the path in the cs3 namespace
@@ -89,22 +93,6 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
 		return
 	}
 
-	ref := &provider.Reference{
-		Spec: &provider.Reference_Path{Path: fn},
-	}
-	req := &provider.StatRequest{Ref: ref}
-	res, err := client.Stat(ctx, req)
-	if err != nil {
-		sublog.Error().Err(err).Msgf("error sending a grpc stat request to ref: %v", ref)
-		w.WriteHeader(http.StatusInternalServerError)
-		return
-	}
-
-	if res.Status.Code != rpc.Code_CODE_OK {
-		HandleErrorStatus(&sublog, w, res.Status)
-		return
-	}
-
 	metadataKeys := []string{}
 	if pf.Allprop != nil {
 		// TODO this changes the behavior and returns all properties if allprops has been set,
@@ -120,6 +108,24 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
 			}
 		}
 	}
+	ref := &provider.Reference{
+		Spec: &provider.Reference_Path{Path: fn},
+	}
+	req := &provider.StatRequest{
+		Ref:                   ref,
+		ArbitraryMetadataKeys: metadataKeys,
+	}
+	res, err := client.Stat(ctx, req)
+	if err != nil {
+		sublog.Error().Err(err).Interface("req", req).Msg("error sending a grpc stat request")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if res.Status.Code != rpc.Code_CODE_OK {
+		HandleErrorStatus(&sublog, w, res.Status)
+		return
+	}
 
 	info := res.Info
 	infos := []*provider.ResourceInfo{info}
@@ -217,10 +223,17 @@ func (s *svc) handlePropfind(w http.ResponseWriter, r *http.Request, ns string)
 func requiresExplicitFetching(n *xml.Name) bool {
 	switch n.Space {
 	case _nsDav:
-		return false
+		switch n.Local {
+		case "quota-available-bytes", "quota-used-bytes":
+			//  A <DAV:allprop> PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes
+			// from https://www.rfc-editor.org/rfc/rfc4331.html#section-2
+			return true
+		default:
+			return false
+		}
 	case _nsOwncloud:
 		switch n.Local {
-		case "favorite", "share-types", "checksums":
+		case "favorite", "share-types", "checksums", "size":
 			return true
 		default:
 			return false
@@ -334,11 +347,24 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
 	}
 
 	var ls *link.PublicShare
-	if md.Opaque != nil && md.Opaque.Map != nil && md.Opaque.Map["link-share"] != nil && md.Opaque.Map["link-share"].Decoder == "json" {
-		ls = &link.PublicShare{}
-		err := json.Unmarshal(md.Opaque.Map["link-share"].Value, ls)
-		if err != nil {
-			sublog.Error().Err(err).Msg("could not unmarshal link json")
+
+	// -1 indicates uncalculated
+	// -2 indicates unknown (default)
+	// -3 indicates unlimited
+	quota := _propQuotaUnknown
+	size := fmt.Sprintf("%d", md.Size)
+	// TODO refactor helper functions: GetOpaqueJSONEncoded(opaque, key string, *struct) err, GetOpaquePlainEncoded(opaque, key) value, err
+	// or use ok like pattern and return bool?
+	if md.Opaque != nil && md.Opaque.Map != nil {
+		if md.Opaque.Map["link-share"] != nil && md.Opaque.Map["link-share"].Decoder == "json" {
+			ls = &link.PublicShare{}
+			err := json.Unmarshal(md.Opaque.Map["link-share"].Value, ls)
+			if err != nil {
+				sublog.Error().Err(err).Msg("could not unmarshal link json")
+			}
+		}
+		if md.Opaque.Map["quota"] != nil && md.Opaque.Map["quota"].Decoder == "plain" {
+			quota = string(md.Opaque.Map["quota"].Value)
 		}
 	}
 
@@ -389,12 +415,15 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
 		}
 
 		// always return size, well nearly always ... public link shares are a little weird
-		size := fmt.Sprintf("%d", md.Size)
 		if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
 			propstatOK.Prop = append(propstatOK.Prop, s.newPropRaw("d:resourcetype", "<d:collection/>"))
 			if ls == nil {
 				propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size))
 			}
+			//  A <DAV:allprop> PROPFIND request SHOULD NOT return DAV:quota-available-bytes and DAV:quota-used-bytes
+			// from https://www.rfc-editor.org/rfc/rfc4331.html#section-2
+			//propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", size))
+			//propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota))
 		} else {
 			propstatOK.Prop = append(propstatOK.Prop,
 				s.newProp("d:resourcetype", ""),
@@ -458,7 +487,6 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
 		// TODO return other properties ... but how do we put them in a namespace?
 	} else {
 		// otherwise return only the requested properties
-		size := fmt.Sprintf("%d", md.Size)
 		for i := range pf.Prop {
 			switch pf.Prop[i].Space {
 			case _nsOwncloud:
@@ -539,7 +567,8 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
 					propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("oc:public-link-expiration", ""))
 				case "size": // phoenix only
 					// TODO we cannot find out if md.Size is set or not because ints in go default to 0
-					// oc:size is also available on folders
+					// TODO what is the difference to d:quota-used-bytes (which only exists for collections)?
+					// oc:size is available on files and folders and behaves like d:getcontentlength or d:quota-used-bytes respectively
 					if ls == nil {
 						propstatOK.Prop = append(propstatOK.Prop, s.newProp("oc:size", size))
 					} else {
@@ -690,6 +719,22 @@ func (s *svc) mdToPropResponse(ctx context.Context, pf *propfindXML, md *provide
 					} else {
 						propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:getlastmodified", ""))
 					}
+				case "quota-used-bytes": // RFC 4331
+					if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
+						// always returns the current usage,
+						// in oc10 there seems to be a bug that makes the size in webdav differ from the one in the user properties, not taking shares into account
+						// in ocis we plan to always mak the quota a property of the storage space
+						propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-used-bytes", size))
+					} else {
+						propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-used-bytes", ""))
+					}
+				case "quota-available-bytes": // RFC 4331
+					if md.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER {
+						// oc10 returns -3 for unlimited, -2 for unknown, -1 for uncalculated
+						propstatOK.Prop = append(propstatOK.Prop, s.newProp("d:quota-available-bytes", quota))
+					} else {
+						propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:quota-available-bytes", ""))
+					}
 				default:
 					propstatNotFound.Prop = append(propstatNotFound.Prop, s.newProp("d:"+pf.Prop[i].Local, ""))
 				}
@@ -763,7 +808,12 @@ func (c *countingReader) Read(p []byte) (int, error) {
 }
 
 func metadataKeyOf(n *xml.Name) string {
-	return fmt.Sprintf("%s/%s", n.Space, n.Local)
+	switch {
+	case n.Space == _nsDav && n.Local == "quota-available-bytes":
+		return "quota"
+	default:
+		return fmt.Sprintf("%s/%s", n.Space, n.Local)
+	}
 }
 
 // http://www.webdav.org/specs/rfc4918.html#ELEMENT_prop (for propfind)

internal/http/services/owncloud/ocs/handlers/cloud/cloud.go

@@ -37,11 +37,12 @@ type Handler struct {
 }
 
 // Init initializes this and any contained handlers
-func (h *Handler) Init(c *config.Config) {
+func (h *Handler) Init(c *config.Config) error {
 	h.UserHandler = new(user.Handler)
-	h.UsersHandler = new(users.Handler)
 	h.CapabilitiesHandler = new(capabilities.Handler)
 	h.CapabilitiesHandler.Init(c)
+	h.UsersHandler = new(users.Handler)
+	return h.UsersHandler.Init(c)
 }
 
 // Handler routes the cloud endpoints

internal/http/services/owncloud/ocs/handlers/cloud/users/users.go

@@ -22,13 +22,28 @@ import (
 	"fmt"
 	"net/http"
 
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
+	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/internal/http/services/owncloud/ocdav"
+	"github.com/cs3org/reva/internal/http/services/owncloud/ocs/config"
 	"github.com/cs3org/reva/internal/http/services/owncloud/ocs/response"
+	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/pkg/rhttp/router"
 	ctxuser "github.com/cs3org/reva/pkg/user"
 )
 
-// The UsersHandler renders user data for the user id given in the url path
+// Handler renders user data for the user id given in the url path
 type Handler struct {
+	gatewayAddr string
+}
+
+// Init initializes this and any contained handlers
+func (h *Handler) Init(c *config.Config) error {
+	h.gatewayAddr = c.GatewaySvc
+	return nil
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -44,7 +59,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if user != u.Username {
-		// FIXME allow fetching other users info?
+		// FIXME allow fetching other users info? only for admins
 		response.WriteOCSError(w, r, http.StatusForbidden, "user id mismatch", fmt.Errorf("%s tried to access %s user info endpoint", u.Id.OpaqueId, user))
 		return
 	}
@@ -53,19 +68,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	head, r.URL.Path = router.ShiftPath(r.URL.Path)
 	switch head {
 	case "":
-		response.WriteOCSSuccess(w, r, &Users{
-			// FIXME query storages? cache a summary?
-			// TODO use list of storages to allow clients to resolve quota status
-			Quota: &Quota{
-				Free:       2840756224000,
-				Used:       5059416668,
-				Total:      2845815640668,
-				Relative:   0.18,
-				Definition: "default",
-			},
-			DisplayName: u.DisplayName,
-			Email:       u.Mail,
-		})
+		h.handleUsers(w, r, u)
 		return
 	case "groups":
 		response.WriteOCSSuccess(w, r, &Groups{})
@@ -100,3 +103,59 @@ type Users struct {
 type Groups struct {
 	Groups []string `json:"groups" xml:"groups>element"`
 }
+
+func (h *Handler) handleUsers(w http.ResponseWriter, r *http.Request, u *userpb.User) {
+	ctx := r.Context()
+	sublog := appctx.GetLogger(r.Context())
+
+	gc, err := pool.GetGatewayServiceClient(h.gatewayAddr)
+	if err != nil {
+		sublog.Error().Err(err).Msg("error getting gateway client")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	getHomeRes, err := gc.GetHome(ctx, &provider.GetHomeRequest{})
+	if err != nil {
+		sublog.Error().Err(err).Msg("error calling GetHome")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	if getHomeRes.Status.Code != rpc.Code_CODE_OK {
+		ocdav.HandleErrorStatus(sublog, w, getHomeRes.Status)
+		return
+	}
+
+	getQuotaRes, err := gc.GetQuota(ctx, &gateway.GetQuotaRequest{
+		Ref: &provider.Reference{
+			Spec: &provider.Reference_Path{
+				Path: getHomeRes.Path,
+			},
+		},
+	})
+	if err != nil {
+		sublog.Error().Err(err).Msg("error calling GetQuota")
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if getQuotaRes.Status.Code != rpc.Code_CODE_OK {
+		ocdav.HandleErrorStatus(sublog, w, getQuotaRes.Status)
+		return
+	}
+
+	response.WriteOCSSuccess(w, r, &Users{
+		// ocs can only return the home storage quota
+		Quota: &Quota{
+			Free: int64(getQuotaRes.TotalBytes - getQuotaRes.UsedBytes),
+			Used: int64(getQuotaRes.UsedBytes),
+			// TODO support negative values or flags for the quota to carry special meaning: -1 = uncalculated, -2 = unknown, -3 = unlimited
+			// for now we can only report total and used
+			Total:      int64(getQuotaRes.TotalBytes),
+			Relative:   float32(float64(getQuotaRes.UsedBytes) / float64(getQuotaRes.TotalBytes)),
+			Definition: "default",
+		},
+		DisplayName: u.DisplayName,
+		Email:       u.Mail,
+	})
+}

internal/http/services/owncloud/ocs/v1.go

@@ -37,15 +37,14 @@ type V1Handler struct {
 }
 
 func (h *V1Handler) init(c *config.Config) error {
+	h.ConfigHandler = new(configHandler.Handler)
+	h.ConfigHandler.Init(c)
 	h.AppsHandler = new(apps.Handler)
 	if err := h.AppsHandler.Init(c); err != nil {
 		return err
 	}
 	h.CloudHandler = new(cloud.Handler)
-	h.CloudHandler.Init(c)
-	h.ConfigHandler = new(configHandler.Handler)
-	h.ConfigHandler.Init(c)
-	return nil
+	return h.CloudHandler.Init(c)
 }
 
 // Handler handles requests