Skip to content

Commit

Permalink
use arbitrary metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@micbar
micbar committed Feb 10, 2022
1 parent 4b143b9 commit aebad4f
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 187 deletions.
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,5 @@ require (
stash.kopano.io/kgol/kcc-go/v5 v5.0.1 // indirect
stash.kopano.io/kgol/oidc-go v0.3.2 // indirect
)

replace github.com/cs3org/reva => ../reva
86 changes: 86 additions & 0 deletions graph/pkg/service/v0/driveitems.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"context"
"fmt"
"net/http"
"net/url"
"path"
"path/filepath"
"time"

cs3rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
Expand Down Expand Up @@ -142,3 +144,87 @@ func cs3ResourceToDriveItem(res *storageprovider.ResourceInfo) (*libregraph.Driv
}
return driveItem, nil
}

func (g Graph) getDriveMetadata(ctx context.Context, space *storageprovider.StorageSpace) (map[string]string, error) {
client := g.GetGatewayClient()
sResp, err := client.Stat(
ctx,
&storageprovider.StatRequest{
Ref: &storageprovider.Reference{
ResourceId: &storageprovider.ResourceId{
StorageId: space.Root.StorageId,
OpaqueId: space.Root.OpaqueId,
},
},
},
)
if err != nil {
g.logger.Debug().Err(err).Interface("space", space).Msg("transport error")
return nil, err
}
if sResp.Status.Code != cs3rpc.Code_CODE_OK {
g.logger.Debug().Interface("space", space).Msg("space not found")
return nil, fmt.Errorf("space not found")
}
md := sResp.Info.ArbitraryMetadata.GetMetadata()
if md == nil {
g.logger.Error().Err(err).Interface("space", space).Msg("could not read metadata from space")
return nil, fmt.Errorf("could not read metadata from space")
}
return md, nil
}

func (g Graph) setSpaceMetadata(ctx context.Context, metadata map[string]string, resourceID *storageprovider.ResourceId) error {
if len(metadata) == 0 {
return nil
}
client := g.GetGatewayClient()
resMd, err := client.SetArbitraryMetadata(
ctx,
&storageprovider.SetArbitraryMetadataRequest{
ArbitraryMetadata: &storageprovider.ArbitraryMetadata{
Metadata: metadata,
},
Ref: &storageprovider.Reference{
ResourceId: resourceID,
},
},
)
if err != nil {
g.logger.Error().Msg("transport error, could not set metadata")
return err
}
if resMd.Status.Code != cs3rpc.Code_CODE_OK {
g.logger.Error().Msg("could not set metadata")
return fmt.Errorf("could not set metadata")
}
return nil
}

func (g Graph) GetSpecialSpaceItems(ctx context.Context, baseURL *url.URL, spaceRootID *storageprovider.ResourceId, metadata map[string]string) []libregraph.DriveItem {
var spaceItems []libregraph.DriveItem
if metadata == nil {
return spaceItems
}
if readmePath, ok := metadata[ReadmePathAttrName]; ok {
readmeItem, err := g.getDriveItem(ctx, spaceRootID, readmePath)
if err != nil {
g.logger.Error().Err(err).Str(ReadmePathAttrName, readmePath).Msg("Could not get readme Item")
} else {
readmeItem.SpecialFolder = &libregraph.SpecialFolder{Name: libregraph.PtrString(ReadmePathSpecialFolderName)}
readmeItem.WebDavUrl = libregraph.PtrString(baseURL.String() + filepath.Join(spaceRootID.OpaqueId, readmePath))
spaceItems = append(spaceItems, *readmeItem)
}
}
if imagePath, ok := metadata[SpaceImageAttrName]; ok {
imageItem, err := g.getDriveItem(ctx, spaceRootID, imagePath)
if err != nil {
g.logger.Error().Err(err).Str(SpaceImageAttrName, imagePath).Msg("Could not get image Item")
} else {
imageItem.SpecialFolder = &libregraph.SpecialFolder{Name: libregraph.PtrString(SpaceImageSpecialFolderName)}
imageItem.WebDavUrl = libregraph.PtrString(baseURL.String() + filepath.Join(spaceRootID.OpaqueId, imagePath))
spaceItems = append(spaceItems, *imageItem)
}
}
return spaceItems
}
236 changes: 49 additions & 187 deletions graph/pkg/service/v0/drives.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"path/filepath"
"strconv"
"strings"
"time"

"github.com/CiscoM31/godata"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
Expand All @@ -19,17 +18,13 @@ import (
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
ctxpkg "github.com/cs3org/reva/pkg/ctx"
"github.com/cs3org/reva/pkg/rhttp"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/graph/pkg/service/v0/errorcode"
"github.com/owncloud/ocis/graph/pkg/service/v0/net"
"github.com/owncloud/ocis/ocis-pkg/service/grpc"
settingssvc "github.com/owncloud/ocis/protogen/gen/ocis/services/settings/v0"
settingsServiceExt "github.com/owncloud/ocis/settings/pkg/service/v0"
"gopkg.in/yaml.v2"

merrors "go-micro.dev/v4/errors"
)

Expand Down Expand Up @@ -284,6 +279,35 @@ func (g Graph) UpdateDrive(w http.ResponseWriter, r *http.Request) {
if drive.Name != nil {
updateSpaceRequest.StorageSpace.Name = *drive.Name
}
metadata := make(map[string]string)
for _, specialItem := range drive.Special {

baseUrl, err := url.Parse(g.config.Spaces.WebDavBase + g.config.Spaces.WebDavPath + driveID)
if err != nil {
g.logger.Error().Err(err)
}
newURL, err := filepath.Rel(baseUrl.String(), *specialItem.WebDavUrl)
if err != nil {
g.logger.Error().Err(err)
}
switch *specialItem.SpecialFolder.Name {
case SpaceImageSpecialFolderName:
metadata[SpaceImageAttrName] = newURL
case ReadmePathSpecialFolderName:
metadata[ReadmePathAttrName] = newURL
}
}

if drive.Description != nil {
metadata[SpaceDescriptionAttrName] = *drive.Description
}

if metadata != nil {
err = g.setSpaceMetadata(r.Context(), metadata, root)
if err != nil {
g.logger.Error().Err(err)
}
}

if drive.Quota.HasTotal() {
user := ctxpkg.ContextMustGetUser(r.Context())
Expand Down Expand Up @@ -325,57 +349,44 @@ func (g Graph) UpdateDrive(w http.ResponseWriter, r *http.Request) {
return
}

updatedDrive, err := g.cs3StorageSpaceToDrive(wdu, resp.StorageSpace)
spaces, err := g.formatDrives(r.Context(), wdu, []*storageprovider.StorageSpace{resp.StorageSpace})
if err != nil {
g.logger.Error().Err(err).Msg("error parsing space")
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}

render.Status(r, http.StatusOK)
render.JSON(w, r, updatedDrive)
render.JSON(w, r, spaces[0])
}

func (g Graph) formatDrives(ctx context.Context, baseURL *url.URL, mds []*storageprovider.StorageSpace) ([]*libregraph.Drive, error) {
responses := make([]*libregraph.Drive, 0, len(mds))
for _, space := range mds {
res, err := g.cs3StorageSpaceToDrive(baseURL, space)
for i := range mds {
res, err := g.cs3StorageSpaceToDrive(baseURL, mds[i])
if err != nil {
return nil, err
}
spaceProperties, err := g.getExtendedSpaceProperties(ctx, space)
md, err := g.getDriveMetadata(ctx, mds[i])
if err != nil {
g.logger.Error().Err(err).Interface("space", space).Msg("error reading extendedSpaceProperties")
g.logger.Error().Err(err).Interface("space", mds[i]).Msg("error reading extendedSpaceProperties")
continue
}
if err == nil {
if spaceProperties.Description != "" {
res.Description = &spaceProperties.Description
}
if len(spaceProperties.Special) > 0 {
s := make([]libregraph.DriveItem, 0, len(spaceProperties.Special))
for name, relativePath := range spaceProperties.Special {
sdi, err := g.getDriveItem(ctx, space.Root, relativePath)
if err != nil {
// TODO cach not found response
g.logger.Debug().Err(err).Interface("space", space).Interface("path", relativePath).Msg("error fetching drive item")
continue
}
n := name // copy the name to a dedicated variable
sdi.SpecialFolder = &libregraph.SpecialFolder{
Name: &n,
}
webdavURL := baseURL.String() + filepath.Join(space.Id.OpaqueId, relativePath)
sdi.WebDavUrl = &webdavURL

// TODO cache until ./.config/ocis/space.yaml file changes
s = append(s, *sdi)
}
res.Special = s
}

if sd, ok := md[SpaceDescriptionAttrName]; ok {
res.Description = libregraph.PtrString(sd)
}
// TODO this overwrites the quota that might already have been mapped in cs3StorageSpaceToDrive above ... move this into the cs3StorageSpaceToDrive method?
res.Quota, err = g.getDriveQuota(ctx, space)
res.Special = g.GetSpecialSpaceItems(
ctx,
baseURL,
&storageprovider.ResourceId{
StorageId: mds[i].Root.StorageId,
OpaqueId: mds[i].Root.OpaqueId,
},
md,
)

res.Quota, err = g.getDriveQuota(ctx, mds[i])
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -578,155 +589,6 @@ func (g Graph) getDriveQuota(ctx context.Context, space *storageprovider.Storage
return &qta, nil
}

// ExtendedSpaceProperties are stored in a file
type ExtendedSpaceProperties struct {
Version string `yaml:"version" json:"version"`
Description string `yaml:"description" json:"description"`
// map of {name} -> {relative path to resource}, eg:
// readme -> readme.md
// image -> .config/ocis/space.png
Special map[string]string `yaml:"special" json:"special"`
}

// generates a space root stat cache key used to detect changes in a space
func spaceRootStatKey(id *storageprovider.ResourceId) string {
if id == nil || id.StorageId == "" || id.OpaqueId == "" {
return ""
}
return "sid:" + id.StorageId + "!oid:" + id.OpaqueId
}

type spacePropertiesEntry struct {
spaceProperties ExtendedSpaceProperties
rootMtime *types.Timestamp
}

func (g Graph) getExtendedSpaceProperties(ctx context.Context, space *storageprovider.StorageSpace) (*ExtendedSpaceProperties, error) {

// if the root is older or equal to our cache we can reuse the cached extended spaces properties
if syc, err := g.spacePropertiesCache.Get(spaceRootStatKey(space.Root)); err == nil {
if spe, ok := syc.(spacePropertiesEntry); ok {
if spe.rootMtime != nil && space.Mtime != nil {
if spe.rootMtime.Seconds > space.Mtime.Seconds { // second precision is good enough
return &spe.spaceProperties, nil
}
}
}
}

client := g.GetGatewayClient()

dlReq := &storageprovider.InitiateFileDownloadRequest{
Ref: &storageprovider.Reference{
ResourceId: &storageprovider.ResourceId{
StorageId: space.Root.StorageId,
OpaqueId: space.Root.OpaqueId,
},
Path: "./.config/ocis/space.yaml",
// TODO what if a public share should have a readme and an image?
// should we just default to a ./Readme.md and ./folder.png/jpg?
// what existing conventions could we use? .desktop file? .env file?
// how should users set a README fo public link file shares? They only point to a file, not a folder that could contain a readme and image
// should weo reuse the readme and image of the space that contains the file shared via link?
},
}
//ctx = metadata.AppendToOutgoingContext(ctx, headers.IfModifiedSince, "TODO grpc has no official cache headers")
// FIXME how can clients retrieve a file just by id?
// The drive Item does currently not have a relative path ...
// so clients would have to make a request by id ... but webdav cannot do that ...
// TODO initiate file download only if the etag does not match
rsp, err := client.InitiateFileDownload(ctx, dlReq)
if err != nil {
return nil, err
}
switch rsp.Status.Code {
case cs3rpc.Code_CODE_OK:
// continue
case cs3rpc.Code_CODE_NOT_FOUND:
// cache an empty instance
spacePropertiesEntry := spacePropertiesEntry{
spaceProperties: ExtendedSpaceProperties{},
rootMtime: space.Mtime,
}
if err := g.spacePropertiesCache.SetWithTTL(spaceRootStatKey(space.Root), spacePropertiesEntry, time.Second*time.Duration(g.config.Spaces.ExtendedSpacePropertiesCacheTTL)); err != nil {
g.logger.Error().Err(err).Msg("could not cache extended space properties")
}

return &spacePropertiesEntry.spaceProperties, nil
default:
return nil, fmt.Errorf("could not initiate download of %s: %s", dlReq.Ref.Path, rsp.Status.Message)
}

var ep, tk string
for _, p := range rsp.Protocols {
if p.Protocol == "spaces" {
ep, tk = p.DownloadEndpoint, p.Token
}
}
if ep == "" {
return nil, fmt.Errorf("space does not support the spaces download protocol")
}

httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, ep, nil)
if err != nil {
return nil, err
}
httpReq.Header.Set(net.HeaderTokenTransport, tk)

httpClient := g.GetHTTPClient()

resp, err := httpClient.Do(httpReq) // nolint:bodyclose
if err != nil {
return nil, err
}

switch resp.StatusCode {
case http.StatusOK:
// continue
case http.StatusNotFound:
// cache an empty instance
spacePropertiesEntry := spacePropertiesEntry{
spaceProperties: ExtendedSpaceProperties{},
rootMtime: space.Mtime,
}
if err := g.spacePropertiesCache.SetWithTTL(spaceRootStatKey(space.Root), spacePropertiesEntry, time.Second*time.Duration(g.config.Spaces.ExtendedSpacePropertiesCacheTTL)); err != nil {
g.logger.Error().Err(err).Msg("could not cache extended space properties")
}

return &spacePropertiesEntry.spaceProperties, nil
default:
return nil, fmt.Errorf("could not get the .space.yaml. Request returned with statuscode %d ", resp.StatusCode)
}

spaceProperties := ExtendedSpaceProperties{}
if err := yaml.NewDecoder(resp.Body).Decode(&spaceProperties); err != nil {
g.logger.Debug().Err(err).Msg("invalid space yaml, ignoring")

// cache an empty instance
// TODO insert an 'invalid yaml' item? how can we return an error to the user?
spacePropertiesEntry := spacePropertiesEntry{
spaceProperties: ExtendedSpaceProperties{},
rootMtime: space.Mtime,
}
if err := g.spacePropertiesCache.SetWithTTL(spaceRootStatKey(space.Root), spacePropertiesEntry, time.Second*time.Duration(g.config.Spaces.ExtendedSpacePropertiesCacheTTL)); err != nil {
g.logger.Error().Err(err).Msg("could not cache extended space properties")
}

return &spacePropertiesEntry.spaceProperties, nil
}

// cache properties
spacePropertiesEntry := spacePropertiesEntry{
spaceProperties: spaceProperties,
rootMtime: space.Mtime,
}
if err := g.spacePropertiesCache.SetWithTTL(spaceRootStatKey(space.Root), spacePropertiesEntry, time.Second*time.Duration(g.config.Spaces.ExtendedSpacePropertiesCacheTTL)); err != nil {
g.logger.Error().Err(err).Msg("could not cache extended space properties")
}

return &spaceProperties, nil
}

func calculateQuotaState(total int64, used int64) (state string) {
percent := (float64(used) / float64(total)) * 100

Expand Down
Loading

0 comments on commit aebad4f

Please sign in to comment.