Skip to content

Commit

Permalink
[spaces 2/2] /dav/spaces endpoint (#1803)
Browse files Browse the repository at this point in the history 
Co-authored-by: David Christofas <dchristofas@owncloud.com>
  • Loading branch information
@butonic
butonic and David Christofas authored Aug 12, 2021
1 parent 04a1cf2 commit 756bdce
Show file tree
Hide file tree
Showing 45 changed files with 1,168 additions and 153 deletions.
58 changes: 58 additions & 0 deletions .drone.star
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def main(ctx):
release(),
litmusOcisOldWebdav(),
litmusOcisNewWebdav(),
litmusOcisSpacesDav(),
] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)


Expand Down Expand Up @@ -517,6 +518,63 @@ def litmusOcisNewWebdav():
],
}

def litmusOcisSpacesDav():
return {
"kind": "pipeline",
"type": "docker",
"name": "litmus-owncloud-spaces-dav",
"platform": {
"os": "linux",
"arch": "amd64",
},
"trigger": {
"event": {
"include": [
"pull_request",
"tag",
],
},
},
"steps": [
makeStep("build-ci"),
{
"name": "revad-services",
"image": "registry.cern.ch/docker.io/library/golang:1.16",
"detach": True,
"commands": [
"cd /drone/src/tests/oc-integration-tests/drone/",
"/drone/src/cmd/revad/revad -c frontend.toml &",
"/drone/src/cmd/revad/revad -c gateway.toml &",
"/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
"/drone/src/cmd/revad/revad -c storage-oc-ocis.toml &",
"/drone/src/cmd/revad/revad -c users.toml",
]
},
{
"name": "sleep-for-revad-start",
"image": "registry.cern.ch/docker.io/library/golang:1.16",
"commands":[
"sleep 5",
],
},
{
"name": "litmus-owncloud-spaces-dav",
"image": "registry.cern.ch/docker.io/owncloud/litmus:latest",
"environment": {
"LITMUS_USERNAME": "einstein",
"LITMUS_PASSWORD": "relativity",
"TESTS": "basic http copymove props",
},
"commands": [
# The spaceid is randomly generated during the first login so we need this hack to construct the correct url.
"curl -s -k -u einstein:relativity -I http://revad-services:20080/remote.php/dav/files/einstein",
"export LITMUS_URL=http://revad-services:20080/remote.php/dav/spaces/123e4567-e89b-12d3-a456-426655440000!$(ls /drone/src/tmp/reva/data/spaces/personal/)",
"/usr/local/bin/litmus-wrapper",
]
},
],
}

def ocisIntegrationTests(parallelRuns, skipExceptParts = []):
pipelines = []
debugPartsEnabled = (len(skipExceptParts) != 0)
Expand Down
7 changes: 7 additions & 0 deletions changelog/unreleased/introduce-dav-spaces.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Enhancement: Introduce new webdav spaces endpoint

Clients can now use a new webdav endpoint `/dav/spaces/<storagespaceid>/relative/path/to/file` to directly access storage spaces.

The `<storagespaceid>` can be retrieved using the ListStorageSpaces CS3 api call.

https://github.com/cs3org/reva/pull/1803
3 changes: 0 additions & 3 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -279,11 +279,8 @@ github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39
github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs=
github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-plugin v1.4.2 h1:yFvG3ufXXpqiMiZx9HLcaK3XbIqQ1WJFR/F1a2CuVw0=
github.com/hashicorp/go-plugin v1.4.2/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
Expand Down
113 changes: 74 additions & 39 deletions internal/grpc/services/gateway/storageprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,16 +136,16 @@ func (s *svc) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSp

if id != nil {
// query that specific storage provider
parts := strings.SplitN(id.OpaqueId, "!", 2)
if len(parts) != 2 {
storageid, opaqeid, err := utils.SplitStorageSpaceID(id.OpaqueId)
if err != nil {
return &provider.ListStorageSpacesResponse{
Status: status.NewInvalidArg(ctx, "space id must be separated by !"),
}, nil
}
res, err := c.GetStorageProviders(ctx, &registry.GetStorageProvidersRequest{
Ref: &provider.Reference{ResourceId: &provider.ResourceId{
StorageId: parts[0], // FIXME REFERENCE the StorageSpaceId is a storageid + an opaqueid
OpaqueId: parts[1],
StorageId: storageid,
OpaqueId: opaqeid,
}},
})
if err != nil {
Expand Down Expand Up @@ -267,15 +267,15 @@ func (s *svc) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorag
func (s *svc) DeleteStorageSpace(ctx context.Context, req *provider.DeleteStorageSpaceRequest) (*provider.DeleteStorageSpaceResponse, error) {
log := appctx.GetLogger(ctx)
// TODO: needs to be fixed
parts := strings.SplitN(req.Id.OpaqueId, "!", 2)
if len(parts) != 2 {
storageid, opaqeid, err := utils.SplitStorageSpaceID(req.Id.OpaqueId)
if err != nil {
return &provider.DeleteStorageSpaceResponse{
Status: status.NewInvalidArg(ctx, "space id must be separated by !"),
}, nil
}
c, err := s.find(ctx, &provider.Reference{ResourceId: &provider.ResourceId{
StorageId: parts[0], // FIXME REFERENCE the StorageSpaceId is a storageid + a opaqueid
OpaqueId: parts[1],
StorageId: storageid,
OpaqueId: opaqeid,
}})
if err != nil {
return &provider.DeleteStorageSpaceResponse{
Expand Down Expand Up @@ -307,6 +307,11 @@ func (s *svc) getHome(_ context.Context) string {

func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) {
log := appctx.GetLogger(ctx)

if utils.IsRelativeReference(req.Ref) {
return s.initiateFileDownload(ctx, req)
}

p, st := s.getPath(ctx, req.Ref)
if st.Code != rpc.Code_CODE_OK {
return &gateway.InitiateFileDownloadResponse{
Expand Down Expand Up @@ -517,6 +522,9 @@ func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFi

func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*gateway.InitiateFileUploadResponse, error) {
log := appctx.GetLogger(ctx)
if utils.IsRelativeReference(req.Ref) {
return s.initiateFileUpload(ctx, req)
}
p, st := s.getPath(ctx, req.Ref)
if st.Code != rpc.Code_CODE_OK {
return &gateway.InitiateFileUploadResponse{
Expand Down Expand Up @@ -739,6 +747,11 @@ func (s *svc) GetPath(ctx context.Context, req *provider.GetPathRequest) (*provi

func (s *svc) CreateContainer(ctx context.Context, req *provider.CreateContainerRequest) (*provider.CreateContainerResponse, error) {
log := appctx.GetLogger(ctx)

if utils.IsRelativeReference(req.Ref) {
return s.createContainer(ctx, req)
}

p, st := s.getPath(ctx, req.Ref)
if st.Code != rpc.Code_CODE_OK {
return &provider.CreateContainerResponse{
Expand Down Expand Up @@ -1248,14 +1261,18 @@ func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.St
}

resPath := req.Ref.GetPath()
if len(providers) == 1 && (resPath == "" || strings.HasPrefix(resPath, providers[0].ProviderPath)) {
if len(providers) == 1 && (utils.IsRelativeReference(req.Ref) || resPath == "" || strings.HasPrefix(resPath, providers[0].ProviderPath)) {
c, err := s.getStorageProviderClient(ctx, providers[0])
if err != nil {
return &provider.StatResponse{
Status: status.NewInternal(ctx, err, "error connecting to storage provider="+providers[0].Address),
}, nil
}
return c.Stat(ctx, req)
rsp, err := c.Stat(ctx, req)
if err != nil || rsp.Status.Code != rpc.Code_CODE_OK {
return rsp, err
}
return rsp, nil
}

return s.statAcrossProviders(ctx, req, providers)
Expand Down Expand Up @@ -1308,14 +1325,18 @@ func (s *svc) statOnProvider(ctx context.Context, req *provider.StatRequest, res
return
}

resPath := path.Clean(req.Ref.GetPath())
newPath := req.Ref.GetPath()
if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
newPath = p.ProviderPath
if utils.IsAbsoluteReference(req.Ref) {
resPath := path.Clean(req.Ref.GetPath())
newPath := req.Ref.GetPath()
if resPath != "." && !strings.HasPrefix(resPath, p.ProviderPath) {
newPath = p.ProviderPath
}
req.Ref = &provider.Reference{Path: newPath}
}
r, err := c.Stat(ctx, &provider.StatRequest{Ref: &provider.Reference{Path: newPath}})

r, err := c.Stat(ctx, req)
if err != nil {
*e = errors.Wrap(err, fmt.Sprintf("gateway: error calling Stat %s on %+v", newPath, p))
*e = errors.Wrap(err, fmt.Sprintf("gateway: error calling Stat %s on %+v", req.Ref, p))
return
}
if res == nil {
Expand All @@ -1325,6 +1346,11 @@ func (s *svc) statOnProvider(ctx context.Context, req *provider.StatRequest, res
}

func (s *svc) Stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) {

if utils.IsRelativeReference(req.Ref) {
return s.stat(ctx, req)
}

p, st := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
if st.Code != rpc.Code_CODE_OK {
return &provider.StatResponse{
Expand Down Expand Up @@ -1639,6 +1665,7 @@ func (s *svc) listContainer(ctx context.Context, req *provider.ListContainerRequ
for _, inf := range infoFromProviders[i] {
if indirects[i] {
p := inf.Path
// TODO do we need to trim prefix here for relative references?
nestedInfos[p] = append(nestedInfos[p], inf)
} else {
infos = append(infos, inf)
Expand Down Expand Up @@ -1673,31 +1700,34 @@ func (s *svc) listContainerOnProvider(ctx context.Context, req *provider.ListCon
return
}

resPath := path.Clean(req.Ref.GetPath())
if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
// The path which we're supposed to list encompasses this provider
// so just return the first child and mark it as indirect
rel, err := filepath.Rel(resPath, p.ProviderPath)
if err != nil {
*e = err
return
}
parts := strings.Split(rel, "/")
p := path.Join(resPath, parts[0])
*ind = true
*res = []*provider.ResourceInfo{
{
Id: &provider.ResourceId{
StorageId: "/",
OpaqueId: uuid.New().String(),
if utils.IsAbsoluteReference(req.Ref) {
resPath := path.Clean(req.Ref.GetPath())
if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
// The path which we're supposed to list encompasses this provider
// so just return the first child and mark it as indirect
rel, err := filepath.Rel(resPath, p.ProviderPath)
if err != nil {
*e = err
return
}
parts := strings.Split(rel, "/")
p := path.Join(resPath, parts[0])
*ind = true
*res = []*provider.ResourceInfo{
{
Id: &provider.ResourceId{
StorageId: "/",
OpaqueId: uuid.New().String(),
},
Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER,
Path: p,
Size: 0,
},
Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER,
Path: p,
Size: 0,
},
}
return
}
return
}

r, err := c.ListContainer(ctx, req)
if err != nil {
*e = errors.Wrap(err, "gateway: error calling ListContainer")
Expand All @@ -1708,6 +1738,11 @@ func (s *svc) listContainerOnProvider(ctx context.Context, req *provider.ListCon

func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) {
log := appctx.GetLogger(ctx)

if utils.IsRelativeReference(req.Ref) {
return s.listContainer(ctx, req)
}

p, st := s.getPath(ctx, req.Ref, req.ArbitraryMetadataKeys...)
if st.Code != rpc.Code_CODE_OK {
return &provider.ListContainerResponse{
Expand Down Expand Up @@ -1896,7 +1931,7 @@ func (s *svc) getPath(ctx context.Context, ref *provider.Reference, keys ...stri
return res.Info.Path, res.Status
}

if ref.Path != "" {
if utils.IsAbsolutePathReference(ref) {
return ref.Path, &rpc.Status{Code: rpc.Code_CODE_OK}
}
return "", &rpc.Status{Code: rpc.Code_CODE_INTERNAL}
Expand Down
3 changes: 2 additions & 1 deletion internal/grpc/services/publicstorageprovider/publicstorageprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"github.com/cs3org/reva/pkg/rgrpc"
"github.com/cs3org/reva/pkg/rgrpc/status"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/utils"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
"go.opencensus.io/trace"
Expand Down Expand Up @@ -577,7 +578,7 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (token st
return "", "", errtypes.BadRequest("need absolute path ref: got " + ref.String())
}

if ref.GetPath() == "" {
if !utils.IsAbsolutePathReference(ref) {
// abort, no valid id nor path
return "", "", errtypes.BadRequest("invalid ref: " + ref.String())
}
Expand Down
Loading

0 comments on commit 756bdce

Please sign in to comment.