From 65014d1f189be7f821b8e5e5efe5b2d244bf444b Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Fri, 14 Jan 2022 10:59:45 +0545 Subject: [PATCH 01/17] [tests-only]Bump the commit id for tests (#2441) * Bump the commit id for tests * Adding failing tests to expected to failure --- .drone.env | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 2 ++ tests/acceptance/expected-failures-on-S3NG-storage.md | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.drone.env b/.drone.env index 86f199a58b..22bb1ee8a6 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=09d584745d6cbd6aebc557d9b78f6130e9b99e2b +CORE_COMMITID=88dc1fc01b8afd254d59cbface6ef48670848e22 CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index d3eed0341e..cf97c7a7b2 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1350,6 +1350,8 @@ _ocs: api compatibility, return correct status code_ - [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) - [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) - [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) +- [apiVersions/fileVersionAuthor.feature:180](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L180) +- [apiVersions/fileVersionAuthor.feature:220](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L220) ### [Allow public link sharing only for certain groups feature not implemented] - [apiSharePublicLink2/allowGroupToCreatePublicLinks.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiSharePublicLink2/allowGroupToCreatePublicLinks.feature#L35) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 6d8ee7cf79..4fc1c4446a 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -24,6 +24,8 @@ Basic file management like up and download, move, copy, properties, quota, trash - [apiVersions/fileVersionAuthor.feature:101](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L101) - [apiVersions/fileVersionAuthor.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L128) - [apiVersions/fileVersionAuthor.feature:155](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L155) +- [apiVersions/fileVersionAuthor.feature:180](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L180) +- [apiVersions/fileVersionAuthor.feature:220](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiVersions/fileVersionAuthor.feature#L220) #### [Getting information about a folder overwritten by a file gives 500 error instead of 404](https://github.com/owncloud/ocis/issues/1239) - [apiWebdavProperties1/copyFile.feature:226](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/copyFile.feature#L226) From 16a4df2051cbe2367e72636f1d2ea35fb938d46b Mon Sep 17 00:00:00 2001 From: Mouratidis Theofilos Date: Mon, 17 Jan 2022 12:19:11 +0100 Subject: [PATCH 02/17] CephFS Reva v0.2 (#1209) --- .dockerignore | 1 - .drone.star | 40 ++ Dockerfile.revad-ceph | 52 ++ Makefile | 12 +- changelog/unreleased/cephfs-driver.md | 3 + .../packages/storage/fs/cephfs/_index.md | 26 + examples/ceph/ceph.conf | 3 + examples/ceph/keyring | 2 + go.mod | 4 + go.sum | 12 + pkg/storage/fs/cephfs/cephfs.go | 591 ++++++++++++++++++ pkg/storage/fs/cephfs/chunking.go | 344 ++++++++++ pkg/storage/fs/cephfs/connections.go | 315 ++++++++++ pkg/storage/fs/cephfs/errors.go | 64 ++ pkg/storage/fs/cephfs/options.go | 90 +++ pkg/storage/fs/cephfs/permissions.go | 325 ++++++++++ pkg/storage/fs/cephfs/unsupported.go | 39 ++ pkg/storage/fs/cephfs/upload.go | 413 ++++++++++++ pkg/storage/fs/cephfs/user.go | 242 +++++++ pkg/storage/fs/cephfs/utils.go | 245 ++++++++ pkg/storage/fs/loader/loader.go | 1 + 21 files changed, 2819 insertions(+), 5 deletions(-) create mode 100644 Dockerfile.revad-ceph create mode 100644 changelog/unreleased/cephfs-driver.md create mode 100644 docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md create mode 100644 examples/ceph/ceph.conf create mode 100644 examples/ceph/keyring create mode 100644 pkg/storage/fs/cephfs/cephfs.go create mode 100644 pkg/storage/fs/cephfs/chunking.go create mode 100644 pkg/storage/fs/cephfs/connections.go create mode 100644 pkg/storage/fs/cephfs/errors.go create mode 100644 pkg/storage/fs/cephfs/options.go create mode 100644 pkg/storage/fs/cephfs/permissions.go create mode 100644 pkg/storage/fs/cephfs/unsupported.go create mode 100644 pkg/storage/fs/cephfs/upload.go create mode 100644 pkg/storage/fs/cephfs/user.go create mode 100644 pkg/storage/fs/cephfs/utils.go diff --git a/.dockerignore b/.dockerignore index 429a7e9d46..ab22e55d14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ .github changelog docs -examples grpc-tests tests tools diff --git a/.drone.star b/.drone.star index 3c60499d7d..92460ad3e3 100644 --- a/.drone.star +++ b/.drone.star @@ -229,6 +229,26 @@ def buildAndPublishDocker(): ], }, }, + { + "name": "publish-docker-revad-ceph-latest", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "latest-ceph", + "dockerfile": "Dockerfile.revad-ceph", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, ], } @@ -481,6 +501,26 @@ def release(): ], }, }, + { + "name": "docker-revad-ceph-tag", + "pull": "always", + "image": "plugins/docker", + "settings": { + "repo": "cs3org/revad", + "tags": "${DRONE_TAG}-ceph", + "dockerfile": "Dockerfile.revad-ceph", + "username": { + "from_secret": "dockerhub_username", + }, + "password": { + "from_secret": "dockerhub_password", + }, + "custom_dns": [ + "128.142.17.5", + "128.142.16.5", + ], + }, + }, ], "depends_on": ["changelog"], } diff --git a/Dockerfile.revad-ceph b/Dockerfile.revad-ceph new file mode 100644 index 0000000000..186acfcc63 --- /dev/null +++ b/Dockerfile.revad-ceph @@ -0,0 +1,52 @@ +# Copyright 2018-2021 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +FROM ceph/daemon-base + +RUN dnf update -y && dnf install -y \ + git \ + gcc \ + make \ + libcephfs-devel \ + librbd-devel \ + librados-devel + +ADD https://golang.org/dl/go1.16.4.linux-amd64.tar.gz \ + go1.16.4.linux-amd64.tar.gz + +RUN rm -rf /usr/local/go && \ + tar -C /usr/local -xzf go1.16.4.linux-amd64.tar.gz && \ + rm go1.16.4.linux-amd64.tar.gz + +ENV PATH /go/bin:/usr/local/go/bin:$PATH +ENV GOPATH /go + +WORKDIR /go/src/github/cs3org/reva +COPY . . +RUN mkdir -p /go/bin \ + make build-revad-cephfs-docker && \ + cp /go/src/github/cs3org/reva/cmd/revad/revad /usr/bin/revad + +RUN cp -r examples/ceph /etc/ + +RUN mkdir -p /etc/revad/ && echo "" > /etc/revad/revad.toml + +EXPOSE 9999 10000 + +ENTRYPOINT [ "/usr/bin/revad" ] +CMD [ "-c", "/etc/revad/revad.toml", "-p", "/var/run/revad.pid" ] diff --git a/Makefile b/Makefile index 465a71ef9b..b686fca1d9 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,9 @@ off: imports: off `go env GOPATH`/bin/goimports -w tools pkg internal cmd -build: imports test-go-version - gofmt -s -w . - go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad - go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva +build: build-revad build-reva test-go-version + +build-cephfs: build-revad-cephfs build-reva tidy: go mod tidy @@ -41,6 +40,9 @@ tidy: build-revad: imports go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +build-revad-cephfs: imports + go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad + build-reva: imports go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva @@ -104,6 +106,8 @@ ci: build-ci test lint-ci # to be run in Docker build build-revad-docker: off go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad +build-revad-cephfs-docker: off + go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad build-reva-docker: off go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva clean: diff --git a/changelog/unreleased/cephfs-driver.md b/changelog/unreleased/cephfs-driver.md new file mode 100644 index 0000000000..9015c61538 --- /dev/null +++ b/changelog/unreleased/cephfs-driver.md @@ -0,0 +1,3 @@ +Enhancement: Reva CephFS module v0.2.1 + +https://github.com/cs3org/reva/pull/1209 \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md b/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md new file mode 100644 index 0000000000..48dc231ebf --- /dev/null +++ b/docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md @@ -0,0 +1,26 @@ +--- +title: "cephfs" +linkTitle: "cephfs" +weight: 10 +description: > + Configuration for the cephfs service +--- + +# _struct: config_ + +{{% dir name="root" type="string" default="/var/tmp/reva/" %}} +Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L34) +{{< highlight toml >}} +[storage.fs.cephfs] +root = "/var/tmp/reva/" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="share_folder" type="string" default="/MyShares" %}} +Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L35) +{{< highlight toml >}} +[storage.fs.cephfs] +share_folder = "/MyShares" +{{< /highlight >}} +{{% /dir %}} + diff --git a/examples/ceph/ceph.conf b/examples/ceph/ceph.conf new file mode 100644 index 0000000000..f2daaa8de5 --- /dev/null +++ b/examples/ceph/ceph.conf @@ -0,0 +1,3 @@ +[global] + fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323' + mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]' diff --git a/examples/ceph/keyring b/examples/ceph/keyring new file mode 100644 index 0000000000..9e555cc1b2 --- /dev/null +++ b/examples/ceph/keyring @@ -0,0 +1,2 @@ +[client.admin] + key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA==' diff --git a/go.mod b/go.mod index abb2defa94..367e062000 100644 --- a/go.mod +++ b/go.mod @@ -13,11 +13,13 @@ require ( github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 + github.com/ceph/go-ceph v0.12.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 + github.com/dgraph-io/ristretto v0.1.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 github.com/gdexlab/go-render v1.0.1 github.com/go-chi/chi/v5 v5.0.7 @@ -40,6 +42,7 @@ require ( github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.20 github.com/mitchellh/copystructure v1.2.0 // indirect @@ -71,6 +74,7 @@ require ( go.opentelemetry.io/otel/trace v1.3.0 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 golang.org/x/term v0.0.0-20210916214954-140adaaadfaf google.golang.org/genproto v0.0.0-20211021150943-2b146023228c diff --git a/go.sum b/go.sum index b2612f2e5c..0f992f1d5a 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= @@ -118,6 +119,8 @@ github.com/c-bata/go-prompt v0.2.5 h1:3zg6PecEywxNn0xiqcXHD96fkbxghD+gdB2tbsYfl+ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/ceph/go-ceph v0.12.0 h1:nlFgKQZXOFR4oMnzXsKwTr79Y6EYDwqTrpigICGy/Tw= +github.com/ceph/go-ceph v0.12.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -140,6 +143,10 @@ github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4a github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -301,6 +308,8 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -510,6 +519,8 @@ github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b h1:Q53idHrTuQDDHyXaxZ6pUl0I9uyD6Z6uKFK3ocX6LzI= +github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b/go.mod h1:KirJrATYGbTyUwVR26xIkaipRqRcMRXBf8N5dacvGus= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9uz/w= github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= @@ -910,6 +921,7 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go new file mode 100644 index 0000000000..e71db2df75 --- /dev/null +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -0,0 +1,591 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +const ( + xattrTrustedNs = "trusted." + xattrEID = xattrTrustedNs + "eid" + xattrMd5 = xattrTrustedNs + "checksum" + xattrMd5ts = xattrTrustedNs + "checksumTS" + xattrRef = xattrTrustedNs + "ref" + xattrUserNs = "user." + snap = ".snap" +) + +type cephfs struct { + conf *Options + conn *connections + adminConn *adminConn + chunkHandler *ChunkHandler +} + +func init() { + registry.Register("cephfs", New) +} + +// New returns an implementation to of the storage.FS interface that talk to +// a ceph filesystem. +func New(m map[string]interface{}) (fs storage.FS, err error) { + c := &Options{} + if err = mapstructure.Decode(m, c); err != nil { + return nil, errors.Wrap(err, "error decoding conf") + } + + c.fillDefaults() + + var cache *connections + if cache, err = newCache(); err != nil { + return nil, errors.New("cephfs: can't create caches") + } + + adminConn := newAdminConn(c.IndexPool) + if adminConn == nil { + return nil, errors.Wrap(err, "cephfs: Couldn't create admin connections") + } + + for _, dir := range []string{c.ShadowFolder, c.UploadFolder} { + err = adminConn.adminMount.MakeDir(dir, dirPermFull) + if err != nil && err.Error() != errFileExists { + return nil, errors.New("cephfs: can't initialise system dir " + dir + ":" + err.Error()) + } + } + + return &cephfs{ + conf: c, + conn: cache, + adminConn: adminConn, + }, nil +} + +func (fs *cephfs) GetHome(ctx context.Context) (string, error) { + if fs.conf.DisableHome { + return "", errtypes.NotSupported("cephfs: GetHome() home supported disabled") + } + + user := fs.makeUser(ctx) + + return user.home, nil +} + +func (fs *cephfs) CreateHome(ctx context.Context) (err error) { + if fs.conf.DisableHome { + return errtypes.NotSupported("cephfs: GetHome() home supported disabled") + } + + user := fs.makeUser(ctx) + + // Stop createhome from running the whole thing because it is called multiple times + if _, err = fs.adminConn.adminMount.Statx(user.home, cephfs2.StatxMode, 0); err == nil { + return + } + + err = walkPath(user.home, func(path string) error { + return fs.adminConn.adminMount.MakeDir(path, dirPermDefault) + }, false) + if err != nil { + return getRevaError(err) + } + + err = fs.adminConn.adminMount.Chown(user.home, uint32(user.UidNumber), uint32(user.GidNumber)) + if err != nil { + return getRevaError(err) + } + + err = fs.adminConn.adminMount.SetXattr(user.home, "ceph.quota.max_bytes", []byte(fmt.Sprint(fs.conf.UserQuotaBytes)), 0) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + err = cv.mount.MakeDir(removeLeadingSlash(fs.conf.ShareFolder), dirPermDefault) + if err != nil && err.Error() == errFileExists { + err = nil + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) CreateDir(ctx context.Context, ref *provider.Reference) error { + user := fs.makeUser(ctx) + path, err := user.resolveRef(ref) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.MakeDir(path, dirPermDefault); err != nil { + return + } + + //TODO(tmourati): Add entry id logic + }) + + return getRevaError(err) +} + +func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { + var path string + user := fs.makeUser(ctx) + path, err = user.resolveRef(ref) + if err != nil { + return err + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.Unlink(path); err != nil && err.Error() == errIsADirectory { + err = cv.mount.RemoveDir(path) + } + + //TODO(tmourati): Add entry id logic + }) + + //has already been deleted by direct mount + if err != nil && err.Error() == errNotFound { + return nil + } + + return getRevaError(err) +} + +func (fs *cephfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { + var oldPath, newPath string + user := fs.makeUser(ctx) + if oldPath, err = user.resolveRef(oldRef); err != nil { + return + } + if newPath, err = user.resolveRef(newRef); err != nil { + return + } + + user.op(func(cv *cacheVal) { + if err = cv.mount.Rename(oldPath, newPath); err != nil { + return + } + + //TODO(tmourati): Add entry id logic, handle already moved file error + }) + + // has already been moved by direct mount + if err != nil && err.Error() == errNotFound { + return nil + } + + return getRevaError(err) +} + +func (fs *cephfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var path string + user := fs.makeUser(ctx) + + if path, err = user.resolveRef(ref); err != nil { + return nil, err + } + + user.op(func(cv *cacheVal) { + var stat Statx + if stat, err = cv.mount.Statx(path, cephfs2.StatxBasicStats, 0); err != nil { + return + } + ri, err = user.fileAsResourceInfo(cv, path, stat, mdKeys) + }) + + return ri, getRevaError(err) +} + +func (fs *cephfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (files []*provider.ResourceInfo, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + var dir *cephfs2.Directory + if dir, err = cv.mount.OpenDir(path); err != nil { + return + } + defer closeDir(dir) + + var entry *cephfs2.DirEntryPlus + var ri *provider.ResourceInfo + for entry, err = dir.ReadDirPlus(cephfs2.StatxBasicStats, 0); entry != nil && err == nil; entry, err = dir.ReadDirPlus(cephfs2.StatxBasicStats, 0) { + if fs.conf.HiddenDirs[entry.Name()] { + continue + } + + ri, err = user.fileAsResourceInfo(cv, filepath.Join(path, entry.Name()), entry.Statx(), mdKeys) + if ri == nil || err != nil { + if err != nil { + log := appctx.GetLogger(ctx) + log.Err(err).Msg("cephfs: error in file as resource info") + } + err = nil + continue + } + + files = append(files, ri) + } + }) + + return files, getRevaError(err) +} + +func (fs *cephfs) Download(ctx context.Context, ref *provider.Reference) (rc io.ReadCloser, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + if strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { + err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") + return + } + rc, err = cv.mount.Open(path, os.O_RDONLY, 0) + }) + + return rc, getRevaError(err) +} + +func (fs *cephfs) ListRevisions(ctx context.Context, ref *provider.Reference) (fvs []*provider.FileVersion, err error) { + //TODO(tmourati): Fix entry id logic + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + if strings.HasPrefix(path, removeLeadingSlash(fs.conf.ShareFolder)) { + err = errtypes.PermissionDenied("cephfs: cannot download under the virtual share folder") + return + } + var dir *cephfs2.Directory + if dir, err = cv.mount.OpenDir(".snap"); err != nil { + return + } + defer closeDir(dir) + + for d, _ := dir.ReadDir(); d != nil; d, _ = dir.ReadDir() { + var revPath string + var stat Statx + var e error + + if strings.HasPrefix(d.Name(), ".") { + continue + } + + revPath, e = resolveRevRef(cv.mount, ref, d.Name()) + if e != nil { + continue + } + stat, e = cv.mount.Statx(revPath, cephfs2.StatxMtime|cephfs2.StatxSize, 0) + if e != nil { + continue + } + fvs = append(fvs, &provider.FileVersion{ + Key: d.Name(), + Size: stat.Size, + Mtime: uint64(stat.Mtime.Sec), + }) + } + }) + + return fvs, getRevaError(err) +} + +func (fs *cephfs) DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (file io.ReadCloser, err error) { + //TODO(tmourati): Fix entry id logic + user := fs.makeUser(ctx) + + user.op(func(cv *cacheVal) { + var revPath string + revPath, err = resolveRevRef(cv.mount, ref, key) + if err != nil { + return + } + + file, err = cv.mount.Open(revPath, os.O_RDONLY, 0) + }) + + return file, getRevaError(err) +} + +func (fs *cephfs) RestoreRevision(ctx context.Context, ref *provider.Reference, key string) (err error) { + //TODO(tmourati): Fix entry id logic + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return errors.Wrap(err, "cephfs: error resolving ref") + } + + user.op(func(cv *cacheVal) { + var revPath string + if revPath, err = resolveRevRef(cv.mount, ref, key); err != nil { + err = errors.Wrap(err, "cephfs: error resolving revision ref "+ref.String()) + return + } + + var src, dst *cephfs2.File + if src, err = cv.mount.Open(revPath, os.O_RDONLY, 0); err != nil { + return + } + defer closeFile(src) + + if dst, err = cv.mount.Open(path, os.O_WRONLY|os.O_TRUNC, 0); err != nil { + return + } + defer closeFile(dst) + + _, err = io.Copy(dst, src) + }) + + return getRevaError(err) +} + +func (fs *cephfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (str string, err error) { + //TODO(tmourati): Add entry id logic + return "", errtypes.NotSupported("cephfs: entry IDs currently not supported") +} + +func (fs *cephfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, removeGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + err = fs.changePerms(ctx, cv.mount, g, path, updateGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + grant := &provider.Grant{Grantee: g} //nil perms will remove the whole grant + err = fs.changePerms(ctx, cv.mount, grant, path, removeGrant) + }) + + return getRevaError(err) +} + +func (fs *cephfs) ListGrants(ctx context.Context, ref *provider.Reference) (glist []*provider.Grant, err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return + } + + user.op(func(cv *cacheVal) { + glist = fs.getFullPermissionSet(ctx, cv.mount, path) + + if glist == nil { + err = errors.New("cephfs: error listing grants on " + path) + } + }) + + return glist, getRevaError(err) +} + +func (fs *cephfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, used uint64, err error) { + user := fs.makeUser(ctx) + + log := appctx.GetLogger(ctx) + user.op(func(cv *cacheVal) { + var buf []byte + buf, err = cv.mount.GetXattr(".", "ceph.quota.max_bytes") + if err != nil { + log.Warn().Msg("cephfs: user quota bytes not set") + total = fs.conf.UserQuotaBytes + } else { + total, _ = strconv.ParseUint(string(buf), 10, 64) + } + + buf, err = cv.mount.GetXattr(".", "ceph.dir.rbytes") + if err == nil { + used, err = strconv.ParseUint(string(buf), 10, 64) + } + }) + + return total, used, getRevaError(err) +} + +func (fs *cephfs) CreateReference(ctx context.Context, path string, targetURI *url.URL) (err error) { + user := fs.makeUser(ctx) + + user.op(func(cv *cacheVal) { + if !strings.HasPrefix(strings.TrimPrefix(path, user.home), fs.conf.ShareFolder) { + err = errors.New("cephfs: can't create reference outside a share folder") + } else { + err = cv.mount.MakeDir(path, dirPermDefault) + } + }) + if err != nil { + return getRevaError(err) + } + + user.op(func(cv *cacheVal) { + err = cv.mount.SetXattr(path, xattrRef, []byte(targetURI.String()), 0) + }) + + return getRevaError(err) +} + +func (fs *cephfs) Shutdown(ctx context.Context) (err error) { + ctx.Done() + fs.conn.clearCache() + _ = fs.adminConn.adminMount.Unmount() + _ = fs.adminConn.adminMount.Release() + fs.adminConn.radosConn.Shutdown() + + return +} + +func (fs *cephfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return err + } + + user.op(func(cv *cacheVal) { + for k, v := range md.Metadata { + if !strings.HasPrefix(k, xattrUserNs) { + k = xattrUserNs + k + } + if e := cv.mount.SetXattr(path, k, []byte(v), 0); e != nil { + err = errors.Wrap(err, e.Error()) + return + } + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { + var path string + user := fs.makeUser(ctx) + if path, err = user.resolveRef(ref); err != nil { + return err + } + + user.op(func(cv *cacheVal) { + for _, key := range keys { + if !strings.HasPrefix(key, xattrUserNs) { + key = xattrUserNs + key + } + if e := cv.mount.RemoveXattr(path, key); e != nil { + err = errors.Wrap(err, e.Error()) + return + } + } + }) + + return getRevaError(err) +} + +func (fs *cephfs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("cephfs: empty recycle not supported") +} + +func (fs *cephfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (r *provider.CreateStorageSpaceResponse, err error) { + return nil, errors.New("cephfs: createStorageSpace not supported") +} + +func (fs *cephfs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) { + panic("implement me") +} + +func (fs *cephfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { + return errors.New("cephfs: restoreRecycleItem not supported") +} + +func (fs *cephfs) PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error { + return errors.New("cephfs: purgeRecycleItem not supported") +} + +func (fs *cephfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) { + return nil, errors.New("cephfs: listStorageSpaces not supported") +} + +func (fs *cephfs) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { + return nil, errors.New("cephfs: updateStorageSpace not supported") +} diff --git a/pkg/storage/fs/cephfs/chunking.go b/pkg/storage/fs/cephfs/chunking.go new file mode 100644 index 0000000000..bb4e48fe2c --- /dev/null +++ b/pkg/storage/fs/cephfs/chunking.go @@ -0,0 +1,344 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + "github.com/google/uuid" +) + +// IsChunked checks if a given path refers to a chunk or not +func IsChunked(fn string) (bool, error) { + // FIXME: also need to check whether the OC-Chunked header is set + return regexp.MatchString(`-chunking-\w+-[0-9]+-[0-9]+$`, fn) +} + +// ChunkBLOBInfo stores info about a particular chunk +type ChunkBLOBInfo struct { + Path string + TransferID string + TotalChunks int + CurrentChunk int +} + +// Not using the resource path in the chunk folder name allows uploading to +// the same folder after a move without having to restart the chunk upload +func (c *ChunkBLOBInfo) uploadID() string { + return fmt.Sprintf("chunking-%s-%d", c.TransferID, c.TotalChunks) +} + +// GetChunkBLOBInfo decodes a chunk name to retrieve info about it. +func GetChunkBLOBInfo(path string) (*ChunkBLOBInfo, error) { + parts := strings.Split(path, "-chunking-") + tail := strings.Split(parts[1], "-") + + totalChunks, err := strconv.Atoi(tail[1]) + if err != nil { + return nil, err + } + + currentChunk, err := strconv.Atoi(tail[2]) + if err != nil { + return nil, err + } + if currentChunk >= totalChunks { + return nil, fmt.Errorf("current chunk:%d exceeds total number of chunks:%d", currentChunk, totalChunks) + } + + return &ChunkBLOBInfo{ + Path: parts[0], + TransferID: tail[0], + TotalChunks: totalChunks, + CurrentChunk: currentChunk, + }, nil +} + +// ChunkHandler manages chunked uploads, storing the chunks in a temporary directory +// until it gets the final chunk which is then returned. +type ChunkHandler struct { + user *User + chunkFolder string +} + +// NewChunkHandler creates a handler for chunked uploads. +func NewChunkHandler(ctx context.Context, fs *cephfs) *ChunkHandler { + return &ChunkHandler{fs.makeUser(ctx), fs.conf.UploadFolder} +} + +func (c *ChunkHandler) getChunkTempFileName() string { + return fmt.Sprintf("__%d_%s", time.Now().Unix(), uuid.New().String()) +} + +func (c *ChunkHandler) getChunkFolderName(i *ChunkBLOBInfo) (path string, err error) { + path = filepath.Join(c.chunkFolder, i.uploadID()) + c.user.op(func(cv *cacheVal) { + err = cv.mount.MakeDir(path, 0777) + }) + + return +} + +func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (finish bool, chunk string, err error) { + var chunkInfo *ChunkBLOBInfo + + chunkInfo, err = GetChunkBLOBInfo(path) + if err != nil { + err = fmt.Errorf("error getting chunk info from path: %s", path) + return + } + + chunkTempFilename := c.getChunkTempFileName() + c.user.op(func(cv *cacheVal) { + var tmpFile *cephfs2.File + target := filepath.Join(c.chunkFolder, chunkTempFilename) + tmpFile, err = cv.mount.Open(target, os.O_CREATE|os.O_WRONLY, filePermDefault) + defer closeFile(tmpFile) + if err != nil { + return + } + _, err = io.Copy(tmpFile, r) + }) + if err != nil { + return + } + + chunksFolderName, err := c.getChunkFolderName(chunkInfo) + if err != nil { + return + } + // c.logger.Info().Log("chunkfolder", chunksFolderName) + + chunkTarget := filepath.Join(chunksFolderName, strconv.Itoa(chunkInfo.CurrentChunk)) + c.user.op(func(cv *cacheVal) { + err = cv.mount.Rename(chunkTempFilename, chunkTarget) + }) + if err != nil { + return + } + + // Check that all chunks are uploaded. + // This is very inefficient, the server has to check that it has all the + // chunks after each uploaded chunk. + // A two-phase upload like DropBox is better, because the server will + // assembly the chunks when the client asks for it. + numEntries := 0 + c.user.op(func(cv *cacheVal) { + var dir *cephfs2.Directory + var entry *cephfs2.DirEntry + var chunkFile, assembledFile *cephfs2.File + + dir, err = cv.mount.OpenDir(chunksFolderName) + defer closeDir(dir) + + for entry, err = dir.ReadDir(); entry != nil && err == nil; entry, err = dir.ReadDir() { + numEntries++ + } + // to remove . and .. + numEntries -= 2 + + if err != nil || numEntries < chunkInfo.TotalChunks { + return + } + + chunk = filepath.Join(c.chunkFolder, c.getChunkTempFileName()) + assembledFile, err = cv.mount.Open(chunk, os.O_CREATE|os.O_WRONLY, filePermDefault) + defer closeFile(assembledFile) + defer deleteFile(cv.mount, chunk) + if err != nil { + return + } + + for i := 0; i < numEntries; i++ { + target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + + chunkFile, err = cv.mount.Open(target, os.O_RDONLY, 0) + if err != nil { + return + } + _, err = io.Copy(assembledFile, chunkFile) + closeFile(chunkFile) + if err != nil { + return + } + } + + // necessary approach in case assembly fails + for i := 0; i < numEntries; i++ { + target := filepath.Join(chunksFolderName, strconv.Itoa(i)) + err = cv.mount.Unlink(target) + if err != nil { + return + } + } + _ = cv.mount.Unlink(chunksFolderName) + }) + + return true, chunk, nil +} + +// WriteChunk saves an intermediate chunk temporarily and assembles all chunks +// once the final one is received. +func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { + finish, chunk, err := c.saveChunk(fn, r) + if err != nil { + return "", "", err + } + + if !finish { + return "", "", nil + } + + chunkInfo, err := GetChunkBLOBInfo(fn) + if err != nil { + return "", "", err + } + + return chunkInfo.Path, chunk, nil + + // TODO(labkode): implement old chunking + + /* + req2 := &provider.StartWriteSessionRequest{} + res2, err := client.StartWriteSession(ctx, req2) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res2.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res2.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + sessID := res2.SessionId + logger.Build().Str("sessID", sessID).Msg(ctx, "got write session id") + + stream, err := client.Write(ctx) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + buffer := make([]byte, 1024*1024*3) + var offset uint64 + var numChunks uint64 + + for { + n, err := fd.Read(buffer) + if n > 0 { + req := &provider.WriteRequest{Data: buffer, Length: uint64(n), SessionId: sessID, Offset: offset} + err = stream.Send(req) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + numChunks++ + offset += uint64(n) + } + + if err == io.EOF { + break + } + + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + } + + res3, err := stream.CloseAndRecv() + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res3.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + req4 := &provider.FinishWriteSessionRequest{Filename: chunkInfo.path, SessionId: sessID} + res4, err := client.FinishWriteSession(ctx, req4) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res4.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res4.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + req.Filename = chunkInfo.path + res, err = client.Stat(ctx, req) + if err != nil { + logger.Error(ctx, err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if res.Status.Code != rpc.Code_CODE_OK { + logger.Println(ctx, res.Status) + w.WriteHeader(http.StatusInternalServerError) + return + } + + md2 := res.Metadata + + w.Header().Add("Content-Type", md2.Mime) + w.Header().Set("ETag", md2.Etag) + w.Header().Set("OC-FileId", md2.Id) + w.Header().Set("OC-ETag", md2.Etag) + t := time.Unix(int64(md2.Mtime), 0) + lastModifiedString := t.Format(time.RFC1123Z) + w.Header().Set("Last-Modified", lastModifiedString) + w.Header().Set("X-OC-MTime", "accepted") + + if md == nil { + w.WriteHeader(http.StatusCreated) + return + } + + w.WriteHeader(http.StatusNoContent) + return + */ +} diff --git a/pkg/storage/fs/cephfs/connections.go b/pkg/storage/fs/cephfs/connections.go new file mode 100644 index 0000000000..7b928eaa63 --- /dev/null +++ b/pkg/storage/fs/cephfs/connections.go @@ -0,0 +1,315 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "time" + + "github.com/ceph/go-ceph/cephfs/admin" + rados2 "github.com/ceph/go-ceph/rados" + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/pkg/errors" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + "github.com/dgraph-io/ristretto" + "golang.org/x/sync/semaphore" +) + +type cacheVal struct { + perm *cephfs2.UserPerm + mount *cephfs2.MountInfo +} + +//TODO: Add to cephfs obj + +type connections struct { + cache *ristretto.Cache + lock *semaphore.Weighted + ctx context.Context + userCache *ristretto.Cache + groupCache *ristretto.Cache +} + +//TODO: make configurable/add to options +var usrLimit int64 = 1e4 + +func newCache() (c *connections, err error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: usrLimit, + BufferItems: 64, + OnEvict: func(item *ristretto.Item) { + v := item.Value.(cacheVal) + v.perm.Destroy() + _ = v.mount.Unmount() + _ = v.mount.Release() + }, + }) + if err != nil { + return + } + + ucache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 10 * usrLimit, + BufferItems: 64, + }) + if err != nil { + return + } + + gcache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, + MaxCost: 10 * usrLimit, + BufferItems: 64, + }) + if err != nil { + return + } + + c = &connections{ + cache: cache, + lock: semaphore.NewWeighted(usrLimit), + ctx: context.Background(), + userCache: ucache, + groupCache: gcache, + } + + return +} + +func (c *connections) clearCache() { + c.cache.Clear() + c.cache.Close() +} + +type adminConn struct { + indexPoolName string + subvolAdmin *admin.FSAdmin + adminMount Mount + radosConn *rados2.Conn + radosIO *rados2.IOContext +} + +func newAdminConn(poolName string) *adminConn { + rados, err := rados2.NewConn() + if err != nil { + return nil + } + if err = rados.ReadDefaultConfigFile(); err != nil { + return nil + } + + if err = rados.Connect(); err != nil { + return nil + } + + pools, err := rados.ListPools() + if err != nil { + rados.Shutdown() + return nil + } + + var radosIO *rados2.IOContext + if in(poolName, pools) { + radosIO, err = rados.OpenIOContext(poolName) + if err != nil { + rados.Shutdown() + return nil + } + } else { + err = rados.MakePool(poolName) + if err != nil { + rados.Shutdown() + return nil + } + radosIO, err = rados.OpenIOContext(poolName) + if err != nil { + rados.Shutdown() + return nil + } + } + + mount, err := cephfs2.CreateFromRados(rados) + if err != nil { + rados.Shutdown() + return nil + } + + if err = mount.Mount(); err != nil { + rados.Shutdown() + destroyCephConn(mount, nil) + return nil + } + + return &adminConn{ + poolName, + admin.NewFromConn(rados), + mount, + rados, + radosIO, + } +} + +func newConn(user *User) *cacheVal { + var perm *cephfs2.UserPerm + mount, err := cephfs2.CreateMount() + if err != nil { + return destroyCephConn(mount, perm) + } + if err = mount.ReadDefaultConfigFile(); err != nil { + return destroyCephConn(mount, perm) + } + if err = mount.Init(); err != nil { + return destroyCephConn(mount, perm) + } + + if user != nil { //nil creates admin conn + perm = cephfs2.NewUserPerm(int(user.UidNumber), int(user.GidNumber), []int{}) + if err = mount.SetMountPerms(perm); err != nil { + return destroyCephConn(mount, perm) + } + } + + if err = mount.MountWithRoot("/"); err != nil { + return destroyCephConn(mount, perm) + } + + if user != nil { + if err = mount.ChangeDir(user.fs.conf.Root); err != nil { + return destroyCephConn(mount, perm) + } + } + + return &cacheVal{ + perm: perm, + mount: mount, + } +} + +func (fs *cephfs) getUserByID(ctx context.Context, uid string) (*userpb.User, error) { + if entity, found := fs.conn.userCache.Get(uid); found { + return entity.(*userpb.User), nil + } + + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getUserResp, err := client.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{ + Claim: "uid", + Value: uid, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get user failed") + } + fs.conn.userCache.SetWithTTL(uid, getUserResp.User, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(getUserResp.User.Id.OpaqueId, getUserResp.User, 1, 24*time.Hour) + + return getUserResp.User, nil +} + +func (fs *cephfs) getUserByOpaqueID(ctx context.Context, oid string) (*userpb.User, error) { + if entity, found := fs.conn.userCache.Get(oid); found { + return entity.(*userpb.User), nil + } + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getUserResp, err := client.GetUser(ctx, &userpb.GetUserRequest{ + UserId: &userpb.UserId{ + OpaqueId: oid, + }, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting user") + } + if getUserResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get user failed") + } + fs.conn.userCache.SetWithTTL(fmt.Sprint(getUserResp.User.UidNumber), getUserResp.User, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(oid, getUserResp.User, 1, 24*time.Hour) + + return getUserResp.User, nil +} + +func (fs *cephfs) getGroupByID(ctx context.Context, gid string) (*grouppb.Group, error) { + if entity, found := fs.conn.groupCache.Get(gid); found { + return entity.(*grouppb.Group), nil + } + + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getGroupResp, err := client.GetGroupByClaim(ctx, &grouppb.GetGroupByClaimRequest{ + Claim: "gid", + Value: gid, + }) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting group") + } + if getGroupResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get group failed") + } + fs.conn.groupCache.SetWithTTL(gid, getGroupResp.Group, 1, 24*time.Hour) + fs.conn.groupCache.SetWithTTL(getGroupResp.Group.Id.OpaqueId, getGroupResp.Group, 1, 24*time.Hour) + + return getGroupResp.Group, nil +} + +func (fs *cephfs) getGroupByOpaqueID(ctx context.Context, oid string) (*grouppb.Group, error) { + if entity, found := fs.conn.groupCache.Get(oid); found { + return entity.(*grouppb.Group), nil + } + client, err := pool.GetGatewayServiceClient(fs.conf.GatewaySvc) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting gateway grpc client") + } + getGroupResp, err := client.GetGroup(ctx, &grouppb.GetGroupRequest{ + GroupId: &grouppb.GroupId{ + OpaqueId: oid, + }, + }) + + if err != nil { + return nil, errors.Wrap(err, "cephfs: error getting group") + } + if getGroupResp.Status.Code != rpc.Code_CODE_OK { + return nil, errors.Wrap(err, "cephfs: grpc get group failed") + } + fs.conn.userCache.SetWithTTL(fmt.Sprint(getGroupResp.Group.GidNumber), getGroupResp.Group, 1, 24*time.Hour) + fs.conn.userCache.SetWithTTL(oid, getGroupResp.Group, 1, 24*time.Hour) + + return getGroupResp.Group, nil +} diff --git a/pkg/storage/fs/cephfs/errors.go b/pkg/storage/fs/cephfs/errors.go new file mode 100644 index 0000000000..a4ab013c97 --- /dev/null +++ b/pkg/storage/fs/cephfs/errors.go @@ -0,0 +1,64 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +/* + #include + #include + #include +*/ +import "C" +import ( + "fmt" + + "github.com/cs3org/reva/pkg/errtypes" +) + +func wrapErrorMsg(code C.int) string { + return fmt.Sprintf("cephfs: ret=-%d, %s", code, C.GoString(C.strerror(code))) +} + +var ( + errNotFound = wrapErrorMsg(C.ENOENT) + errFileExists = wrapErrorMsg(C.EEXIST) + errNoSpaceLeft = wrapErrorMsg(C.ENOSPC) + errIsADirectory = wrapErrorMsg(C.EISDIR) + errPermissionDenied = wrapErrorMsg(C.EACCES) +) + +func getRevaError(err error) error { + if err == nil { + return nil + } + switch err.Error() { + case errNotFound: + return errtypes.NotFound("cephfs: dir entry not found") + case errPermissionDenied: + return errtypes.PermissionDenied("cephfs: permission denied") + case errFileExists: + return errtypes.AlreadyExists("cephfs: file already exists") + case errNoSpaceLeft: + return errtypes.InsufficientStorage("cephfs: no space left on device") + default: + return errtypes.InternalError(err.Error()) + } +} diff --git a/pkg/storage/fs/cephfs/options.go b/pkg/storage/fs/cephfs/options.go new file mode 100644 index 0000000000..0b4b81f0e6 --- /dev/null +++ b/pkg/storage/fs/cephfs/options.go @@ -0,0 +1,90 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "path/filepath" + + "github.com/cs3org/reva/pkg/sharedconf" +) + +// Options for the cephfs module +type Options struct { + GatewaySvc string `mapstructure:"gatewaysvc"` + IndexPool string `mapstructure:"index_pool"` + Root string `mapstructure:"root"` + ShadowFolder string `mapstructure:"shadow_folder"` + ShareFolder string `mapstructure:"share_folder"` + UploadFolder string `mapstructure:"uploads"` + UserLayout string `mapstructure:"user_layout"` + + DisableHome bool `mapstructure:"disable_home"` + UserQuotaBytes uint64 `mapstructure:"user_quota_bytes"` + HiddenDirs map[string]bool +} + +func (c *Options) fillDefaults() { + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) + + if c.IndexPool == "" { + c.IndexPool = "path_index" + } + + if c.Root == "" { + c.Root = "/home" + } else { + c.Root = addLeadingSlash(c.Root) //force absolute path in case leading "/" is omitted + } + + if c.ShadowFolder == "" { + c.ShadowFolder = "/.reva_hidden" + } else { + c.ShadowFolder = addLeadingSlash(c.ShadowFolder) + } + + if c.ShareFolder == "" { + c.ShareFolder = "/Shares" + } else { + c.ShareFolder = addLeadingSlash(c.ShareFolder) + } + + if c.UploadFolder == "" { + c.UploadFolder = ".uploads" + } + c.UploadFolder = filepath.Join(c.ShadowFolder, c.UploadFolder) + + if c.UserLayout == "" { + c.UserLayout = "{{.Username}}" + } + + c.HiddenDirs = map[string]bool{ + ".": true, + "..": true, + removeLeadingSlash(c.ShadowFolder): true, + } + + c.DisableHome = false // it is currently only home based + + if c.UserQuotaBytes == 0 { + c.UserQuotaBytes = 50000000000 + } +} diff --git a/pkg/storage/fs/cephfs/permissions.go b/pkg/storage/fs/cephfs/permissions.go new file mode 100644 index 0000000000..8b4a32484d --- /dev/null +++ b/pkg/storage/fs/cephfs/permissions.go @@ -0,0 +1,325 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "errors" + "fmt" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/maxymania/go-system/posix_acl" +) + +var perms = map[rune][]string{ + 'r': { + "Stat", + "GetPath", + "GetQuota", + "InitiateFileDownload", + "ListGrants", + }, + 'w': { + "AddGrant", + "CreateContainer", + "Delete", + "InitiateFileUpload", + "Move", + "RemoveGrant", + "PurgeRecycle", + "RestoreFileVersion", + "RestoreRecycleItem", + "UpdateGrant", + }, + 'x': { + "ListRecycle", + "ListContainer", + "ListFileVersions", + }, +} + +const ( + aclXattr = "system.posix_acl_access" +) + +var op2int = map[rune]uint16{'r': 4, 'w': 2, 'x': 1} + +func getPermissionSet(user *User, stat *cephfs2.CephStatx, mount Mount, path string) (perm *provider.ResourcePermissions) { + perm = &provider.ResourcePermissions{} + + if int64(stat.Uid) == user.UidNumber || int64(stat.Gid) == user.GidNumber { + updatePerms(perm, "rwx", false) + return + } + + acls := &posix_acl.Acl{} + var xattr []byte + var err error + if xattr, err = mount.GetXattr(path, aclXattr); err != nil { + return nil + } + acls.Decode(xattr) + + group, err := user.fs.getGroupByID(user.ctx, fmt.Sprint(stat.Gid)) + + for _, acl := range acls.List { + rwx := strings.Split(acl.String(), ":")[2] + switch acl.GetType() { + case posix_acl.ACL_USER: + if int64(acl.GetID()) == user.UidNumber { + updatePerms(perm, rwx, false) + } + case posix_acl.ACL_GROUP: + if int64(acl.GetID()) == user.GidNumber || in(group.GroupName, user.Groups) { + updatePerms(perm, rwx, false) + } + case posix_acl.ACL_MASK: + updatePerms(perm, rwx, true) + case posix_acl.ACL_OTHERS: + updatePerms(perm, rwx, false) + } + } + + return +} + +func (fs *cephfs) getFullPermissionSet(ctx context.Context, mount Mount, path string) (permList []*provider.Grant) { + acls := &posix_acl.Acl{} + var xattr []byte + var err error + if xattr, err = mount.GetXattr(path, aclXattr); err != nil { + return nil + } + acls.Decode(xattr) + + for _, acl := range acls.List { + rwx := strings.Split(acl.String(), ":")[2] + switch acl.GetType() { + case posix_acl.ACL_USER: + user, err := fs.getUserByID(ctx, fmt.Sprint(acl.GetID())) + if err != nil { + return nil + } + userGrant := &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_USER, + Id: &provider.Grantee_UserId{UserId: user.Id}, + }, + Permissions: &provider.ResourcePermissions{}, + } + updatePerms(userGrant.Permissions, rwx, false) + permList = append(permList, userGrant) + case posix_acl.ACL_GROUP: + group, err := fs.getGroupByID(ctx, fmt.Sprint(acl.GetID())) + if err != nil { + return nil + } + groupGrant := &provider.Grant{ + Grantee: &provider.Grantee{ + Type: provider.GranteeType_GRANTEE_TYPE_GROUP, + Id: &provider.Grantee_GroupId{GroupId: group.Id}, + }, + Permissions: &provider.ResourcePermissions{}, + } + updatePerms(groupGrant.Permissions, rwx, false) + permList = append(permList, groupGrant) + } + } + + return +} + +/* +func permToIntRefl(p *provider.ResourcePermissions) (result uint16) { + if p == nil { return 0b111 } //rwx + + item := reflect.ValueOf(p).Elem() + for _, op := range "rwx" { + for _, perm := range perms[op] { + if item.FieldByName(perm).Bool() { + result |= op2int[op] + break //if value is 1 then bitwise OR can never change it again + } + } + } + + return +} +*/ + +func permToInt(rp *provider.ResourcePermissions) (result uint16) { + if rp == nil { + return 0b111 // rwx + } + if rp.Stat || rp.GetPath || rp.GetQuota || rp.ListGrants || rp.InitiateFileDownload { + result |= 4 + } + if rp.CreateContainer || rp.Move || rp.Delete || rp.InitiateFileUpload || rp.AddGrant || rp.UpdateGrant || + rp.RemoveGrant || rp.DenyGrant || rp.RestoreFileVersion || rp.PurgeRecycle || rp.RestoreRecycleItem { + result |= 2 + } + if rp.ListRecycle || rp.ListContainer || rp.ListFileVersions { + result |= 1 + } + + return +} + +const ( + updateGrant = iota + removeGrant = iota +) + +func (fs *cephfs) changePerms(ctx context.Context, mt Mount, grant *provider.Grant, path string, method int) (err error) { + buf, err := mt.GetXattr(path, aclXattr) + if err != nil { + return + } + acls := &posix_acl.Acl{} + acls.Decode(buf) + var sid posix_acl.AclSID + + switch grant.Grantee.Type { + case provider.GranteeType_GRANTEE_TYPE_USER: + var user *userpb.User + if user, err = fs.getUserByOpaqueID(ctx, grant.Grantee.GetUserId().OpaqueId); err != nil { + return + } + sid.SetUid(uint32(user.UidNumber)) + case provider.GranteeType_GRANTEE_TYPE_GROUP: + var group *grouppb.Group + if group, err = fs.getGroupByOpaqueID(ctx, grant.Grantee.GetGroupId().OpaqueId); err != nil { + return + } + sid.SetGid(uint32(group.GidNumber)) + default: + return errors.New("cephfs: invalid grantee type") + } + + var found = false + var i int + for i = range acls.List { + if acls.List[i].AclSID == sid { + found = true + } + } + + if method == updateGrant { + if found { + acls.List[i].Perm |= permToInt(grant.Permissions) + if acls.List[i].Perm == 0 { // remove empty grant + acls.List = append(acls.List[:i], acls.List[i+1:]...) + } + } else { + acls.List = append(acls.List, posix_acl.AclElement{ + AclSID: sid, + Perm: permToInt(grant.Permissions), + }) + } + } else { //removeGrant + if found { + acls.List[i].Perm &^= permToInt(grant.Permissions) //bitwise and-not, to clear bits on Perm + if acls.List[i].Perm == 0 { // remove empty grant + acls.List = append(acls.List[:i], acls.List[i+1:]...) + } + } + } + + err = mt.SetXattr(path, aclXattr, acls.Encode(), 0) + + return +} + +/* +func updatePermsRefl(rp *provider.ResourcePermissions, acl string, unset bool) { + if rp == nil { return } + for _, t := range "rwx" { + if strings.ContainsRune(acl, t) { + for _, i := range perms[t] { + reflect.ValueOf(rp).Elem().FieldByName(i).SetBool(true) + } + } else if unset { + for _, i := range perms[t] { + reflect.ValueOf(rp).Elem().FieldByName(i).SetBool(false) + } + } + } +} +*/ + +func updatePerms(rp *provider.ResourcePermissions, acl string, unset bool) { + if rp == nil { + return + } + if strings.ContainsRune(acl, 'r') { + rp.Stat = true + rp.GetPath = true + rp.GetQuota = true + rp.InitiateFileDownload = true + rp.ListGrants = true + } else if unset { + rp.Stat = false + rp.GetPath = false + rp.GetQuota = false + rp.InitiateFileDownload = false + rp.ListGrants = false + } + if strings.ContainsRune(acl, 'w') { + rp.AddGrant = true + rp.DenyGrant = true + rp.CreateContainer = true + rp.Delete = true + rp.InitiateFileUpload = true + rp.Move = true + rp.RemoveGrant = true + rp.PurgeRecycle = true + rp.RestoreFileVersion = true + rp.RestoreRecycleItem = true + rp.UpdateGrant = true + } else if unset { + rp.AddGrant = false + rp.DenyGrant = false + rp.CreateContainer = false + rp.Delete = false + rp.InitiateFileUpload = false + rp.Move = false + rp.RemoveGrant = false + rp.PurgeRecycle = false + rp.RestoreFileVersion = false + rp.RestoreRecycleItem = false + rp.UpdateGrant = false + } + if strings.ContainsRune(acl, 'x') { + rp.ListRecycle = true + rp.ListContainer = true + rp.ListFileVersions = true + } else if unset { + rp.ListRecycle = false + rp.ListContainer = false + rp.ListFileVersions = false + } +} diff --git a/pkg/storage/fs/cephfs/unsupported.go b/pkg/storage/fs/cephfs/unsupported.go new file mode 100644 index 0000000000..a337f3f789 --- /dev/null +++ b/pkg/storage/fs/cephfs/unsupported.go @@ -0,0 +1,39 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build !ceph +// +build !ceph + +package cephfs + +import ( + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" +) + +func init() { + registry.Register("cephfs", New) +} + +// New returns an implementation to of the storage.FS interface that talk to +// a ceph filesystem. +func New(m map[string]interface{}) (storage.FS, error) { + return nil, errors.New("cephfs: revad was compiled without CephFS support") +} diff --git a/pkg/storage/fs/cephfs/upload.go b/pkg/storage/fs/cephfs/upload.go new file mode 100644 index 0000000000..2d8eccf7c9 --- /dev/null +++ b/pkg/storage/fs/cephfs/upload.go @@ -0,0 +1,413 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + ctx2 "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/utils" + "github.com/google/uuid" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/pkg/handler" +) + +func (fs *cephfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + user := fs.makeUser(ctx) + upload, err := fs.GetUpload(ctx, ref.GetPath()) + if err != nil { + metadata := map[string]string{"sizedeferred": "true"} + uploadIDs, err := fs.InitiateUpload(ctx, ref, 0, metadata) + if err != nil { + return err + } + if upload, err = fs.GetUpload(ctx, uploadIDs["simple"]); err != nil { + return errors.Wrap(err, "cephfs: error retrieving upload") + } + } + + uploadInfo := upload.(*fileUpload) + + p := uploadInfo.info.Storage["InternalDestination"] + ok, err := IsChunked(p) + if err != nil { + return errors.Wrap(err, "cephfs: error checking path") + } + if ok { + var assembledFile string + p, assembledFile, err = NewChunkHandler(ctx, fs).WriteChunk(p, r) + if err != nil { + return err + } + if p == "" { + if err = uploadInfo.Terminate(ctx); err != nil { + return errors.Wrap(err, "cephfs: error removing auxiliary files") + } + return errtypes.PartialContent(ref.String()) + } + uploadInfo.info.Storage["InternalDestination"] = p + + user.op(func(cv *cacheVal) { + r, err = cv.mount.Open(assembledFile, os.O_RDONLY, 0) + }) + if err != nil { + return errors.Wrap(err, "cephfs: error opening assembled file") + } + defer r.Close() + defer user.op(func(cv *cacheVal) { + _ = cv.mount.Unlink(assembledFile) + }) + } + + if _, err := uploadInfo.WriteChunk(ctx, 0, r); err != nil { + return errors.Wrap(err, "cephfs: error writing to binary file") + } + + return uploadInfo.FinishUpload(ctx) +} + +func (fs *cephfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + user := fs.makeUser(ctx) + np, err := user.resolveRef(ref) + if err != nil { + return nil, errors.Wrap(err, "cephfs: error resolving reference") + } + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(np), + "dir": filepath.Dir(np), + }, + Size: uploadLength, + } + + if metadata != nil { + if metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + if _, ok := metadata["sizedeferred"]; ok { + info.SizeIsDeferred = true + } + } + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return nil, err + } + + info, _ = upload.GetInfo(ctx) + + return map[string]string{ + "simple": info.ID, + "tus": info.ID, + }, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *cephfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) +} + +func (fs *cephfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("cephfs: NewUpload") + + user := fs.makeUser(ctx) + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("cephfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("cephfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + np := filepath.Join(info.MetaData["dir"], info.MetaData["filename"]) + + info.ID = uuid.New().String() + + binPath := fs.getUploadPath(info.ID) + + info.Storage = map[string]string{ + "Type": "Cephfs", + "BinPath": binPath, + "InternalDestination": np, + + "Idp": user.Id.Idp, + "UserId": user.Id.OpaqueId, + "UserName": user.Username, + "UserType": utils.UserTypeToString(user.Id.Type), + + "LogLevel": log.GetLevel().String(), + } + + // Create binary file with no content + user.op(func(cv *cacheVal) { + var f *cephfs2.File + defer closeFile(f) + f, err = cv.mount.Open(binPath, os.O_CREATE|os.O_WRONLY, filePermDefault) + if err != nil { + return + } + }) + //TODO: if we get two same upload ids, the second one can't upload at all + if err != nil { + return + } + + upload = &fileUpload{ + info: info, + binPath: binPath, + infoPath: binPath + ".info", + fs: fs, + ctx: ctx, + } + + if !info.SizeIsDeferred && info.Size == 0 { + log.Debug().Interface("info", info).Msg("cephfs: finishing upload for empty file") + // no need to create info file and finish directly + err = upload.FinishUpload(ctx) + + return + } + + // writeInfo creates the file by itself if necessary + err = upload.(*fileUpload).writeInfo() + + return +} + +func (fs *cephfs) getUploadPath(uploadID string) string { + return filepath.Join(fs.conf.UploadFolder, uploadID) +} + +// GetUpload returns the Upload for the given upload id +func (fs *cephfs) GetUpload(ctx context.Context, id string) (fup tusd.Upload, err error) { + binPath := fs.getUploadPath(id) + info := tusd.FileInfo{} + if err != nil { + return nil, errtypes.NotFound("bin path for upload " + id + " not found") + } + infoPath := binPath + ".info" + + var data bytes.Buffer + f, err := fs.adminConn.adminMount.Open(infoPath, os.O_RDONLY, 0) + if err != nil { + return + } + _, err = io.Copy(&data, f) + if err != nil { + return + } + if err = json.Unmarshal(data.Bytes(), &info); err != nil { + return + } + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + ctx = ctx2.ContextSetUser(ctx, u) + user := fs.makeUser(ctx) + + var stat Statx + user.op(func(cv *cacheVal) { + stat, err = cv.mount.Statx(binPath, cephfs2.StatxSize, 0) + }) + if err != nil { + return + } + info.Offset = int64(stat.Size) + + return &fileUpload{ + info: info, + binPath: binPath, + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *cephfs + // a context with a user + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// GetReader returns an io.Reader for the upload +func (upload *fileUpload) GetReader(ctx context.Context) (file io.Reader, err error) { + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + file, err = cv.mount.Open(upload.binPath, os.O_RDONLY, 0) + }) + return +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (n int64, err error) { + var file io.WriteCloser + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + file, err = cv.mount.Open(upload.binPath, os.O_WRONLY|os.O_APPEND, 0) + }) + if err != nil { + return 0, err + } + defer file.Close() + + n, err = io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for OwnCloudStore it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() + + return n, err +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + + if err != nil { + return err + } + user := upload.fs.makeUser(upload.ctx) + user.op(func(cv *cacheVal) { + var file io.WriteCloser + if file, err = cv.mount.Open(upload.infoPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, filePermDefault); err != nil { + return + } + defer file.Close() + + _, err = io.Copy(file, bytes.NewReader(data)) + }) + + return err +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { + + np := upload.info.Storage["InternalDestination"] + + // TODO check etag with If-Match header + // if destination exists + // if _, err := os.Stat(np); err == nil { + // the local storage does not store metadata + // the fileid is based on the path, so no we do not need to copy it to the new file + // the local storage does not track revisions + // } + + // if destination exists + // if _, err := os.Stat(np); err == nil { + // create revision + // if err := upload.fs.archiveRevision(upload.ctx, np); err != nil { + // return err + // } + // } + + user := upload.fs.makeUser(upload.ctx) + log := appctx.GetLogger(ctx) + + user.op(func(cv *cacheVal) { + err = cv.mount.Rename(upload.binPath, np) + }) + if err != nil { + return errors.Wrap(err, upload.binPath) + } + + // only delete the upload if it was successfully written to the fs + user.op(func(cv *cacheVal) { + err = cv.mount.Unlink(upload.infoPath) + }) + if err != nil { + if err.Error() != errNotFound { + log.Err(err).Interface("info", upload.info).Msg("cephfs: could not delete upload metadata") + } + } + + // TODO: set mtime if specified in metadata + + return +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returns a a TerminatableUpload +func (fs *cephfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) (err error) { + user := upload.fs.makeUser(upload.ctx) + + user.op(func(cv *cacheVal) { + if err = cv.mount.Unlink(upload.infoPath); err != nil { + return + } + err = cv.mount.Unlink(upload.binPath) + }) + + return +} diff --git a/pkg/storage/fs/cephfs/user.go b/pkg/storage/fs/cephfs/user.go new file mode 100644 index 0000000000..e99686adfd --- /dev/null +++ b/pkg/storage/fs/cephfs/user.go @@ -0,0 +1,242 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "context" + "fmt" + "path/filepath" + "strconv" + "strings" + "syscall" + + "github.com/cs3org/reva/pkg/errtypes" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + ctx2 "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/pkg/errors" +) + +type callBack func(cb *cacheVal) + +// User custom type to add functionality to current struct +type User struct { + *userv1beta1.User + fs *cephfs + ctx context.Context + home string +} + +func (fs *cephfs) makeUser(ctx context.Context) *User { + u := ctx2.ContextMustGetUser(ctx) + home := filepath.Join(fs.conf.Root, templates.WithUser(u, fs.conf.UserLayout)) + return &User{u, fs, ctx, home} +} + +func (user *User) absPath(path string) string { + //shares will always be absolute to avoid prepending the user path to the path of the file's owner + if !filepath.IsAbs(path) { + path = filepath.Join(user.home, path) + } + + return path +} + +func (user *User) op(cb callBack) { + conn := user.fs.conn + if err := conn.lock.Acquire(conn.ctx, 1); err != nil { + return + } + defer conn.lock.Release(1) + + val, found := conn.cache.Get(user.Id.OpaqueId) + if !found { + cvalue := newConn(user) + if cvalue != nil { + conn.cache.Set(user.Id.OpaqueId, cvalue, 1) + } else { + return + } + cb(cvalue) + return + } + + cb(val.(*cacheVal)) +} + +func (user *User) fileAsResourceInfo(cv *cacheVal, path string, stat *cephfs2.CephStatx, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var ( + _type provider.ResourceType + target string + size uint64 + buf []byte + ) + + switch int(stat.Mode) & syscall.S_IFMT { + case syscall.S_IFDIR: + _type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + if buf, err = cv.mount.GetXattr(path, "ceph.dir.rbytes"); err == nil { + size, err = strconv.ParseUint(string(buf), 10, 64) + } + case syscall.S_IFLNK: + _type = provider.ResourceType_RESOURCE_TYPE_SYMLINK + target, err = cv.mount.Readlink(path) + case syscall.S_IFREG: + _type = provider.ResourceType_RESOURCE_TYPE_FILE + size = stat.Size + default: + return nil, errors.New("cephfs: unknown entry type") + } + + if err != nil { + return + } + + var xattrs []string + keys := make(map[string]bool, len(mdKeys)) + for _, key := range mdKeys { + keys[key] = true + } + if keys["*"] || len(keys) == 0 { + mdKeys = []string{} + keys = map[string]bool{} + } + mx := make(map[string]string) + if xattrs, err = cv.mount.ListXattr(path); err == nil { + for _, xattr := range xattrs { + if len(mdKeys) == 0 || keys[xattr] { + if buf, err := cv.mount.GetXattr(path, xattr); err == nil { + mx[xattr] = string(buf) + } + } + } + } + + //TODO(tmourati): Add entry id logic here + + var etag string + if isDir(_type) { + rctime, _ := cv.mount.GetXattr(path, "ceph.dir.rctime") + etag = fmt.Sprint(stat.Inode) + ":" + string(rctime) + } else { + etag = fmt.Sprint(stat.Inode) + ":" + strconv.FormatInt(stat.Ctime.Sec, 10) + } + + mtime := &typesv1beta1.Timestamp{ + Seconds: uint64(stat.Mtime.Sec), + Nanos: uint32(stat.Mtime.Nsec), + } + + perms := getPermissionSet(user, stat, cv.mount, path) + + for key := range mx { + if !strings.HasPrefix(key, xattrUserNs) { + delete(mx, key) + } + } + + var checksum provider.ResourceChecksum + var md5 string + if _type == provider.ResourceType_RESOURCE_TYPE_FILE { + md5tsBA, err := cv.mount.GetXattr(path, xattrMd5ts) //local error inside if scope + if err == nil { + md5ts, _ := strconv.ParseInt(string(md5tsBA), 10, 64) + if stat.Mtime.Sec == md5ts { + md5BA, err := cv.mount.GetXattr(path, xattrMd5) + if err != nil { + md5, err = calcChecksum(path, cv.mount, stat) + } else { + md5 = string(md5BA) + } + } else { + md5, err = calcChecksum(path, cv.mount, stat) + } + } else { + md5, err = calcChecksum(path, cv.mount, stat) + } + + if err != nil && err.Error() == errPermissionDenied { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET + } else if err != nil { + return nil, errors.New("cephfs: error calculating checksum of file") + } else { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_MD5 + checksum.Sum = md5 + } + } else { + checksum.Type = provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET + } + + var ownerID *userv1beta1.UserId + if stat.Uid != 0 { + var owner *userv1beta1.User + if int64(stat.Uid) != user.UidNumber { + owner, err = user.fs.getUserByID(user.ctx, fmt.Sprint(stat.Uid)) + } else { + owner = user.User + } + + if owner == nil { + return nil, errors.New("cephfs: error getting owner of entry: " + path) + } + + ownerID = owner.Id + } else { + ownerID = &userv1beta1.UserId{OpaqueId: "root"} + } + + ri = &provider.ResourceInfo{ + Type: _type, + Id: &provider.ResourceId{OpaqueId: fmt.Sprint(stat.Inode)}, + Checksum: &checksum, + Etag: etag, + MimeType: mime.Detect(isDir(_type), path), + Mtime: mtime, + Path: path, + PermissionSet: perms, + Size: size, + Owner: ownerID, + Target: target, + ArbitraryMetadata: &provider.ArbitraryMetadata{Metadata: mx}, + } + + return +} + +func (user *User) resolveRef(ref *provider.Reference) (str string, err error) { + if ref == nil { + return "", fmt.Errorf("cephfs: nil reference") + } + + if str = ref.GetPath(); str == "" { + return "", errtypes.NotSupported("cephfs: entry IDs not currently supported") + } + + str = removeLeadingSlash(str) //path must be relative + + return +} diff --git a/pkg/storage/fs/cephfs/utils.go b/pkg/storage/fs/cephfs/utils.go new file mode 100644 index 0000000000..e25b3bc5fa --- /dev/null +++ b/pkg/storage/fs/cephfs/utils.go @@ -0,0 +1,245 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +//go:build ceph +// +build ceph + +package cephfs + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + cephfs2 "github.com/ceph/go-ceph/cephfs" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// Mount type +type Mount = *cephfs2.MountInfo + +// Statx type +type Statx = *cephfs2.CephStatx + +var dirPermFull = uint32(0777) +var dirPermDefault = uint32(0775) +var filePermDefault = uint32(0660) + +func closeDir(directory *cephfs2.Directory) { + if directory != nil { + _ = directory.Close() + } +} + +func closeFile(file *cephfs2.File) { + if file != nil { + _ = file.Close() + } +} + +func destroyCephConn(mt Mount, perm *cephfs2.UserPerm) *cacheVal { + if perm != nil { + perm.Destroy() + } + if mt != nil { + _ = mt.Release() + } + return nil +} + +func deleteFile(mount *cephfs2.MountInfo, path string) { + _ = mount.Unlink(path) +} + +func isDir(t provider.ResourceType) bool { + return t == provider.ResourceType_RESOURCE_TYPE_CONTAINER +} + +func (fs *cephfs) makeFIDPath(fid string) string { + return "" //filepath.Join(fs.conf.EIDFolder, fid) +} + +func (fs *cephfs) makeFID(absolutePath string, inode string) (rid *provider.ResourceId, err error) { + sum := md5.New() + sum.Write([]byte(absolutePath)) + fid := fmt.Sprintf("%s-%s", hex.EncodeToString(sum.Sum(nil)), inode) + rid = &provider.ResourceId{OpaqueId: fid} + + _ = fs.adminConn.adminMount.Link(absolutePath, fs.makeFIDPath(fid)) + _ = fs.adminConn.adminMount.SetXattr(absolutePath, xattrEID, []byte(fid), 0) + + return +} + +func (fs *cephfs) getFIDPath(cv *cacheVal, path string) (fid string, err error) { + var buffer []byte + if buffer, err = cv.mount.GetXattr(path, xattrEID); err != nil { + return + } + + return fs.makeFIDPath(string(buffer)), err +} + +func calcChecksum(filepath string, mt Mount, stat Statx) (checksum string, err error) { + file, err := mt.Open(filepath, os.O_RDONLY, 0) + defer closeFile(file) + if err != nil { + return + } + hash := md5.New() + if _, err = io.Copy(hash, file); err != nil { + return + } + checksum = hex.EncodeToString(hash.Sum(nil)) + // we don't care if they fail, the checksum will just be recalculated if an error happens + _ = mt.SetXattr(filepath, xattrMd5ts, []byte(strconv.FormatInt(stat.Mtime.Sec, 10)), 0) + _ = mt.SetXattr(filepath, xattrMd5, []byte(checksum), 0) + + return +} + +func resolveRevRef(mt Mount, ref *provider.Reference, revKey string) (str string, err error) { + var buf []byte + if ref.GetResourceId() != nil { + str, err = mt.Readlink(filepath.Join(snap, revKey, ref.ResourceId.OpaqueId)) + if err != nil { + return "", fmt.Errorf("cephfs: invalid reference %+v", ref) + } + } else if str = ref.GetPath(); str != "" { + buf, err = mt.GetXattr(str, xattrEID) + if err != nil { + return + } + str, err = mt.Readlink(filepath.Join(snap, revKey, string(buf))) + if err != nil { + return + } + } else { + return "", fmt.Errorf("cephfs: empty reference %+v", ref) + } + + return filepath.Join(snap, revKey, str), err +} + +func removeLeadingSlash(path string) string { + return filepath.Join(".", path) +} + +func addLeadingSlash(path string) string { + return filepath.Join("/", path) +} + +func in(lookup string, list []string) bool { + for _, item := range list { + if item == lookup { + return true + } + } + return false +} + +func pathGenerator(path string, reverse bool, str chan string) { + if reverse { + str <- path + for i := range path { + if path[len(path)-i-1] == filepath.Separator { + str <- path[:len(path)-i-1] + } + } + } else { + for i := range path { + if path[i] == filepath.Separator { + str <- path[:i] + } + } + str <- path + } + + close(str) +} + +func walkPath(path string, f func(string) error, reverse bool) (err error) { + paths := make(chan string) + go pathGenerator(path, reverse, paths) + for path := range paths { + if path == "" { + continue + } + if err = f(path); err != nil && err.Error() != errFileExists && err.Error() != errNotFound { + break + } else { + err = nil + } + } + + return +} + +func (fs *cephfs) writeIndex(oid string, value string) (err error) { + return fs.adminConn.radosIO.WriteFull(oid, []byte(value)) +} + +func (fs *cephfs) removeIndex(oid string) error { + return fs.adminConn.radosIO.Delete(oid) +} + +func (fs *cephfs) resolveIndex(oid string) (fullPath string, err error) { + var i int + var currPath strings.Builder + root := string(filepath.Separator) + offset := uint64(0) + io := fs.adminConn.radosIO + bsize := 4096 + buffer := make([]byte, bsize) + for { + for { //read object + i, err = io.Read(oid, buffer, offset) + offset += uint64(bsize) + currPath.Write(buffer) + if err == nil && i >= bsize { + buffer = buffer[:0] + continue + } else { + offset = 0 + break + } + } + if err != nil { + return + } + + ss := strings.SplitN(currPath.String(), string(filepath.Separator), 2) + if len(ss) != 2 { + if currPath.String() == root { + return + } + + return "", fmt.Errorf("cephfs: entry id is not in the form of \"parentID/entryname\"") + } + parentOID := ss[0] + entryName := ss[1] + fullPath = filepath.Join(entryName, fullPath) + oid = parentOID + currPath.Reset() + } +} diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index f9ff86af04..cd88c5ddc7 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -20,6 +20,7 @@ package loader import ( // Load core storage filesystem backends. + _ "github.com/cs3org/reva/pkg/storage/fs/cephfs" _ "github.com/cs3org/reva/pkg/storage/fs/eos" _ "github.com/cs3org/reva/pkg/storage/fs/eosgrpc" _ "github.com/cs3org/reva/pkg/storage/fs/eosgrpchome" From 81e8416a29d22bc9457d462eecc69a01b368bbda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:02 +0100 Subject: [PATCH 03/17] [Build-deps]: Bump github.com/minio/minio-go/v7 from 7.0.20 to 7.0.21 (#2449) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 367e062000..1d2a89f9f0 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 - github.com/minio/minio-go/v7 v7.0.20 + github.com/minio/minio-go/v7 v7.0.21 github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 diff --git a/go.sum b/go.sum index 0f992f1d5a..fba8360edb 100644 --- a/go.sum +++ b/go.sum @@ -526,8 +526,8 @@ github.com/mileusna/useragent v1.0.2 h1:DgVKtiPnjxlb73z9bCwgdUvU2nQNQ97uhgfO8l9u github.com/mileusna/useragent v1.0.2/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.20 h1:0+Xt1SkCKDgcx5cmo3UxXcJ37u5Gy+/2i/+eQYqmYJw= -github.com/minio/minio-go/v7 v7.0.20/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= +github.com/minio/minio-go/v7 v7.0.21 h1:xrc4BQr1Fa4s5RwY0xfMjPZFJ1bcYBCCHYlngBdWV+k= +github.com/minio/minio-go/v7 v7.0.21/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= From 9c814c6a0e5c198b1bbd14cc8f202a754734c05d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:23 +0100 Subject: [PATCH 04/17] [Build-deps]: Bump github.com/hashicorp/go-hclog from 1.0.0 to 1.1.0 (#2448) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1d2a89f9f0..8c28a73665 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/go-hclog v1.0.0 + github.com/hashicorp/go-hclog v1.1.0 github.com/hashicorp/go-plugin v1.4.3 github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect diff --git a/go.sum b/go.sum index fba8360edb..fe8056c784 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0 h1:bkKf0BeBXcSYa7f5Fyi9gMuQ8gNsxeiNpZjR6VxNZeo= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= +github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= From 7117894ca302df750f1ca9a500cb13cf31db17f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:11:43 +0100 Subject: [PATCH 05/17] [Build-deps]: Bump github.com/BurntSushi/toml from 0.4.1 to 1.0.0 (#2446) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c28a73665..de36d8ed5d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cs3org/reva require ( bou.ke/monkey v1.0.2 contrib.go.opencensus.io/exporter/prometheus v0.4.0 - github.com/BurntSushi/toml v0.4.1 + github.com/BurntSushi/toml v1.0.0 github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible diff --git a/go.sum b/go.sum index fe8056c784..a88045a2cb 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= From 4cd33f0c4825f79a1cedb5a562cc847947ebdc4b Mon Sep 17 00:00:00 2001 From: Willy Kloucek <34452982+wkloucek@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:02:31 +0100 Subject: [PATCH 06/17] revert go-sqlite downgrade (#2461) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de36d8ed5d..fcbb1fa64b 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 - github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/mattn/go-sqlite3 v1.14.10 github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b github.com/mileusna/useragent v1.0.2 github.com/minio/minio-go/v7 v7.0.21 diff --git a/go.sum b/go.sum index a88045a2cb..7690018b81 100644 --- a/go.sum +++ b/go.sum @@ -513,8 +513,8 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= +github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= From 6f0ac6bb2e01b1600417593648999a52925172d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:02:44 +0100 Subject: [PATCH 07/17] [Build-deps]: Bump github.com/google/go-cmp from 0.5.6 to 0.5.7 (#2466) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fcbb1fa64b..598811e8e0 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.2 github.com/gomodule/redigo v1.8.8 - github.com/google/go-cmp v0.5.6 + github.com/google/go-cmp v0.5.7 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 7690018b81..9159089e25 100644 --- a/go.sum +++ b/go.sum @@ -369,8 +369,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= From a649b7d2d0468e8f45e1f37a41dbeee45436aec3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:03:01 +0100 Subject: [PATCH 08/17] [Build-deps]: Bump github.com/aws/aws-sdk-go from 1.42.27 to 1.42.39 (#2467) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 598811e8e0..6d29c50484 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/ReneKroon/ttlcache/v2 v2.11.0 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/aws/aws-sdk-go v1.42.27 + github.com/aws/aws-sdk-go v1.42.39 github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 diff --git a/go.sum b/go.sum index 9159089e25..557492855d 100644 --- a/go.sum +++ b/go.sum @@ -100,8 +100,8 @@ github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9 github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.41.13/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.27 h1:kxsBXQg3ee6LLbqjp5/oUeDgG7TENFrWYDmEVnd7spU= -github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= +github.com/aws/aws-sdk-go v1.42.39 h1:6Lso73VoCI8Zmv3zAMv4BNg2gHAKNOlbLv1s/ew90SI= +github.com/aws/aws-sdk-go v1.42.39/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= From fdad1a529465300cd1bdaabbcae650336a221c4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:03:37 +0100 Subject: [PATCH 09/17] [Build-deps]: Bump github.com/ceph/go-ceph from 0.12.0 to 0.13.0 (#2468) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6d29c50484..397df4b1f9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/beevik/etree v1.1.0 github.com/bluele/gcache v0.0.2 github.com/c-bata/go-prompt v0.2.5 - github.com/ceph/go-ceph v0.12.0 + github.com/ceph/go-ceph v0.13.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e diff --git a/go.sum b/go.sum index 557492855d..618f451c23 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,8 @@ github.com/c-bata/go-prompt v0.2.5 h1:3zg6PecEywxNn0xiqcXHD96fkbxghD+gdB2tbsYfl+ github.com/c-bata/go-prompt v0.2.5/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/ceph/go-ceph v0.12.0 h1:nlFgKQZXOFR4oMnzXsKwTr79Y6EYDwqTrpigICGy/Tw= -github.com/ceph/go-ceph v0.12.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= +github.com/ceph/go-ceph v0.13.0 h1:69dgIPlNHD2OCz98T0benI4++vcnShGcpQK4RIALjw4= +github.com/ceph/go-ceph v0.13.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= From cf04a9f91b06572557f3f7660a209d6dffc5189b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:04:05 +0100 Subject: [PATCH 10/17] [Build-deps]: Bump github.com/onsi/gomega from 1.17.0 to 1.18.0 (#2469) --- go.mod | 4 ++-- go.sum | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 397df4b1f9..1fe3123044 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.17.0 + github.com/onsi/gomega v1.18.0 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.4 github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -75,7 +75,7 @@ require ( golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/term v0.0.0-20210916214954-140adaaadfaf google.golang.org/genproto v0.0.0-20211021150943-2b146023228c google.golang.org/grpc v1.42.0 diff --git a/go.sum b/go.sum index 618f451c23..cf019e6049 100644 --- a/go.sum +++ b/go.sum @@ -392,6 +392,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -575,11 +576,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.0 h1:ngbYoRctxjl8SiF7XgP0NxBFbfHcg3wfHMMaFHWwMTM= +github.com/onsi/gomega v1.18.0/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -960,8 +964,9 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210916214954-140adaaadfaf h1:Ihq/mm/suC88gF8WFcVwk+OV6Tq+wyA1O0E5UEvDglI= From 393f831f7f44e62d3d230ba2b5242daea71f2ad5 Mon Sep 17 00:00:00 2001 From: David Christofas Date: Mon, 24 Jan 2022 13:09:03 +0100 Subject: [PATCH 11/17] Use permissions API in decomposedfs (#2341) --- .drone.star | 1 + .../unreleased/cs3-permissions-service.md | 5 + cmd/revad/runtime/loader.go | 1 + go.mod | 2 +- go.sum | 4 +- internal/grpc/services/gateway/gateway.go | 1 + internal/grpc/services/gateway/permissions.go | 39 +++++++ internal/grpc/services/loader/loader.go | 1 + .../grpc/services/permissions/permissions.go | 104 ++++++++++++++++++ .../storageprovider/storageprovider.go | 15 +-- pkg/permission/manager/demo/demo.go | 43 ++++++++ pkg/permission/manager/loader/loader.go | 25 +++++ pkg/permission/manager/registry/registry.go | 34 ++++++ pkg/permission/permission.go | 28 +++++ pkg/rgrpc/todo/pool/pool.go | 21 ++++ pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 +- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql.go | 2 +- pkg/storage/fs/s3/s3.go | 2 +- pkg/storage/storage.go | 2 +- .../utils/decomposedfs/decomposedfs.go | 3 + .../utils/decomposedfs/options/options.go | 2 + pkg/storage/utils/decomposedfs/spaces.go | 45 ++++++-- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/storage/utils/localfs/localfs.go | 2 +- tests/oc-integration-tests/drone/gateway.toml | 2 + .../drone/permissions-ocis-ci.toml | 12 ++ .../drone/storage-home-ocis.toml | 2 + .../drone/storage-users-ocis.toml | 2 + tests/oc-integration-tests/local/gateway.toml | 2 + .../local/permissions-ocis-ci.toml | 12 ++ .../local/storage-home.toml | 1 + .../local/storage-users.toml | 1 + 34 files changed, 389 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/cs3-permissions-service.md create mode 100644 internal/grpc/services/gateway/permissions.go create mode 100644 internal/grpc/services/permissions/permissions.go create mode 100644 pkg/permission/manager/demo/demo.go create mode 100644 pkg/permission/manager/loader/loader.go create mode 100644 pkg/permission/manager/registry/registry.go create mode 100644 pkg/permission/permission.go create mode 100644 tests/oc-integration-tests/drone/permissions-ocis-ci.toml create mode 100644 tests/oc-integration-tests/local/permissions-ocis-ci.toml diff --git a/.drone.star b/.drone.star index 92460ad3e3..eeed6bd9ba 100644 --- a/.drone.star +++ b/.drone.star @@ -719,6 +719,7 @@ def litmusOcisSpacesDav(): "/drone/src/cmd/revad/revad -c gateway.toml &", "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &", "/drone/src/cmd/revad/revad -c users.toml", ], }, diff --git a/changelog/unreleased/cs3-permissions-service.md b/changelog/unreleased/cs3-permissions-service.md new file mode 100644 index 0000000000..b792173e20 --- /dev/null +++ b/changelog/unreleased/cs3-permissions-service.md @@ -0,0 +1,5 @@ +Enhancement: Use CS3 permissions API + +Added calls to the CS3 permissions API to the decomposedfs in order to check the user permissions. + +https://github.com/cs3org/reva/pull/2341 diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index 93f5c68ace..a0df692027 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -38,6 +38,7 @@ import ( _ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/manager/loader" + _ "github.com/cs3org/reva/pkg/permission/manager/loader" _ "github.com/cs3org/reva/pkg/publicshare/manager/loader" _ "github.com/cs3org/reva/pkg/rhttp/datatx/manager/loader" _ "github.com/cs3org/reva/pkg/share/cache/loader" diff --git a/go.mod b/go.mod index 1fe3123044..2fdb36d86d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-oidc v2.2.1+incompatible github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 + github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 github.com/dgraph-io/ristretto v0.1.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 diff --git a/go.sum b/go.sum index cf019e6049..ae31f5ef35 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8 h1:PqOprF37OvwCbAN5W23znknGk6N/LMayqLAeP904FHE= -github.com/cs3org/go-cs3apis v0.0.0-20211214102047-7ce3134d7bf8/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654 h1:ha5tiuuFyDrwKUrVEc3TrRDFgTKVQ9NGDRmEP0PRPno= +github.com/cs3org/go-cs3apis v0.0.0-20211214102128-4e8745ab1654/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 3a8a6203a5..25819dfb22 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -57,6 +57,7 @@ type config struct { GroupProviderEndpoint string `mapstructure:"groupprovidersvc"` DataTxEndpoint string `mapstructure:"datatx"` DataGatewayEndpoint string `mapstructure:"datagateway"` + PermissionsEndpoint string `mapstructure:"permissionssvc"` CommitShareToStorageGrant bool `mapstructure:"commit_share_to_storage_grant"` CommitShareToStorageRef bool `mapstructure:"commit_share_to_storage_ref"` DisableHomeCreationOnLogin bool `mapstructure:"disable_home_creation_on_login"` diff --git a/internal/grpc/services/gateway/permissions.go b/internal/grpc/services/gateway/permissions.go new file mode 100644 index 0000000000..2b1806633a --- /dev/null +++ b/internal/grpc/services/gateway/permissions.go @@ -0,0 +1,39 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package gateway + +import ( + "context" + + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/pkg/errors" +) + +func (s *svc) CheckPermission(ctx context.Context, req *permissions.CheckPermissionRequest) (*permissions.CheckPermissionResponse, error) { + c, err := pool.GetPermissionsClient(s.c.PermissionsEndpoint) + if err != nil { + err = errors.Wrap(err, "gateway: error calling GetPermissionssClient") + return &permissions.CheckPermissionResponse{ + Status: status.NewInternal(ctx, err, "error getting permissions client"), + }, nil + } + return c.CheckPermission(ctx, req) +} diff --git a/internal/grpc/services/loader/loader.go b/internal/grpc/services/loader/loader.go index 118eeed39e..e0161997d2 100644 --- a/internal/grpc/services/loader/loader.go +++ b/internal/grpc/services/loader/loader.go @@ -33,6 +33,7 @@ import ( _ "github.com/cs3org/reva/internal/grpc/services/ocminvitemanager" _ "github.com/cs3org/reva/internal/grpc/services/ocmproviderauthorizer" _ "github.com/cs3org/reva/internal/grpc/services/ocmshareprovider" + _ "github.com/cs3org/reva/internal/grpc/services/permissions" _ "github.com/cs3org/reva/internal/grpc/services/preferences" _ "github.com/cs3org/reva/internal/grpc/services/publicshareprovider" _ "github.com/cs3org/reva/internal/grpc/services/publicstorageprovider" diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go new file mode 100644 index 0000000000..4479fdb88c --- /dev/null +++ b/internal/grpc/services/permissions/permissions.go @@ -0,0 +1,104 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package permissions + +import ( + "context" + "fmt" + + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + "github.com/cs3org/reva/pkg/permission" + "github.com/cs3org/reva/pkg/permission/manager/registry" + "github.com/cs3org/reva/pkg/rgrpc" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +func init() { + rgrpc.Register("permissions", New) +} + +type config struct { + Driver string `mapstructure:"driver" docs:"localhome;The permission driver to be used."` + Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/permission/permission.go"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return c, nil +} + +type service struct { + manager permission.Manager +} + +// New returns a new PermissionsServiceServer +func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + + f, ok := registry.NewFuncs[c.Driver] + if !ok { + return nil, fmt.Errorf("could not get permission manager '%s'", c.Driver) + } + manager, err := f(c.Drivers[c.Driver]) + if err != nil { + return nil, err + } + + service := &service{manager: manager} + return service, nil +} + +func (s *service) Close() error { + return nil +} + +func (s *service) UnprotectedEndpoints() []string { + return []string{} +} + +func (s *service) Register(ss *grpc.Server) { + permissions.RegisterPermissionsAPIServer(ss, s) +} + +func (s *service) CheckPermission(ctx context.Context, req *permissions.CheckPermissionRequest) (*permissions.CheckPermissionResponse, error) { + var subject string + switch ref := req.SubjectRef.Spec.(type) { + case *permissions.SubjectReference_UserId: + subject = ref.UserId.OpaqueId + case *permissions.SubjectReference_GroupId: + subject = ref.GroupId.OpaqueId + } + var status *rpc.Status + if ok := s.manager.CheckPermission(req.Permission, subject, req.Ref); ok { + status = &rpc.Status{Code: rpc.Code_CODE_OK} + } else { + status = &rpc.Status{Code: rpc.Code_CODE_PERMISSION_DENIED} + } + return &permissions.CheckPermissionResponse{Status: status}, nil +} diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 13dc51ed15..82e09db0bb 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -20,7 +20,6 @@ package storageprovider import ( "context" - "encoding/json" "fmt" "net/url" "os" @@ -579,19 +578,7 @@ func hasNodeID(s *provider.StorageSpace) bool { func (s *service) ListStorageSpaces(ctx context.Context, req *provider.ListStorageSpacesRequest) (*provider.ListStorageSpacesResponse, error) { log := appctx.GetLogger(ctx) - // This is just a quick hack to get the users permission into reva. - // Replace this as soon as we have a proper system to check the users permissions. - opaque := req.Opaque - var permissions map[string]struct{} - if opaque != nil { - entry := opaque.Map["permissions"] - err := json.Unmarshal(entry.Value, &permissions) - if err != nil { - return nil, err - } - } - - spaces, err := s.storage.ListStorageSpaces(ctx, req.Filters, permissions) + spaces, err := s.storage.ListStorageSpaces(ctx, req.Filters) if err != nil { var st *rpc.Status switch err.(type) { diff --git a/pkg/permission/manager/demo/demo.go b/pkg/permission/manager/demo/demo.go new file mode 100644 index 0000000000..7bb6c10779 --- /dev/null +++ b/pkg/permission/manager/demo/demo.go @@ -0,0 +1,43 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package demo + +import ( + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/permission" + "github.com/cs3org/reva/pkg/permission/manager/registry" +) + +func init() { + registry.Register("demo", New) +} + +// New returns a new demo permission manager +func New(c map[string]interface{}) (permission.Manager, error) { + return manager{}, nil +} + +type manager struct { +} + +func (m manager) CheckPermission(permission string, subject string, ref *provider.Reference) bool { + // We can currently return true all the time. + // Once we beginn testing roles we need to somehow check the roles of the users here + return true +} diff --git a/pkg/permission/manager/loader/loader.go b/pkg/permission/manager/loader/loader.go new file mode 100644 index 0000000000..5f0bbc5774 --- /dev/null +++ b/pkg/permission/manager/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load permission manager drivers + _ "github.com/cs3org/reva/pkg/permission/manager/demo" + // Add your own here +) diff --git a/pkg/permission/manager/registry/registry.go b/pkg/permission/manager/registry/registry.go new file mode 100644 index 0000000000..26f55bebad --- /dev/null +++ b/pkg/permission/manager/registry/registry.go @@ -0,0 +1,34 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import "github.com/cs3org/reva/pkg/permission" + +// NewFunc is the function that permission managers +// should register at init time. +type NewFunc func(map[string]interface{}) (permission.Manager, error) + +// NewFuncs is a map containing all the registered share managers. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new permission manager new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/permission/permission.go b/pkg/permission/permission.go new file mode 100644 index 0000000000..e5e5c76a52 --- /dev/null +++ b/pkg/permission/permission.go @@ -0,0 +1,28 @@ +// Copyright 2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package permission + +import ( + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// Manager defines the interface for the permission service driver +type Manager interface { + CheckPermission(permission string, subject string, ref *provider.Reference) bool +} diff --git a/pkg/rgrpc/todo/pool/pool.go b/pkg/rgrpc/todo/pool/pool.go index 550d7d7499..b2c94b1219 100644 --- a/pkg/rgrpc/todo/pool/pool.go +++ b/pkg/rgrpc/todo/pool/pool.go @@ -32,6 +32,7 @@ import ( ocmcore "github.com/cs3org/go-cs3apis/cs3/ocm/core/v1beta1" invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" + permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" preferences "github.com/cs3org/go-cs3apis/cs3/preferences/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" @@ -70,6 +71,7 @@ var ( ocmCores = newProvider() publicShareProviders = newProvider() preferencesProviders = newProvider() + permissionsProviders = newProvider() appRegistries = newProvider() appProviders = newProvider() storageRegistries = newProvider() @@ -349,6 +351,25 @@ func GetPreferencesClient(endpoint string) (preferences.PreferencesAPIClient, er return v, nil } +// GetPermissionsClient returns a new PermissionsClient. +func GetPermissionsClient(endpoint string) (permissions.PermissionsAPIClient, error) { + permissionsProviders.m.Lock() + defer permissionsProviders.m.Unlock() + + if c, ok := permissionsProviders.conn[endpoint]; ok { + return c.(permissions.PermissionsAPIClient), nil + } + + conn, err := NewConn(endpoint) + if err != nil { + return nil, err + } + + v := permissions.NewPermissionsAPIClient(conn) + permissionsProviders.conn[endpoint] = v + return v, nil +} + // GetAppRegistryClient returns a new AppRegistryClient. func GetAppRegistryClient(endpoint string) (appregistry.RegistryAPIClient, error) { appRegistries.m.Lock() diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 04e5b54912..34e2fe3452 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -783,7 +783,7 @@ func (nc *StorageDriver) Unlock(ctx context.Context, ref *provider.Reference) er } // ListStorageSpaces as defined in the storage.FS interface -func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, f []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (nc *StorageDriver) ListStorageSpaces(ctx context.Context, f []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { bodyStr, _ := json.Marshal(f) _, respBody, err := nc.do(ctx, Action{"ListStorageSpaces", string(bodyStr)}) if err != nil { diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index b81edd643b..f4565a54b5 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -987,7 +987,7 @@ var _ = Describe("Nextcloud", func() { }, } filters := []*provider.ListStorageSpacesRequest_Filter{filter1, filter2, filter3} - spaces, err := nc.ListStorageSpaces(ctx, filters, nil) + spaces, err := nc.ListStorageSpaces(ctx, filters) Expect(err).ToNot(HaveOccurred()) Expect(len(spaces)).To(Equal(1)) // https://github.com/cs3org/go-cs3apis/blob/970eec3/cs3/storage/provider/v1beta1/resources.pb.go#L1341-L1366 diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 5e0a65bce8..c75f82fde1 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -2255,7 +2255,7 @@ func (fs *ocfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativeP return fs.propagate(ctx, tgt) } -func (fs *ocfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *ocfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 64803fc945..ca38b13afe 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -1950,7 +1950,7 @@ func (fs *owncloudsqlfs) HashFile(path string) (string, string, string, error) { } } -func (fs *owncloudsqlfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *owncloudsqlfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { // TODO(corby): Implement return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 063800326e..76799b9047 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -702,7 +702,7 @@ func (fs *s3FS) RestoreRecycleItem(ctx context.Context, basePath, key, relativeP return errtypes.NotSupported("restore recycle") } -func (fs *s3FS) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *s3FS) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4256d10cdf..ae5cba3428 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -62,7 +62,7 @@ type FS interface { GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error Unlock(ctx context.Context, ref *provider.Reference) error - ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) + ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 5e40163e6e..b3ac9e138c 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -39,6 +39,7 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" @@ -101,6 +102,8 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore) (storage.FS, error) lu.Options = o tp := tree.New(o.Root, o.TreeTimeAccounting, o.TreeSizeAccounting, lu, bs) + + o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr) return New(o, lu, p, tp) } diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index 181d20f19e..892b185a22 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -53,6 +53,8 @@ type Options struct { Owner string `mapstructure:"owner"` OwnerIDP string `mapstructure:"owner_idp"` OwnerType string `mapstructure:"owner_type"` + + GatewayAddr string `mapstructure:"gateway_addr"` } // New returns a new Options instance for the given configuration diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index b34bb57f65..989ae0c124 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -28,12 +28,14 @@ import ( "strings" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + permissionsv1beta1 "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ocsconv "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/utils" @@ -156,7 +158,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr // The list can be filtered by space type or space id. // Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space // The spaceid is a concatenation of storageid + "!" + nodeid -func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, permissions map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { // TODO check filters // TODO when a space symlink is broken delete the space for cleanup @@ -200,6 +202,28 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide return spaces, nil } + client, err := pool.GetGatewayServiceClient(fs.o.GatewayAddr) + if err != nil { + return nil, err + } + + checkRes, err := client.CheckPermission(ctx, &permissionsv1beta1.CheckPermissionRequest{ + Permission: "list-all-spaces", + SubjectRef: &permissionsv1beta1.SubjectReference{ + Spec: &permissionsv1beta1.SubjectReference_UserId{ + UserId: u.Id, + }, + }, + }) + if err != nil { + return nil, err + } + + canListAllSpaces := false + if checkRes.Status.Code == v1beta11.Code_CODE_OK { + canListAllSpaces = true + } + for i := range matches { // always read link in case storage space id != node id if target, err := os.Readlink(matches[i]); err != nil { @@ -226,7 +250,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } // TODO apply more filters - space, err := fs.storageSpaceFromNode(ctx, n, matches[i], spaceType, permissions) + space, err := fs.storageSpaceFromNode(ctx, n, matches[i], spaceType, canListAllSpaces) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not convert to storage space") continue @@ -329,7 +353,7 @@ func (fs *Decomposedfs) createStorageSpace(ctx context.Context, spaceType, nodeI return nil } -func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Node, nodePath, spaceType string, permissions map[string]struct{}) (*provider.StorageSpace, error) { +func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Node, nodePath, spaceType string, canListAllSpaces bool) (*provider.StorageSpace, error) { owner, err := node.Owner() if err != nil { return nil, err @@ -357,13 +381,14 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, node *node.Nod user := ctxpkg.ContextMustGetUser(ctx) // filter out spaces user cannot access (currently based on stat permission) - _, canListAllSpaces := permissions["list-all-spaces"] - p, err := node.ReadUserPermissions(ctx, user) - if err != nil { - return nil, err - } - if !(canListAllSpaces || p.Stat) { - return nil, errors.New("user is not allowed to Stat the space") + if !canListAllSpaces { + p, err := node.ReadUserPermissions(ctx, user) + if err != nil { + return nil, err + } + if !p.Stat { + return nil, errors.New("user is not allowed to Stat the space") + } } space.Owner = &userv1beta1.User{ // FIXME only return a UserID, not a full blown user object diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 50cdc8dfd9..76034da770 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -1628,7 +1628,7 @@ func (fs *eosfs) RestoreRecycleItem(ctx context.Context, basePath, key, relative return fs.c.RestoreDeletedEntry(ctx, auth, key) } -func (fs *eosfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *eosfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index aa55d27715..4c1a68f225 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -1286,7 +1286,7 @@ func (fs *localfs) RestoreRecycleItem(ctx context.Context, basePath, key, relati return fs.propagate(ctx, localRestorePath) } -func (fs *localfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, _ map[string]struct{}) ([]*provider.StorageSpace, error) { +func (fs *localfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { return nil, errtypes.NotSupported("list storage spaces") } diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index e00e43158d..85c17b20c2 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -30,6 +30,8 @@ ocmcoresvc = "localhost:14000" ocmshareprovidersvc = "localhost:14000" ocminvitemanagersvc = "localhost:14000" ocmproviderauthorizersvc = "localhost:14000" +# permissions +permissionssvc = "localhost:10000" # other commit_share_to_storage_grant = true commit_share_to_storage_ref = true diff --git a/tests/oc-integration-tests/drone/permissions-ocis-ci.toml b/tests/oc-integration-tests/drone/permissions-ocis-ci.toml new file mode 100644 index 0000000000..ef025245f3 --- /dev/null +++ b/tests/oc-integration-tests/drone/permissions-ocis-ci.toml @@ -0,0 +1,12 @@ +# This config file will start a reva service that: +# - serves the ocis ci permissions service +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:10000" + +[grpc.services.permissions] +driver = "demo" + +[grpc.services.publicshareprovider.drivers.ocisci] diff --git a/tests/oc-integration-tests/drone/storage-home-ocis.toml b/tests/oc-integration-tests/drone/storage-home-ocis.toml index dbc0748f0a..098aaf7fca 100644 --- a/tests/oc-integration-tests/drone/storage-home-ocis.toml +++ b/tests/oc-integration-tests/drone/storage-home-ocis.toml @@ -23,12 +23,14 @@ mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true data_server_url = "http://revad-services:12001/data" enable_home_creation = true +gateway_addr = "0.0.0.0:19000" [grpc.services.storageprovider.drivers.ocis] root = "/drone/src/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" # we have a locally running dataprovider [http] diff --git a/tests/oc-integration-tests/drone/storage-users-ocis.toml b/tests/oc-integration-tests/drone/storage-users-ocis.toml index 2d6ef8fc2d..795ba41d54 100644 --- a/tests/oc-integration-tests/drone/storage-users-ocis.toml +++ b/tests/oc-integration-tests/drone/storage-users-ocis.toml @@ -19,12 +19,14 @@ mount_path = "/users" mount_id = "123e4567-e89b-12d3-a456-426655440000" expose_data_server = true data_server_url = "http://revad-services:11001/data" +gateway_addr = "0.0.0.0:19000" [grpc.services.storageprovider.drivers.ocis] root = "/drone/src/tmp/reva/data" treetime_accounting = true treesize_accounting = true userprovidersvc = "localhost:18000" +gateway_addr = "0.0.0.0:19000" # we have a locally running dataprovider [http] diff --git a/tests/oc-integration-tests/local/gateway.toml b/tests/oc-integration-tests/local/gateway.toml index 6a4f46ee23..39d387d724 100644 --- a/tests/oc-integration-tests/local/gateway.toml +++ b/tests/oc-integration-tests/local/gateway.toml @@ -30,6 +30,8 @@ ocmcoresvc = "localhost:14000" ocmshareprovidersvc = "localhost:14000" ocminvitemanagersvc = "localhost:14000" ocmproviderauthorizersvc = "localhost:14000" +# permissions +permissionssvc = "localhost:10000" # other commit_share_to_storage_grant = true commit_share_to_storage_ref = true diff --git a/tests/oc-integration-tests/local/permissions-ocis-ci.toml b/tests/oc-integration-tests/local/permissions-ocis-ci.toml new file mode 100644 index 0000000000..ef025245f3 --- /dev/null +++ b/tests/oc-integration-tests/local/permissions-ocis-ci.toml @@ -0,0 +1,12 @@ +# This config file will start a reva service that: +# - serves the ocis ci permissions service +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:10000" + +[grpc.services.permissions] +driver = "demo" + +[grpc.services.publicshareprovider.drivers.ocisci] diff --git a/tests/oc-integration-tests/local/storage-home.toml b/tests/oc-integration-tests/local/storage-home.toml index 03b4d6ab6f..cd019d9dff 100644 --- a/tests/oc-integration-tests/local/storage-home.toml +++ b/tests/oc-integration-tests/local/storage-home.toml @@ -30,6 +30,7 @@ root = "/var/tmp/reva/data" enable_home = true treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" #user_layout = # do we need owner for users? #owner = 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad diff --git a/tests/oc-integration-tests/local/storage-users.toml b/tests/oc-integration-tests/local/storage-users.toml index d2023fc72a..cb37e05342 100644 --- a/tests/oc-integration-tests/local/storage-users.toml +++ b/tests/oc-integration-tests/local/storage-users.toml @@ -38,3 +38,4 @@ root = "/var/tmp/reva/data" enable_home = false treetime_accounting = true treesize_accounting = true +gateway_addr = "0.0.0.0:19000" From 9ed5f32e966e1e65e58b65fcaadc1fa705169960 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Tue, 25 Jan 2022 17:41:03 +0545 Subject: [PATCH 12/17] [tests-only]Bump Core Commit Id (#2451) --- .drone.env | 2 +- .drone.star | 4 +-- .../expected-failures-on-OCIS-storage.md | 31 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 31 +++++++++---------- 4 files changed, 31 insertions(+), 37 deletions(-) diff --git a/.drone.env b/.drone.env index 22bb1ee8a6..d0a052d1f2 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=88dc1fc01b8afd254d59cbface6ef48670848e22 +CORE_COMMITID=a91d8d0f045c512aa76ecc92372a0b5ae7b4e3cd CORE_BRANCH=master diff --git a/.drone.star b/.drone.star index eeed6bd9ba..be7b562a6e 100644 --- a/.drone.star +++ b/.drone.star @@ -808,7 +808,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", @@ -883,7 +883,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis", + "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index cf97c7a7b2..aea236385d 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -42,7 +42,7 @@ _ocdav: double check the webdav property parsing when custom namespaces are used Synchronization features like etag propagation, setting mtime and locking files #### [Uploading an old method chunked file with checksum should fail using new DAV path](https://github.com/owncloud/ocis/issues/2323) -- [apiMain/checksums.feature:263](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L263) +- [apiMain/checksums.feature:369](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L369) #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [apiWebdavLocks/exclusiveLocks.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/exclusiveLocks.feature#L18) @@ -975,20 +975,20 @@ Scenario Outline: search for entry with emoji by pattern - [apiWebdavOperations/search.feature:255](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L255) Scenario: search for entries across various folders by tags using REPORT method And other missing implementation of favorites -- [apiFavorites/favorites.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L128) -- [apiFavorites/favorites.feature:129](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L129) -- [apiFavorites/favorites.feature:148](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L148) -- [apiFavorites/favorites.feature:149](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L149) -- [apiFavorites/favorites.feature:176](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L176) -- [apiFavorites/favorites.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L177) +- [apiFavorites/favorites.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L158) +- [apiFavorites/favorites.feature:159](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L159) +- [apiFavorites/favorites.feature:183](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L183) +- [apiFavorites/favorites.feature:184](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L184) +- [apiFavorites/favorites.feature:216](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L216) +- [apiFavorites/favorites.feature:217](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L217) - [apiFavorites/favoritesSharingToShares.feature:21](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L21) - [apiFavorites/favoritesSharingToShares.feature:22](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L22) -- [apiFavorites/favoritesSharingToShares.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L35) -- [apiFavorites/favoritesSharingToShares.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L36) -- [apiFavorites/favoritesSharingToShares.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L48) -- [apiFavorites/favoritesSharingToShares.feature:49](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L49) -- [apiFavorites/favoritesSharingToShares.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L62) -- [apiFavorites/favoritesSharingToShares.feature:63](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L63) +- [apiFavorites/favoritesSharingToShares.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L40) +- [apiFavorites/favoritesSharingToShares.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L41) +- [apiFavorites/favoritesSharingToShares.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L58) +- [apiFavorites/favoritesSharingToShares.feature:59](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L59) +- [apiFavorites/favoritesSharingToShares.feature:77](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L77) +- [apiFavorites/favoritesSharingToShares.feature:78](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L78) #### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285) Scenario Outline: Unauthenticated call @@ -1375,7 +1375,4 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) -- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 4fc1c4446a..18631fd5a5 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -56,7 +56,7 @@ _ocdav: double check the webdav property parsing when custom namespaces are used Synchronization features like etag propagation, setting mtime and locking files #### [Uploading an old method chunked file with checksum should fail using new DAV path](https://github.com/owncloud/ocis/issues/2323) -- [apiMain/checksums.feature:263](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L263) +- [apiMain/checksums.feature:369](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiMain/checksums.feature#L369) #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [apiWebdavLocks/exclusiveLocks.feature:18](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavLocks/exclusiveLocks.feature#L18) @@ -990,20 +990,20 @@ Scenario Outline: search for entry with emoji by pattern - [apiWebdavOperations/search.feature:255](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/search.feature#L255) Scenario: search for entries across various folders by tags using REPORT method And other missing implementation of favorites -- [apiFavorites/favorites.feature:128](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L128) -- [apiFavorites/favorites.feature:129](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L129) -- [apiFavorites/favorites.feature:148](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L148) -- [apiFavorites/favorites.feature:149](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L149) -- [apiFavorites/favorites.feature:176](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L176) -- [apiFavorites/favorites.feature:177](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L177) +- [apiFavorites/favorites.feature:158](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L158) +- [apiFavorites/favorites.feature:159](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L159) +- [apiFavorites/favorites.feature:183](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L183) +- [apiFavorites/favorites.feature:184](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L184) +- [apiFavorites/favorites.feature:216](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L216) +- [apiFavorites/favorites.feature:217](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favorites.feature#L217) - [apiFavorites/favoritesSharingToShares.feature:21](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L21) - [apiFavorites/favoritesSharingToShares.feature:22](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L22) -- [apiFavorites/favoritesSharingToShares.feature:35](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L35) -- [apiFavorites/favoritesSharingToShares.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L36) -- [apiFavorites/favoritesSharingToShares.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L48) -- [apiFavorites/favoritesSharingToShares.feature:49](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L49) -- [apiFavorites/favoritesSharingToShares.feature:62](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L62) -- [apiFavorites/favoritesSharingToShares.feature:63](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L63) +- [apiFavorites/favoritesSharingToShares.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L40) +- [apiFavorites/favoritesSharingToShares.feature:41](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L41) +- [apiFavorites/favoritesSharingToShares.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L58) +- [apiFavorites/favoritesSharingToShares.feature:59](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L59) +- [apiFavorites/favoritesSharingToShares.feature:77](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L77) +- [apiFavorites/favoritesSharingToShares.feature:78](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiFavorites/favoritesSharingToShares.feature#L78) #### [WWW-Authenticate header for unauthenticated requests is not clear](https://github.com/owncloud/ocis/issues/2285) Scenario Outline: Unauthenticated call @@ -1375,7 +1375,4 @@ _ocs: api compatibility, return correct status code_ ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) - [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) - -### [Calculate personal space id in tests when running on reva](https://github.com/owncloud/core/issues/39617) -- [apiWebdavUpload1/uploadFile.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavUpload1/uploadFile.feature#L32) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file From 0e8c017e2767273e50dfebd4fb61154ec0bb53d8 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 26 Jan 2022 08:50:24 +0545 Subject: [PATCH 13/17] Bump commit id for tests 2022-01-25 (#2474) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 20 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 20 +++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.drone.env b/.drone.env index d0a052d1f2..a336b97fad 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=a91d8d0f045c512aa76ecc92372a0b5ae7b4e3cd +CORE_COMMITID=a8b0dd3d164da66e060290e23b278817d444d15d CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index aea236385d..009c3da2a2 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -607,16 +607,16 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [apiShareOperationsToShares2/uploadToShare.feature:168](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L168) -- [apiShareOperationsToShares2/uploadToShare.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L169) -- [apiShareOperationsToShares2/uploadToShare.feature:188](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L188) -- [apiShareOperationsToShares2/uploadToShare.feature:189](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L189) -- [apiShareOperationsToShares2/uploadToShare.feature:210](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L210) -- [apiShareOperationsToShares2/uploadToShare.feature:211](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L211) -- [apiShareOperationsToShares2/uploadToShare.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L230) -- [apiShareOperationsToShares2/uploadToShare.feature:231](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L231) -- [apiShareOperationsToShares2/uploadToShare.feature:252](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L252) -- [apiShareOperationsToShares2/uploadToShare.feature:253](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L253) +- [apiShareOperationsToShares2/uploadToShare.feature:193](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L193) +- [apiShareOperationsToShares2/uploadToShare.feature:194](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L194) +- [apiShareOperationsToShares2/uploadToShare.feature:218](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L218) +- [apiShareOperationsToShares2/uploadToShare.feature:219](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L219) +- [apiShareOperationsToShares2/uploadToShare.feature:245](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L245) +- [apiShareOperationsToShares2/uploadToShare.feature:246](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L246) +- [apiShareOperationsToShares2/uploadToShare.feature:270](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L270) +- [apiShareOperationsToShares2/uploadToShare.feature:271](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L271) +- [apiShareOperationsToShares2/uploadToShare.feature:297](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L297) +- [apiShareOperationsToShares2/uploadToShare.feature:298](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L298) #### [not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) - [apiShareOperationsToShares1/changingFilesShare.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares1/changingFilesShare.feature#L24) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 18631fd5a5..5991fa4bf8 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -596,16 +596,16 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [apiShareOperationsToShares2/uploadToShare.feature:168](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L168) -- [apiShareOperationsToShares2/uploadToShare.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L169) -- [apiShareOperationsToShares2/uploadToShare.feature:188](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L188) -- [apiShareOperationsToShares2/uploadToShare.feature:189](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L189) -- [apiShareOperationsToShares2/uploadToShare.feature:210](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L210) -- [apiShareOperationsToShares2/uploadToShare.feature:211](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L211) -- [apiShareOperationsToShares2/uploadToShare.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L230) -- [apiShareOperationsToShares2/uploadToShare.feature:231](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L231) -- [apiShareOperationsToShares2/uploadToShare.feature:252](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L252) -- [apiShareOperationsToShares2/uploadToShare.feature:253](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L253) +- [apiShareOperationsToShares2/uploadToShare.feature:193](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L193) +- [apiShareOperationsToShares2/uploadToShare.feature:194](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L194) +- [apiShareOperationsToShares2/uploadToShare.feature:218](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L218) +- [apiShareOperationsToShares2/uploadToShare.feature:219](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L219) +- [apiShareOperationsToShares2/uploadToShare.feature:245](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L245) +- [apiShareOperationsToShares2/uploadToShare.feature:246](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L246) +- [apiShareOperationsToShares2/uploadToShare.feature:270](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L270) +- [apiShareOperationsToShares2/uploadToShare.feature:271](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L271) +- [apiShareOperationsToShares2/uploadToShare.feature:297](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L297) +- [apiShareOperationsToShares2/uploadToShare.feature:298](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/uploadToShare.feature#L298) #### [not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) From b3f53a14b3e4655d01510085a3914a96ff17f696 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Wed, 26 Jan 2022 16:21:20 +0545 Subject: [PATCH 14/17] Bump commit id for issue-ocis-3030 (#2476) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 35 +++++++++++-------- .../expected-failures-on-S3NG-storage.md | 35 +++++++++++-------- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/.drone.env b/.drone.env index a336b97fad..d0afb183e2 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=a8b0dd3d164da66e060290e23b278817d444d15d +CORE_COMMITID=65a12d1858c0708cb1ca2ad620e139e0d33b73ae CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 009c3da2a2..3f089be08d 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -914,26 +914,26 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [Trying to access another user's file gives http 403 instead of 404](https://github.com/owncloud/ocis/issues/2175) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavDELETEAuth.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L38) Scenario: send DELETE requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPFINDAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L39) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPPATCHAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L40) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavMKCOLAuth.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L36) Scenario: send MKCOL requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavDELETEAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L58) Scenario: send DELETE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPFINDAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L57) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPPATCHAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L58) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L54) Scenario: send MKCOL requests to another user's webDav endpoints as normal user #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [apiAuthWebDav/webDavLOCKAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L40) Scenario: send LOCK requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavLOCKAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L58) Scenario: send LOCK requests to another user's webDav endpoints as normal user #### [Renaming a resource to banned name is allowed](https://github.com/owncloud/ocis/issues/1295) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavMOVEAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L39) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMOVEAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L57) Scenario: send MOVE requests to another user's webDav endpoints as normal user #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavPOSTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L40) Scenario: send POST requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPOSTAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L58) Scenario: send POST requests to another user's webDav endpoints as normal user #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) -- [apiAuthWebDav/webDavSpecialURLs.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L24) -- [apiAuthWebDav/webDavSpecialURLs.feature:69](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L69) -- [apiAuthWebDav/webDavSpecialURLs.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L91) +- [apiAuthWebDav/webDavSpecialURLs.feature:34](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L34) +- [apiAuthWebDav/webDavSpecialURLs.feature:121](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L121) +- [apiAuthWebDav/webDavSpecialURLs.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L163) #### [Default capabilities for normal user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) #### [Difference in response content of status.php and default capabilities](https://github.com/owncloud/ocis/issues/1286) @@ -1071,8 +1071,8 @@ Scenario Outline: A disabled user cannot use webdav - [apiAuth/cors.feature:185](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuth/cors.feature#L185) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [apiAuthWebDav/webDavDELETEAuth.feature:80](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L80) -- [apiAuthWebDav/webDavDELETEAuth.feature:94](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L94) +- [apiAuthWebDav/webDavDELETEAuth.feature:136](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L136) +- [apiAuthWebDav/webDavDELETEAuth.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L162) #### [various sharing settings cannot be set](https://github.com/owncloud/ocis/issues/1328) - [apiCapabilities/capabilities.feature:8](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiCapabilities/capabilities.feature#L8) @@ -1374,5 +1374,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:125](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L125) ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) -- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file +- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +#### [Creating a new folder which is a substring of Shares leads to Unknown Error](https://github.com/owncloud/ocis/issues/3033) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 5991fa4bf8..d5547fa96f 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -929,26 +929,26 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [Trying to access another user's file gives http 403 instead of 404](https://github.com/owncloud/ocis/issues/2175) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavDELETEAuth.feature:38](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L38) Scenario: send DELETE requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPFINDAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L39) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavPROPPATCHAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L40) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user -- [apiAuthWebDav/webDavMKCOLAuth.feature:36](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L36) Scenario: send MKCOL requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavDELETEAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L58) Scenario: send DELETE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPFINDAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPFINDAuth.feature#L57) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPROPPATCHAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPROPPATCHAuth.feature#L58) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMKCOLAuth.feature#L54) Scenario: send MKCOL requests to another user's webDav endpoints as normal user #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [apiAuthWebDav/webDavLOCKAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L40) Scenario: send LOCK requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavLOCKAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavLOCKAuth.feature#L58) Scenario: send LOCK requests to another user's webDav endpoints as normal user #### [Renaming a resource to banned name is allowed](https://github.com/owncloud/ocis/issues/1295) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavMOVEAuth.feature:39](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L39) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavMOVEAuth.feature:57](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavMOVEAuth.feature#L57) Scenario: send MOVE requests to another user's webDav endpoints as normal user #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [apiAuthWebDav/webDavPOSTAuth.feature:40](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L40) Scenario: send POST requests to another user's webDav endpoints as normal user +- [apiAuthWebDav/webDavPOSTAuth.feature:58](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavPOSTAuth.feature#L58) Scenario: send POST requests to another user's webDav endpoints as normal user #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) -- [apiAuthWebDav/webDavSpecialURLs.feature:24](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L24) -- [apiAuthWebDav/webDavSpecialURLs.feature:69](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L69) -- [apiAuthWebDav/webDavSpecialURLs.feature:91](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L91) +- [apiAuthWebDav/webDavSpecialURLs.feature:34](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L34) +- [apiAuthWebDav/webDavSpecialURLs.feature:121](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L121) +- [apiAuthWebDav/webDavSpecialURLs.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavSpecialURLs.feature#L163) #### [Default capabilities for normal user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) #### [Difference in response content of status.php and default capabilities](https://github.com/owncloud/ocis/issues/1286) @@ -1086,8 +1086,8 @@ Scenario Outline: A disabled user cannot use webdav - [apiAuth/cors.feature:185](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuth/cors.feature#L185) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [apiAuthWebDav/webDavDELETEAuth.feature:80](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L80) -- [apiAuthWebDav/webDavDELETEAuth.feature:94](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L94) +- [apiAuthWebDav/webDavDELETEAuth.feature:136](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L136) +- [apiAuthWebDav/webDavDELETEAuth.feature:162](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiAuthWebDav/webDavDELETEAuth.feature#L162) #### [various sharing settings cannot be set](https://github.com/owncloud/ocis/issues/1328) - [apiCapabilities/capabilities.feature:8](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiCapabilities/capabilities.feature#L8) @@ -1374,5 +1374,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:125](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L125) ### [Content-type is not multipart/byteranges when downloading file with Range Header](https://github.com/owncloud/ocis/issues/2677) -- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) -- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) \ No newline at end of file +- [apiWebdavOperations/downloadFile.feature:169](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L169) +- [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) + +#### [Creating a new folder which is a substring of Shares leads to Unknown Error](https://github.com/owncloud/ocis/issues/3033) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file From b1be9881d26929add056c7a0885fff57d9540e64 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Thu, 27 Jan 2022 14:38:48 +0100 Subject: [PATCH 15/17] Use ocs permission objects in the reva GRPC client (#2478) --- changelog/unreleased/share-create-perm-fix.md | 7 +++ cmd/reva/common.go | 1 - cmd/reva/share-create.go | 44 ++----------------- 3 files changed, 11 insertions(+), 41 deletions(-) create mode 100644 changelog/unreleased/share-create-perm-fix.md diff --git a/changelog/unreleased/share-create-perm-fix.md b/changelog/unreleased/share-create-perm-fix.md new file mode 100644 index 0000000000..5dda4c3234 --- /dev/null +++ b/changelog/unreleased/share-create-perm-fix.md @@ -0,0 +1,7 @@ +Bugfix: Use ocs permission objects in the reva GRPC client + +There was a bug introduced by differing CS3APIs permission definitions +for the same role across services. This is a first step in making +all services use consistent definitions. + +https://github.com/cs3org/reva/pull/2478 \ No newline at end of file diff --git a/cmd/reva/common.go b/cmd/reva/common.go index ed630c1128..64f80ac566 100644 --- a/cmd/reva/common.go +++ b/cmd/reva/common.go @@ -31,7 +31,6 @@ import ( const ( viewerPermission string = "viewer" - readerPermission string = "reader" editorPermission string = "editor" collabPermission string = "collab" denyPermission string = "denied" diff --git a/cmd/reva/share-create.go b/cmd/reva/share-create.go index 3713b76e6e..7042c73615 100644 --- a/cmd/reva/share-create.go +++ b/cmd/reva/share-create.go @@ -28,6 +28,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/utils" "github.com/jedib0t/go-pretty/table" "github.com/pkg/errors" @@ -158,48 +159,11 @@ func getGrantType(t string) provider.GranteeType { func getSharePerm(p string) (*provider.ResourcePermissions, error) { switch p { case viewerPermission: - return &provider.ResourcePermissions{ - GetPath: true, - ListContainer: true, - Stat: true, - }, nil - case readerPermission: - return &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - }, nil + return conversions.NewViewerRole().CS3ResourcePermissions(), nil case editorPermission: - return &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - CreateContainer: true, - Delete: true, - InitiateFileUpload: true, - RestoreFileVersion: true, - Move: true, - }, nil + return conversions.NewEditorRole().CS3ResourcePermissions(), nil case collabPermission: - return &provider.ResourcePermissions{ - GetPath: true, - InitiateFileDownload: true, - ListFileVersions: true, - ListContainer: true, - Stat: true, - CreateContainer: true, - Delete: true, - InitiateFileUpload: true, - RestoreFileVersion: true, - Move: true, - AddGrant: true, - UpdateGrant: true, - RemoveGrant: true, - }, nil + return conversions.NewCoownerRole().CS3ResourcePermissions(), nil case denyPermission: return &provider.ResourcePermissions{}, nil default: From 0229254ea6928216361578dd97ae8f5d15e64c70 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Fri, 28 Jan 2022 00:27:23 +0545 Subject: [PATCH 16/17] Add end-of-line to expected-failures files (#2482) --- tests/acceptance/expected-failures-on-OCIS-storage.md | 4 +++- tests/acceptance/expected-failures-on-S3NG-storage.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 3f089be08d..e2a1582163 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -1382,4 +1382,6 @@ _ocs: api compatibility, return correct status code_ - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file + +Note: always have an empty line at the end of this file. +The bash script that processes this file may not process a scenario reference on the last line. diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index d5547fa96f..c96df82f7e 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -1382,4 +1382,6 @@ _ocs: api compatibility, return correct status code_ - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:60](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L60) \ No newline at end of file + +Note: always have an empty line at the end of this file. +The bash script that processes this file may not process a scenario reference on the last line. From 19246df75e34fb11c902f9793628b7cbe0b50050 Mon Sep 17 00:00:00 2001 From: Amrita <54478846+amrita-shrestha@users.noreply.github.com> Date: Fri, 28 Jan 2022 20:34:15 +0545 Subject: [PATCH 17/17] Bump the commit id for tests (#2489) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 27 ++++++++------ .../expected-failures-on-S3NG-storage.md | 35 +++++++++++-------- 3 files changed, 39 insertions(+), 25 deletions(-) diff --git a/.drone.env b/.drone.env index d0afb183e2..4f28f08f0f 100644 --- a/.drone.env +++ b/.drone.env @@ -1,3 +1,3 @@ # The test runner source for API tests -CORE_COMMITID=65a12d1858c0708cb1ca2ad620e139e0d33b73ae +CORE_COMMITID=0dadfbe475438dd97c192cb93643ef8d95b71faa CORE_BRANCH=master diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index e2a1582163..704ea364f9 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -35,8 +35,10 @@ _ocdav: double check the webdav property parsing when custom namespaces are used - [apiWebdavProperties1/setFileProperties.feature:64](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/setFileProperties.feature#L64) #### [Cannot set custom webDav properties](https://github.com/owncloud/product/issues/264) -- [apiWebdavProperties2/getFileProperties.feature:254](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L254) -- [apiWebdavProperties2/getFileProperties.feature:274](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L274) +- [apiWebdavProperties2/getFileProperties.feature:348](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L348) +- [apiWebdavProperties2/getFileProperties.feature:353](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L353) +- [apiWebdavProperties2/getFileProperties.feature:389](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L389) +- [apiWebdavProperties2/getFileProperties.feature:394](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L394) ### Sync Synchronization features like etag propagation, setting mtime and locking files @@ -597,13 +599,13 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei - [apiWebdavProperties1/getQuota.feature:28](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/getQuota.feature#L28) #### [cannot get share-types webdav property](https://github.com/owncloud/ocis/issues/567) -- [apiWebdavProperties2/getFileProperties.feature:174](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L174) -- [apiWebdavProperties2/getFileProperties.feature:175](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L175) +- [apiWebdavProperties2/getFileProperties.feature:229](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L229) +- [apiWebdavProperties2/getFileProperties.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L230) #### [Private link support](https://github.com/owncloud/product/issues/201) #### [oc:privatelink property not returned in webdav responses](https://github.com/owncloud/product/issues/262) -- [apiWebdavProperties2/getFileProperties.feature:232](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L232) -- [apiWebdavProperties2/getFileProperties.feature:233](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L233) +- [apiWebdavProperties2/getFileProperties.feature:295](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L295) +- [apiWebdavProperties2/getFileProperties.feature:296](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L296) #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ @@ -1307,12 +1309,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L163) #### [[OC-storage] share-types field empty for shared file folder in webdav response](https://github.com/owncloud/ocis/issues/2144) -- [apiWebdavProperties2/getFileProperties.feature:156](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L156) -- [apiWebdavProperties2/getFileProperties.feature:157](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L157) +- [apiWebdavProperties2/getFileProperties.feature:207](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L207) +- [apiWebdavProperties2/getFileProperties.feature:208](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L208) #### [Creating a public link with all permissions(31) fails](https://github.com/owncloud/ocis/issues/2145) -- [apiWebdavProperties2/getFileProperties.feature:206](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L206) -- [apiWebdavProperties2/getFileProperties.feature:207](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L207) +- [apiWebdavProperties2/getFileProperties.feature:265](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L265) +- [apiWebdavProperties2/getFileProperties.feature:266](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L266) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) - [apiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareUpdateToShares/updateShare.feature#L241) @@ -1383,5 +1385,10 @@ _ocs: api compatibility, return correct status code_ - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) - [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +#### [moveShareInsideAnotherShare behaves differently on oCIS than oC10](https://github.com/owncloud/ocis/issues/3047) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:25](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L25) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:86](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L86) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:100](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L100) + Note: always have an empty line at the end of this file. The bash script that processes this file may not process a scenario reference on the last line. diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index c96df82f7e..632dbc39b2 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -50,8 +50,10 @@ _ocdav: double check the webdav property parsing when custom namespaces are used - [apiWebdavProperties1/setFileProperties.feature:64](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/setFileProperties.feature#L64) #### [Cannot set custom webDav properties](https://github.com/owncloud/product/issues/264) -- [apiWebdavProperties2/getFileProperties.feature:254](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L254) -- [apiWebdavProperties2/getFileProperties.feature:274](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L274) +- [apiWebdavProperties2/getFileProperties.feature:348](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L348) +- [apiWebdavProperties2/getFileProperties.feature:353](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L353) +- [apiWebdavProperties2/getFileProperties.feature:389](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L389) +- [apiWebdavProperties2/getFileProperties.feature:394](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L394) ### Sync Synchronization features like etag propagation, setting mtime and locking files @@ -586,13 +588,13 @@ Scenario Outline: Retrieving folder quota when quota is set and a file was recei - [apiWebdavProperties1/getQuota.feature:28](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/getQuota.feature#L28) #### [cannot get share-types webdav property](https://github.com/owncloud/ocis/issues/567) -- [apiWebdavProperties2/getFileProperties.feature:174](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L174) -- [apiWebdavProperties2/getFileProperties.feature:175](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L175) +- [apiWebdavProperties2/getFileProperties.feature:229](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L229) +- [apiWebdavProperties2/getFileProperties.feature:230](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L230) #### [Private link support](https://github.com/owncloud/product/issues/201) #### [oc:privatelink property not returned in webdav responses](https://github.com/owncloud/product/issues/262) -- [apiWebdavProperties2/getFileProperties.feature:232](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L232) -- [apiWebdavProperties2/getFileProperties.feature:233](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L233) +- [apiWebdavProperties2/getFileProperties.feature:295](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L295) +- [apiWebdavProperties2/getFileProperties.feature:296](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L296) #### [changing user quota gives ocs status 103 / Cannot set quota](https://github.com/owncloud/product/issues/247) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ @@ -1322,12 +1324,12 @@ _ocs: api compatibility, return correct status code_ - [apiShareOperationsToShares2/shareAccessByID.feature:163](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareOperationsToShares2/shareAccessByID.feature#L163) #### [[OC-storage] share-types field empty for shared file folder in webdav response](https://github.com/owncloud/ocis/issues/2144) -- [apiWebdavProperties2/getFileProperties.feature:156](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L156) -- [apiWebdavProperties2/getFileProperties.feature:157](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L157) +- [apiWebdavProperties2/getFileProperties.feature:207](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L207) +- [apiWebdavProperties2/getFileProperties.feature:208](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L208) #### [Creating a public link with all permissions(31) fails](https://github.com/owncloud/ocis/issues/2145) -- [apiWebdavProperties2/getFileProperties.feature:206](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L206) -- [apiWebdavProperties2/getFileProperties.feature:207](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L207) +- [apiWebdavProperties2/getFileProperties.feature:265](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L265) +- [apiWebdavProperties2/getFileProperties.feature:266](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties2/getFileProperties.feature#L266) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) - [apiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareUpdateToShares/updateShare.feature#L241) @@ -1378,10 +1380,15 @@ _ocs: api compatibility, return correct status code_ - [apiWebdavOperations/downloadFile.feature:170](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavOperations/downloadFile.feature#L170) #### [Creating a new folder which is a substring of Shares leads to Unknown Error](https://github.com/owncloud/ocis/issues/3033) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) -- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:29](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L29) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:32](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L32) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:45](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L45) +- [apiWebdavProperties1/createFileFolderWhenSharesExist.feature:48](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiWebdavProperties1/createFileFolderWhenSharesExist.feature#L48 + +#### [moveShareInsideAnotherShare behaves differently on oCIS than oC10](https://github.com/owncloud/ocis/issues/3047) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:25](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L25) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:86](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L86) +- [apiShareManagementToShares/moveShareInsideAnotherShare.feature:100](https://github.com/owncloud/core/blob/master/tests/acceptance/features/apiShareManagementToShares/moveShareInsideAnotherShare.feature#L100) Note: always have an empty line at the end of this file. The bash script that processes this file may not process a scenario reference on the last line.