Merge branch 'master' into eos-lock

.dockerignore

@@ -1,7 +1,6 @@
 .github
 changelog
 docs
-examples
 grpc-tests
 tests
 tools

.drone.env

@@ -1,3 +1,3 @@
 # The test runner source for API tests
-CORE_COMMITID=09d584745d6cbd6aebc557d9b78f6130e9b99e2b
+CORE_COMMITID=0dadfbe475438dd97c192cb93643ef8d95b71faa
 CORE_BRANCH=master

.drone.star

@@ -229,6 +229,26 @@ def buildAndPublishDocker():
                     ],
                 },
             },
+            {
+                "name": "publish-docker-revad-ceph-latest",
+                "pull": "always",
+                "image": "plugins/docker",
+                "settings": {
+                    "repo": "cs3org/revad",
+                    "tags": "latest-ceph",
+                    "dockerfile": "Dockerfile.revad-ceph",
+                    "username": {
+                        "from_secret": "dockerhub_username",
+                    },
+                    "password": {
+                        "from_secret": "dockerhub_password",
+                    },
+                    "custom_dns": [
+                        "128.142.17.5",
+                        "128.142.16.5",
+                    ],
+                },
+            },
         ],
     }
 
@@ -481,6 +501,26 @@ def release():
                     ],
                 },
             },
+            {
+                "name": "docker-revad-ceph-tag",
+                "pull": "always",
+                "image": "plugins/docker",
+                "settings": {
+                    "repo": "cs3org/revad",
+                    "tags": "${DRONE_TAG}-ceph",
+                    "dockerfile": "Dockerfile.revad-ceph",
+                    "username": {
+                        "from_secret": "dockerhub_username",
+                    },
+                    "password": {
+                        "from_secret": "dockerhub_password",
+                    },
+                    "custom_dns": [
+                        "128.142.17.5",
+                        "128.142.16.5",
+                    ],
+                },
+            },
         ],
         "depends_on": ["changelog"],
     }
@@ -679,6 +719,7 @@ def litmusOcisSpacesDav():
                     "/drone/src/cmd/revad/revad -c gateway.toml &",
                     "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &",
                     "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &",
+                    "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
                     "/drone/src/cmd/revad/revad -c users.toml",
                 ],
             },
@@ -767,7 +808,7 @@ def ocisIntegrationTests(parallelRuns, skipExceptParts = []):
                             "REVA_LDAP_HOSTNAME": "ldap",
                             "TEST_REVA": "true",
                             "SEND_SCENARIO_LINE_REFERENCES": "true",
-                            "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis",
+                            "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023",
                             "DIVIDE_INTO_NUM_PARTS": parallelRuns,
                             "RUN_PART": runPart,
                             "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
@@ -842,7 +883,7 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):
                             "REVA_LDAP_HOSTNAME": "ldap",
                             "TEST_REVA": "true",
                             "SEND_SCENARIO_LINE_REFERENCES": "true",
-                            "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis",
+                            "BEHAT_FILTER_TAGS": "~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis&&~@personalSpace&&~@issue-ocis-3023",
                             "DIVIDE_INTO_NUM_PARTS": parallelRuns,
                             "RUN_PART": runPart,
                             "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md",

Dockerfile.revad-ceph

@@ -0,0 +1,52 @@
+# Copyright 2018-2021 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# In applying this license, CERN does not waive the privileges and immunities
+# granted to it by virtue of its status as an Intergovernmental Organization
+# or submit itself to any jurisdiction.
+
+FROM ceph/daemon-base
+
+RUN dnf update -y && dnf install -y \
+  git \
+  gcc \
+  make \
+  libcephfs-devel \
+  librbd-devel \
+  librados-devel
+
+ADD https://golang.org/dl/go1.16.4.linux-amd64.tar.gz \
+  go1.16.4.linux-amd64.tar.gz
+
+RUN rm -rf /usr/local/go && \
+  tar -C /usr/local -xzf go1.16.4.linux-amd64.tar.gz && \
+  rm go1.16.4.linux-amd64.tar.gz
+
+ENV PATH /go/bin:/usr/local/go/bin:$PATH
+ENV GOPATH /go
+
+WORKDIR /go/src/github/cs3org/reva
+COPY . .
+RUN mkdir -p /go/bin \
+    make build-revad-cephfs-docker && \
+    cp /go/src/github/cs3org/reva/cmd/revad/revad /usr/bin/revad
+
+RUN cp -r examples/ceph /etc/
+
+RUN mkdir -p /etc/revad/ && echo "" > /etc/revad/revad.toml
+
+EXPOSE 9999 10000
+
+ENTRYPOINT [ "/usr/bin/revad" ]
+CMD [ "-c", "/etc/revad/revad.toml", "-p", "/var/run/revad.pid" ]

Makefile

@@ -30,17 +30,19 @@ off:
 imports: off
 	`go env GOPATH`/bin/goimports -w tools pkg internal cmd
 
-build: imports test-go-version
-	gofmt -s -w .
-	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
-	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
+build: build-revad build-reva test-go-version
+
+build-cephfs: build-revad-cephfs build-reva
 
 tidy:
 	go mod tidy
 
 build-revad: imports
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
 
+build-revad-cephfs: imports
+	go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad
+
 build-reva: imports
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
 
@@ -104,6 +106,8 @@ ci: build-ci test  lint-ci
 # to be run in Docker build
 build-revad-docker: off
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/revad/revad ./cmd/revad
+build-revad-cephfs-docker: off
+	go build -ldflags ${BUILD_FLAGS} -tags ceph -o ./cmd/revad/revad ./cmd/revad
 build-reva-docker: off
 	go build -ldflags ${BUILD_FLAGS} -o ./cmd/reva/reva ./cmd/reva
 clean:

changelog/unreleased/cephfs-driver.md

@@ -0,0 +1,3 @@
+Enhancement: Reva CephFS module v0.2.1
+
+https://github.com/cs3org/reva/pull/1209

changelog/unreleased/cs3-permissions-service.md

@@ -0,0 +1,5 @@
+Enhancement: Use CS3 permissions API
+
+Added calls to the CS3 permissions API to the decomposedfs in order to check the user permissions.
+
+https://github.com/cs3org/reva/pull/2341

changelog/unreleased/share-create-perm-fix.md

@@ -0,0 +1,7 @@
+Bugfix: Use ocs permission objects in the reva GRPC client
+
+There was a bug introduced by differing CS3APIs permission definitions
+for the same role across services. This is a first step in making
+all services use consistent definitions.
+
+https://github.com/cs3org/reva/pull/2478

cmd/reva/common.go

@@ -31,7 +31,6 @@ import (
 
 const (
 	viewerPermission string = "viewer"
-	readerPermission string = "reader"
 	editorPermission string = "editor"
 	collabPermission string = "collab"
 	denyPermission   string = "denied"

cmd/reva/share-create.go

@@ -28,6 +28,7 @@ import (
 	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions"
 	"github.com/cs3org/reva/pkg/utils"
 	"github.com/jedib0t/go-pretty/table"
 	"github.com/pkg/errors"
@@ -158,48 +159,11 @@ func getGrantType(t string) provider.GranteeType {
 func getSharePerm(p string) (*provider.ResourcePermissions, error) {
 	switch p {
 	case viewerPermission:
-		return &provider.ResourcePermissions{
-			GetPath:       true,
-			ListContainer: true,
-			Stat:          true,
-		}, nil
-	case readerPermission:
-		return &provider.ResourcePermissions{
-			GetPath:              true,
-			InitiateFileDownload: true,
-			ListFileVersions:     true,
-			ListContainer:        true,
-			Stat:                 true,
-		}, nil
+		return conversions.NewViewerRole().CS3ResourcePermissions(), nil
 	case editorPermission:
-		return &provider.ResourcePermissions{
-			GetPath:              true,
-			InitiateFileDownload: true,
-			ListFileVersions:     true,
-			ListContainer:        true,
-			Stat:                 true,
-			CreateContainer:      true,
-			Delete:               true,
-			InitiateFileUpload:   true,
-			RestoreFileVersion:   true,
-			Move:                 true,
-		}, nil
+		return conversions.NewEditorRole().CS3ResourcePermissions(), nil
 	case collabPermission:
-		return &provider.ResourcePermissions{
-			GetPath:              true,
-			InitiateFileDownload: true,
-			ListFileVersions:     true,
-			ListContainer:        true,
-			Stat:                 true,
-			CreateContainer:      true,
-			Delete:               true,
-			InitiateFileUpload:   true,
-			RestoreFileVersion:   true,
-			Move:                 true,
-			AddGrant:             true,
-			UpdateGrant:          true,
-			RemoveGrant:          true,
-		}, nil
+		return conversions.NewCoownerRole().CS3ResourcePermissions(), nil
 	case denyPermission:
 		return &provider.ResourcePermissions{}, nil
 	default:

cmd/revad/runtime/loader.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/share/manager/loader"
+	_ "github.com/cs3org/reva/pkg/permission/manager/loader"
 	_ "github.com/cs3org/reva/pkg/publicshare/manager/loader"
 	_ "github.com/cs3org/reva/pkg/rhttp/datatx/manager/loader"
 	_ "github.com/cs3org/reva/pkg/share/cache/loader"

docs/content/en/docs/config/packages/storage/fs/cephfs/_index.md

@@ -0,0 +1,26 @@
+---
+title: "cephfs"
+linkTitle: "cephfs"
+weight: 10
+description: >
+  Configuration for the cephfs service
+---
+
+# _struct: config_
+
+{{% dir name="root" type="string" default="/var/tmp/reva/" %}}
+Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L34)
+{{< highlight toml >}}
+[storage.fs.cephfs]
+root = "/var/tmp/reva/"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="share_folder" type="string" default="/MyShares" %}}
+Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/cephfs/cephfs.go#L35)
+{{< highlight toml >}}
+[storage.fs.cephfs]
+share_folder = "/MyShares"
+{{< /highlight >}}
+{{% /dir %}}
+

examples/ceph/ceph.conf

@@ -0,0 +1,3 @@
+[global]
+  fsid = '8aaa35c4-75dc-42e5-a812-cbc1cdfd3323'
+  mon_host = '[v2:188.184.96.178:3300/0,v1:188.184.96.178:6789/0] [v2:188.185.88.76:3300/0,v1:188.185.88.76:6789/0] [v2:188.185.126.6:3300/0,v1:188.185.126.6:6789/0]'

examples/ceph/keyring

@@ -0,0 +1,2 @@
+[client.admin]
+  key = 'AQAu88Fg5iekGhAAeVP0Td05PuybytuRJgBRqA=='

go.mod

@@ -3,21 +3,23 @@ module github.com/cs3org/reva
 require (
 	bou.ke/monkey v1.0.2
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0
-	github.com/BurntSushi/toml v0.4.1
+	github.com/BurntSushi/toml v1.0.0
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/ReneKroon/ttlcache/v2 v2.11.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go v1.42.27
+	github.com/aws/aws-sdk-go v1.42.39
 	github.com/beevik/etree v1.1.0
 	github.com/bluele/gcache v0.0.2
 	github.com/c-bata/go-prompt v0.2.5
+	github.com/ceph/go-ceph v0.13.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e
 	github.com/cs3org/go-cs3apis v0.0.0-20220126114148-64c025ccdd19
 	github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8
+	github.com/dgraph-io/ristretto v0.1.0
 	github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59
 	github.com/gdexlab/go-render v1.0.1
 	github.com/go-chi/chi/v5 v5.0.7
@@ -28,25 +30,26 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.2
 	github.com/gomodule/redigo v1.8.8
-	github.com/google/go-cmp v0.5.6
+	github.com/google/go-cmp v0.5.7
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
-	github.com/hashicorp/go-hclog v1.0.0
+	github.com/hashicorp/go-hclog v1.1.0
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jedib0t/go-pretty v4.3.0+incompatible
 	github.com/juliangruber/go-intersect v1.1.0
-	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/mattn/go-sqlite3 v1.14.10
+	github.com/maxymania/go-system v0.0.0-20170110133659-647cc364bf0b
 	github.com/mileusna/useragent v1.0.2
-	github.com/minio/minio-go/v7 v7.0.20
+	github.com/minio/minio-go/v7 v7.0.21
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.17.0
+	github.com/onsi/gomega v1.18.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.4
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
@@ -71,7 +74,8 @@ require (
 	go.opentelemetry.io/otel/trace v1.3.0
 	golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
-	golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
 	golang.org/x/term v0.0.0-20210916214954-140adaaadfaf
 	google.golang.org/genproto v0.0.0-20211021150943-2b146023228c
 	google.golang.org/grpc v1.43.0