From 6bf8ee13da368a81d8d98cf426e9b119de04d0be Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 11 Jun 2021 12:17:41 +0200 Subject: [PATCH 01/20] Extend app registry with AddProvider method and mimetype filters --- changelog/unreleased/app-registry-refactor.md | 5 ++ cmd/revad/runtime/loader.go | 1 + go.mod | 2 + go.sum | 3 +- .../grpc/services/appprovider/appprovider.go | 43 ++++++++------- .../services/appprovider/appprovider_test.go | 8 +-- .../grpc/services/appregistry/appregistry.go | 29 ++++++---- internal/grpc/services/gateway/appregistry.go | 55 ++++++++++++++++++- .../grpc/services/gateway/storageprovider.go | 1 + pkg/app/app.go | 11 +--- pkg/app/provider/demo/demo.go | 9 ++- pkg/app/provider/loader/loader.go | 25 +++++++++ pkg/app/provider/registry/registry.go | 34 ++++++++++++ pkg/app/registry/static/static.go | 19 +++---- pkg/auth/manager/registry/registry.go | 2 +- 15 files changed, 188 insertions(+), 59 deletions(-) create mode 100644 changelog/unreleased/app-registry-refactor.md create mode 100644 pkg/app/provider/loader/loader.go create mode 100644 pkg/app/provider/registry/registry.go diff --git a/changelog/unreleased/app-registry-refactor.md b/changelog/unreleased/app-registry-refactor.md new file mode 100644 index 0000000000..bc3eaa842f --- /dev/null +++ b/changelog/unreleased/app-registry-refactor.md @@ -0,0 +1,5 @@ +Enhancement: Extend app registry with AddProvider method and mimetype filters + +https://github.com/cs3org/reva/pull/1785 +https://github.com/cs3org/cs3apis/pull/131 +https://github.com/cs3org/reva/issues/1779 diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index 0b90f93a6a..ba169a1d92 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -27,6 +27,7 @@ import ( _ "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/loader" _ "github.com/cs3org/reva/internal/http/interceptors/loader" _ "github.com/cs3org/reva/internal/http/services/loader" + _ "github.com/cs3org/reva/pkg/app/provider/loader" _ "github.com/cs3org/reva/pkg/appauth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/registry/loader" diff --git a/go.mod b/go.mod index 7e8a20ee9b..87c62b9597 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,8 @@ require ( go 1.16 replace ( + //github.com/cs3org/go-cs3apis => ../cs3apis/build/go-cs3apis + github.com/cs3org/go-cs3apis => github.com/ishank011/go-cs3apis v0.0.0-20210611101519-1497ab8e8312 github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a github.com/oleiade/reflections => github.com/oleiade/reflections v1.0.1 google.golang.org/grpc => google.golang.org/grpc v1.26.0 // temporary downgrade diff --git a/go.sum b/go.sum index e07ae259ea..ee7cdbf984 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20210702091910-85a56bfd027f h1:l09QSEPO8DI3V2hBnc6KhTsrNg/DTyBYjCTwSb/HR6Q= -github.com/cs3org/go-cs3apis v0.0.0-20210702091910-85a56bfd027f/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index b7545f7927..06e41bb7cf 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -33,7 +33,7 @@ import ( providerpb "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" "github.com/cs3org/reva/pkg/app" - "github.com/cs3org/reva/pkg/app/provider/demo" + "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" @@ -55,11 +55,26 @@ type service struct { } type config struct { - Driver string `mapstructure:"driver"` - Demo map[string]interface{} `mapstructure:"demo"` - IopSecret string `mapstructure:"iopsecret" docs:";The iopsecret used to connect to the wopiserver."` - WopiURL string `mapstructure:"wopiurl" docs:";The wopiserver's URL."` - WopiBrURL string `mapstructure:"wopibridgeurl" docs:";The wopibridge's URL."` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` + IopSecret string `mapstructure:"iopsecret" docs:";The iopsecret used to connect to the wopiserver."` + WopiURL string `mapstructure:"wopiurl" docs:";The wopiserver's URL."` + WopiBrURL string `mapstructure:"wopibridgeurl" docs:";The wopibridge's URL."` +} + +func (c *config) init() { + if c.Driver == "" { + c.Driver = "demo" + } +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + c.init() + return c, nil } // New creates a new AppProviderService @@ -85,14 +100,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return service, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func (s *service) Close() error { return nil } @@ -106,12 +113,10 @@ func (s *service) Register(ss *grpc.Server) { } func getProvider(c *config) (app.Provider, error) { - switch c.Driver { - case "demo": - return demo.New(c.Demo) - default: - return nil, errtypes.NotFound("driver not found: " + c.Driver) + if f, ok := registry.NewFuncs[c.Driver]; ok { + return f(c.Drivers[c.Driver]) } + return nil, errtypes.NotFound("driver not found: " + c.Driver) } func (s *service) getWopiAppEndpoints(ctx context.Context) (map[string]interface{}, error) { diff --git a/internal/grpc/services/appprovider/appprovider_test.go b/internal/grpc/services/appprovider/appprovider_test.go index 14b83d0d7c..8950a20a1b 100644 --- a/internal/grpc/services/appprovider/appprovider_test.go +++ b/internal/grpc/services/appprovider/appprovider_test.go @@ -37,13 +37,13 @@ func Test_parseConfig(t *testing.T) { name: "all configurations set", m: map[string]interface{}{ "Driver": "demo", - "Demo": map[string]interface{}{"a": "b", "c": "d"}, + "Drivers": map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, "IopSecret": "very-secret", "WopiURL": "https://my.wopi:9871", }, want: &config{ Driver: "demo", - Demo: map[string]interface{}{"a": "b", "c": "d"}, + Drivers: map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, IopSecret: "very-secret", WopiURL: "https://my.wopi:9871", }, @@ -64,8 +64,8 @@ func Test_parseConfig(t *testing.T) { name: "undefined settings type", m: map[string]interface{}{"Not-Defined": 123}, want: &config{ - Driver: "", - Demo: map[string]interface{}(nil), + Driver: "demo", + Drivers: map[string]map[string]interface{}(nil), IopSecret: "", WopiURL: "", }, diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 3f2f99db6c..93ba74382c 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -102,25 +102,26 @@ func (s *svc) GetAppProviders(ctx context.Context, req *registrypb.GetAppProvide }, nil } - provider := format(p) res := ®istrypb.GetAppProvidersResponse{ Status: status.NewOK(ctx), - Providers: []*registrypb.ProviderInfo{provider}, + Providers: []*registrypb.ProviderInfo{p}, } return res, nil } +func (s *svc) AddAppProvider(ctx context.Context, req *registrypb.AddAppProviderRequest) (*registrypb.AddAppProviderResponse, error) { + return ®istrypb.AddAppProviderResponse{ + Status: status.NewUnimplemented(ctx, errtypes.NotSupported("AddAppProvider not implemented"), "AddAppProvider not implemented"), + }, nil +} + func (s *svc) ListAppProviders(ctx context.Context, req *registrypb.ListAppProvidersRequest) (*registrypb.ListAppProvidersResponse, error) { - pvds, err := s.registry.ListProviders(ctx) + providers, err := s.registry.ListProviders(ctx) if err != nil { return ®istrypb.ListAppProvidersResponse{ Status: status.NewInternal(ctx, err, "error listing the app providers"), }, nil } - providers := make([]*registrypb.ProviderInfo, 0, len(pvds)) - for _, pvd := range pvds { - providers = append(providers, format(pvd)) - } res := ®istrypb.ListAppProvidersResponse{ Status: status.NewOK(ctx), @@ -129,8 +130,14 @@ func (s *svc) ListAppProviders(ctx context.Context, req *registrypb.ListAppProvi return res, nil } -func format(p *app.ProviderInfo) *registrypb.ProviderInfo { - return ®istrypb.ProviderInfo{ - Address: p.Location, - } +func (s *svc) GetDefaultAppProviderForMimeType(ctx context.Context, req *registrypb.GetDefaultAppProviderForMimeTypeRequest) (*registrypb.GetDefaultAppProviderForMimeTypeResponse, error) { + return ®istrypb.GetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewUnimplemented(ctx, errtypes.NotSupported("GetDefaultAppProviderForMimeType not implemented"), "GetDefaultAppProviderForMimeType not implemented"), + }, nil +} + +func (s *svc) SetDefaultAppProviderForMimeType(ctx context.Context, req *registrypb.SetDefaultAppProviderForMimeTypeRequest) (*registrypb.SetDefaultAppProviderForMimeTypeResponse, error) { + return ®istrypb.SetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewUnimplemented(ctx, errtypes.NotSupported("SetDefaultAppProviderForMimeType not implemented"), "SetDefaultAppProviderForMimeType not implemented"), + }, nil } diff --git a/internal/grpc/services/gateway/appregistry.go b/internal/grpc/services/gateway/appregistry.go index a02f18f9de..d08dcd2dc7 100644 --- a/internal/grpc/services/gateway/appregistry.go +++ b/internal/grpc/services/gateway/appregistry.go @@ -38,7 +38,24 @@ func (s *svc) GetAppProviders(ctx context.Context, req *registry.GetAppProviders res, err := c.GetAppProviders(ctx, req) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling ListShares") + return nil, errors.Wrap(err, "gateway: error calling GetAppProviders") + } + + return res, nil +} + +func (s *svc) AddAppProvider(ctx context.Context, req *registry.AddAppProviderRequest) (*registry.AddAppProviderResponse, error) { + c, err := pool.GetAppRegistryClient(s.c.AppRegistryEndpoint) + if err != nil { + err = errors.Wrap(err, "gateway: error calling GetAppRegistryClient") + return ®istry.AddAppProviderResponse{ + Status: status.NewInternal(ctx, err, "error getting user share provider client"), + }, nil + } + + res, err := c.AddAppProvider(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling AddAppProvider") } return res, nil @@ -55,7 +72,41 @@ func (s *svc) ListAppProviders(ctx context.Context, req *registry.ListAppProvide res, err := c.ListAppProviders(ctx, req) if err != nil { - return nil, errors.Wrap(err, "gateway: error calling ListShares") + return nil, errors.Wrap(err, "gateway: error calling ListAppProviders") + } + + return res, nil +} + +func (s *svc) GetDefaultAppProviderForMimeType(ctx context.Context, req *registry.GetDefaultAppProviderForMimeTypeRequest) (*registry.GetDefaultAppProviderForMimeTypeResponse, error) { + c, err := pool.GetAppRegistryClient(s.c.AppRegistryEndpoint) + if err != nil { + err = errors.Wrap(err, "gateway: error calling GetAppRegistryClient") + return ®istry.GetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewInternal(ctx, err, "error getting user share provider client"), + }, nil + } + + res, err := c.GetDefaultAppProviderForMimeType(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling GetDefaultAppProviderForMimeType") + } + + return res, nil +} + +func (s *svc) SetDefaultAppProviderForMimeType(ctx context.Context, req *registry.SetDefaultAppProviderForMimeTypeRequest) (*registry.SetDefaultAppProviderForMimeTypeResponse, error) { + c, err := pool.GetAppRegistryClient(s.c.AppRegistryEndpoint) + if err != nil { + err = errors.Wrap(err, "gateway: error calling GetAppRegistryClient") + return ®istry.SetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewInternal(ctx, err, "error getting user share provider client"), + }, nil + } + + res, err := c.SetDefaultAppProviderForMimeType(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling SetDefaultAppProviderForMimeType") } return res, nil diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 33e5d7ff27..591030418a 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -513,6 +513,7 @@ func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFi func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*gateway.InitiateFileUploadResponse, error) { log := appctx.GetLogger(ctx) + log.Info().Msgf("InitiateFileUpload got %+v", req) p, st := s.getPath(ctx, req.Ref) if st.Code != rpc.Code_CODE_OK { return &gateway.InitiateFileUploadResponse{ diff --git a/pkg/app/app.go b/pkg/app/app.go index 1534094f2d..8e461235b9 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -21,20 +21,15 @@ package app import ( "context" + registry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) // Registry is the interface that application registries implement // for discovering application providers type Registry interface { - FindProvider(ctx context.Context, mimeType string) (*ProviderInfo, error) - ListProviders(ctx context.Context) ([]*ProviderInfo, error) -} - -// ProviderInfo contains the information -// about a Application Provider -type ProviderInfo struct { - Location string + FindProvider(ctx context.Context, mimeType string) (*registry.ProviderInfo, error) + ListProviders(ctx context.Context) ([]*registry.ProviderInfo, error) } // Provider is the interface that application providers implement diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go index 28134083e5..5be990ffd5 100644 --- a/pkg/app/provider/demo/demo.go +++ b/pkg/app/provider/demo/demo.go @@ -25,9 +25,14 @@ import ( "github.com/cs3org/reva/pkg/app" providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/mitchellh/mapstructure" ) +func init() { + registry.Register("demo", New) +} + type provider struct { iframeUIProvider string } @@ -49,8 +54,8 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. +// New returns an implementation to of the app.Provider interface that +// connects to an application in the backend. func New(m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { diff --git a/pkg/app/provider/loader/loader.go b/pkg/app/provider/loader/loader.go new file mode 100644 index 0000000000..f09c752773 --- /dev/null +++ b/pkg/app/provider/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load core application providers. + _ "github.com/cs3org/reva/pkg/app/provider/demo" + // Add your own here +) diff --git a/pkg/app/provider/registry/registry.go b/pkg/app/provider/registry/registry.go new file mode 100644 index 0000000000..43d5ed0188 --- /dev/null +++ b/pkg/app/provider/registry/registry.go @@ -0,0 +1,34 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import "github.com/cs3org/reva/pkg/app" + +// NewFunc is the function that app provider implementations +// should register to at init time. +type NewFunc func(map[string]interface{}) (app.Provider, error) + +// NewFuncs is a map containing all the registered app providers. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new app provider new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/app/registry/static/static.go b/pkg/app/registry/static/static.go index 6b419098d8..0d3f7ecbc9 100644 --- a/pkg/app/registry/static/static.go +++ b/pkg/app/registry/static/static.go @@ -22,6 +22,7 @@ import ( "context" "strings" + registrypb "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/sharedconf" @@ -40,18 +41,18 @@ func (c *config) init() { } } -func (b *registry) ListProviders(ctx context.Context) ([]*app.ProviderInfo, error) { - var providers = make([]*app.ProviderInfo, 0, len(b.rules)) +func (b *registry) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { + var providers = make([]*registrypb.ProviderInfo, 0, len(b.rules)) for _, address := range b.rules { - providers = append(providers, &app.ProviderInfo{ - Location: address, + providers = append(providers, ®istrypb.ProviderInfo{ + Address: address, }) } return providers, nil } -func (b *registry) FindProvider(ctx context.Context, mimeType string) (*app.ProviderInfo, error) { - // find longest match +func (b *registry) FindProvider(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) { + // find the longest match var match string for prefix := range b.rules { @@ -64,8 +65,8 @@ func (b *registry) FindProvider(ctx context.Context, mimeType string) (*app.Prov return nil, errtypes.NotFound("application provider not found for mime type " + mimeType) } - p := &app.ProviderInfo{ - Location: b.rules[match], + p := ®istrypb.ProviderInfo{ + Address: b.rules[match], } return p, nil } @@ -82,8 +83,6 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. func New(m map[string]interface{}) (app.Registry, error) { c, err := parseConfig(m) if err != nil { diff --git a/pkg/auth/manager/registry/registry.go b/pkg/auth/manager/registry/registry.go index 37a164ae1b..52141f3d07 100644 --- a/pkg/auth/manager/registry/registry.go +++ b/pkg/auth/manager/registry/registry.go @@ -21,7 +21,7 @@ package registry import "github.com/cs3org/reva/pkg/auth" // NewFunc is the function that auth implementations -// should register at init time. +// should register to at init time. type NewFunc func(map[string]interface{}) (auth.Manager, error) // NewFuncs is a map containing all the registered auth managers. From 07bea67bbbcfa8e5cf2de097742b427b616155e0 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Fri, 11 Jun 2021 12:37:26 +0200 Subject: [PATCH 02/20] Add a registry for app registry pkg --- cmd/revad/runtime/loader.go | 1 + go.mod | 1 - .../grpc/services/appregistry/appregistry.go | 30 +++++----- .../services/appregistry/appregistry_test.go | 8 +-- .../grpc/services/gateway/storageprovider.go | 1 - pkg/app/registry/loader/loader.go | 25 +++++++++ pkg/app/registry/registry/registry.go | 34 ++++++++++++ pkg/app/registry/static/static.go | 55 ++++++++++--------- pkg/auth/registry/registry/registry.go | 6 +- 9 files changed, 114 insertions(+), 47 deletions(-) create mode 100644 pkg/app/registry/loader/loader.go create mode 100644 pkg/app/registry/registry/registry.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index ba169a1d92..241f625bae 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -28,6 +28,7 @@ import ( _ "github.com/cs3org/reva/internal/http/interceptors/loader" _ "github.com/cs3org/reva/internal/http/services/loader" _ "github.com/cs3org/reva/pkg/app/provider/loader" + _ "github.com/cs3org/reva/pkg/app/registry/loader" _ "github.com/cs3org/reva/pkg/appauth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/registry/loader" diff --git a/go.mod b/go.mod index 87c62b9597..85a3e1b126 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,6 @@ require ( go 1.16 replace ( - //github.com/cs3org/go-cs3apis => ../cs3apis/build/go-cs3apis github.com/cs3org/go-cs3apis => github.com/ishank011/go-cs3apis v0.0.0-20210611101519-1497ab8e8312 github.com/eventials/go-tus => github.com/andrewmostello/go-tus v0.0.0-20200314041820-904a9904af9a github.com/oleiade/reflections => github.com/oleiade/reflections v1.0.1 diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 93ba74382c..74449f183b 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -25,7 +25,7 @@ import ( registrypb "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" "github.com/cs3org/reva/pkg/app" - "github.com/cs3org/reva/pkg/app/registry/static" + "github.com/cs3org/reva/pkg/app/registry/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" @@ -37,7 +37,7 @@ func init() { } type svc struct { - registry app.Registry + reg app.Registry } func (s *svc) Close() error { @@ -53,8 +53,14 @@ func (s *svc) Register(ss *grpc.Server) { } type config struct { - Driver string `mapstructure:"driver"` - Static map[string]interface{} `mapstructure:"static"` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` +} + +func (c *config) init() { + if c.Driver == "" { + c.Driver = "static" + } } // New creates a new StorageRegistryService @@ -65,13 +71,13 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } - registry, err := getRegistry(c) + reg, err := getRegistry(c) if err != nil { return nil, err } svc := &svc{ - registry: registry, + reg: reg, } return svc, nil @@ -86,16 +92,14 @@ func parseConfig(m map[string]interface{}) (*config, error) { } func getRegistry(c *config) (app.Registry, error) { - switch c.Driver { - case "static": - return static.New(c.Static) - default: - return nil, errtypes.NotFound("driver not found: " + c.Driver) + if f, ok := registry.NewFuncs[c.Driver]; ok { + return f(c.Drivers[c.Driver]) } + return nil, errtypes.NotFound("appregistrysvc: driver not found: " + c.Driver) } func (s *svc) GetAppProviders(ctx context.Context, req *registrypb.GetAppProvidersRequest) (*registrypb.GetAppProvidersResponse, error) { - p, err := s.registry.FindProvider(ctx, req.ResourceInfo.MimeType) + p, err := s.reg.FindProvider(ctx, req.ResourceInfo.MimeType) if err != nil { return ®istrypb.GetAppProvidersResponse{ Status: status.NewInternal(ctx, err, "error looking for the app provider"), @@ -116,7 +120,7 @@ func (s *svc) AddAppProvider(ctx context.Context, req *registrypb.AddAppProvider } func (s *svc) ListAppProviders(ctx context.Context, req *registrypb.ListAppProvidersRequest) (*registrypb.ListAppProvidersResponse, error) { - providers, err := s.registry.ListProviders(ctx) + providers, err := s.reg.ListProviders(ctx) if err != nil { return ®istrypb.ListAppProvidersResponse{ Status: status.NewInternal(ctx, err, "error listing the app providers"), diff --git a/internal/grpc/services/appregistry/appregistry_test.go b/internal/grpc/services/appregistry/appregistry_test.go index b2c7cb4e9c..158e1e282d 100644 --- a/internal/grpc/services/appregistry/appregistry_test.go +++ b/internal/grpc/services/appregistry/appregistry_test.go @@ -120,7 +120,7 @@ func Test_ListAppProviders(t *testing.T) { } ss := &svc{ - registry: rr, + reg: rr, } got, err := ss.ListAppProviders(context.Background(), nil) @@ -237,7 +237,7 @@ func Test_GetAppProviders(t *testing.T) { } ss := &svc{ - registry: rr, + reg: rr, } for _, tt := range tests { @@ -274,12 +274,12 @@ func TestNew(t *testing.T) { { name: "not existing driver", m: map[string]interface{}{"Driver": "doesnotexist"}, - wantErr: "error: not found: driver not found: doesnotexist", + wantErr: "error: not found: appregistrysvc: driver not found: doesnotexist", }, { name: "empty", m: map[string]interface{}{}, - wantErr: "error: not found: driver not found: ", + wantErr: "error: not found: appregistrysvc: driver not found: ", }, { name: "extra not existing field in setting", diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 591030418a..33e5d7ff27 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -513,7 +513,6 @@ func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFi func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*gateway.InitiateFileUploadResponse, error) { log := appctx.GetLogger(ctx) - log.Info().Msgf("InitiateFileUpload got %+v", req) p, st := s.getPath(ctx, req.Ref) if st.Code != rpc.Code_CODE_OK { return &gateway.InitiateFileUploadResponse{ diff --git a/pkg/app/registry/loader/loader.go b/pkg/app/registry/loader/loader.go new file mode 100644 index 0000000000..7c06a89f92 --- /dev/null +++ b/pkg/app/registry/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load core app registry drivers. + _ "github.com/cs3org/reva/pkg/app/registry/static" + // Add your own here +) diff --git a/pkg/app/registry/registry/registry.go b/pkg/app/registry/registry/registry.go new file mode 100644 index 0000000000..0650a2a75f --- /dev/null +++ b/pkg/app/registry/registry/registry.go @@ -0,0 +1,34 @@ +// Copyright 2018-2021 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import "github.com/cs3org/reva/pkg/app" + +// NewFunc is the function that app provider implementations +// should register to at init time. +type NewFunc func(map[string]interface{}) (app.Registry, error) + +// NewFuncs is a map containing all the registered app registry backends. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new app registry new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/app/registry/static/static.go b/pkg/app/registry/static/static.go index 0d3f7ecbc9..bfde41d7ef 100644 --- a/pkg/app/registry/static/static.go +++ b/pkg/app/registry/static/static.go @@ -24,13 +24,18 @@ import ( registrypb "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" "github.com/cs3org/reva/pkg/app" + "github.com/cs3org/reva/pkg/app/registry/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/sharedconf" "github.com/mitchellh/mapstructure" ) -type registry struct { - rules map[string]string +func init() { + registry.Register("static", New) +} + +type config struct { + Rules map[string]string `mapstructure:"rules"` } func (c *config) init() { @@ -41,7 +46,28 @@ func (c *config) init() { } } -func (b *registry) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + return c, nil +} + +type reg struct { + rules map[string]string +} + +func New(m map[string]interface{}) (app.Registry, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + c.init() + return ®{rules: c.Rules}, nil +} + +func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { var providers = make([]*registrypb.ProviderInfo, 0, len(b.rules)) for _, address := range b.rules { providers = append(providers, ®istrypb.ProviderInfo{ @@ -51,7 +77,7 @@ func (b *registry) ListProviders(ctx context.Context) ([]*registrypb.ProviderInf return providers, nil } -func (b *registry) FindProvider(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) { +func (b *reg) FindProvider(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) { // find the longest match var match string @@ -70,24 +96,3 @@ func (b *registry) FindProvider(ctx context.Context, mimeType string) (*registry } return p, nil } - -type config struct { - Rules map[string]string -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - -func New(m map[string]interface{}) (app.Registry, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - c.init() - return ®istry{rules: c.Rules}, nil -} diff --git a/pkg/auth/registry/registry/registry.go b/pkg/auth/registry/registry/registry.go index 48e13898fc..0170885f37 100644 --- a/pkg/auth/registry/registry/registry.go +++ b/pkg/auth/registry/registry/registry.go @@ -20,14 +20,14 @@ package registry import "github.com/cs3org/reva/pkg/auth" -// NewFunc is the function that auth provider implementations +// NewFunc is the function that auth registry implementations // should register at init time. type NewFunc func(map[string]interface{}) (auth.Registry, error) -// NewFuncs is a map containing all the registered auth backends. +// NewFuncs is a map containing all the registered auth registries. var NewFuncs = map[string]NewFunc{} -// Register registers a new auth provider new function. +// Register registers a new auth registry new function. // Not safe for concurrent use. Safe for use from package init. func Register(name string, f NewFunc) { NewFuncs[name] = f From 914b6902dea17547d72aa9f445c0680ad9a78575 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 15 Jun 2021 18:03:43 +0200 Subject: [PATCH 03/20] Refactor static registry --- .../grpc/services/appregistry/appregistry.go | 5 +- .../services/appregistry/appregistry_test.go | 102 ++++++++++------ pkg/app/app.go | 5 +- pkg/app/registry/static/static.go | 113 ++++++++++++++---- 4 files changed, 165 insertions(+), 60 deletions(-) diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 74449f183b..419cb03757 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -88,6 +88,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { if err := mapstructure.Decode(m, c); err != nil { return nil, err } + c.init() return c, nil } @@ -99,7 +100,7 @@ func getRegistry(c *config) (app.Registry, error) { } func (s *svc) GetAppProviders(ctx context.Context, req *registrypb.GetAppProvidersRequest) (*registrypb.GetAppProvidersResponse, error) { - p, err := s.reg.FindProvider(ctx, req.ResourceInfo.MimeType) + p, err := s.reg.FindProviders(ctx, req.ResourceInfo.MimeType) if err != nil { return ®istrypb.GetAppProvidersResponse{ Status: status.NewInternal(ctx, err, "error looking for the app provider"), @@ -108,7 +109,7 @@ func (s *svc) GetAppProviders(ctx context.Context, req *registrypb.GetAppProvide res := ®istrypb.GetAppProvidersResponse{ Status: status.NewOK(ctx), - Providers: []*registrypb.ProviderInfo{p}, + Providers: p, } return res, nil } diff --git a/internal/grpc/services/appregistry/appregistry_test.go b/internal/grpc/services/appregistry/appregistry_test.go index 158e1e282d..684a1c3ce6 100644 --- a/internal/grpc/services/appregistry/appregistry_test.go +++ b/internal/grpc/services/appregistry/appregistry_test.go @@ -38,15 +38,21 @@ func (a ByAddress) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func Test_ListAppProviders(t *testing.T) { tests := []struct { - name string - rules map[string]interface{} - want *registrypb.ListAppProvidersResponse + name string + providers map[string]interface{} + want *registrypb.ListAppProvidersResponse }{ { name: "simple test", - rules: map[string]interface{}{ - "text/json": "some Address", - "currently/ignored": "an other address", + providers: map[string]interface{}{ + "some Address": map[string]interface{}{ + "address": "some Address", + "mimetypes": []string{"text/json"}, + }, + "another address": map[string]interface{}{ + "address": "another address", + "mimetypes": []string{"currently/ignored"}, + }, }, // only Status and Providers will be asserted in the tests @@ -57,27 +63,36 @@ func Test_ListAppProviders(t *testing.T) { Message: "", }, Providers: []*registrypb.ProviderInfo{ - {Address: "some Address"}, - {Address: "an other address"}, + { + Address: "some Address", + MimeTypes: []string{"text/json"}, + }, + { + Address: "another address", + MimeTypes: []string{"currently/ignored"}, + }, }, }, }, { - name: "rules is nil", - rules: nil, + name: "providers is nil", + providers: nil, want: ®istrypb.ListAppProvidersResponse{ Status: &rpcv1beta1.Status{ Code: 1, Trace: "00000000000000000000000000000000", }, Providers: []*registrypb.ProviderInfo{ - {Address: ""}, + { + Address: "", + MimeTypes: []string{"text/plain"}, + }, }, }, }, { - name: "empty rules", - rules: map[string]interface{}{}, + name: "empty providers", + providers: map[string]interface{}{}, // only Status and Providers will be asserted in the tests want: ®istrypb.ListAppProvidersResponse{ @@ -87,14 +102,17 @@ func Test_ListAppProviders(t *testing.T) { Message: "", }, Providers: []*registrypb.ProviderInfo{ - {Address: ""}, + { + Address: "", + MimeTypes: []string{"text/plain"}, + }, }, }, }, { - name: "rule value is nil", - rules: map[string]interface{}{ - "text/json": nil, + name: "provider value is nil", + providers: map[string]interface{}{ + "some Address": nil, }, // only Status and Providers will be asserted in the tests @@ -104,16 +122,14 @@ func Test_ListAppProviders(t *testing.T) { Trace: "00000000000000000000000000000000", Message: "", }, - Providers: []*registrypb.ProviderInfo{ - {Address: ""}, - }, + Providers: []*registrypb.ProviderInfo{nil}, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rr, err := static.New(map[string]interface{}{"Rules": tt.rules}) + rr, err := static.New(map[string]interface{}{"Providers": tt.providers}) if err != nil { t.Errorf("could not create registry error = %v", err) return @@ -137,13 +153,19 @@ func Test_ListAppProviders(t *testing.T) { } func Test_GetAppProviders(t *testing.T) { - rules := map[string]interface{}{ - "text/json": "JSON format", - "image/bmp": "Windows OS/2 Bitmap Graphics", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "Microsoft Word (OpenXML)", - "application/vnd.oasis.opendocument.presentation": "OpenDocument presentation document", - "application/vnd.apple.installer+xml": "Apple Installer Package", - "text/xml": "XML", + providers := map[string]interface{}{ + "text appprovider addr": map[string]interface{}{ + "address": "text appprovider addr", + "mimetypes": []string{"text/json", "text/xml"}, + }, + "image appprovider addr": map[string]interface{}{ + "address": "image appprovider addr", + "mimetypes": []string{"image/bmp"}, + }, + "misc appprovider addr": map[string]interface{}{ + "address": "misc appprovider addr", + "mimetypes": []string{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.presentation", "application/vnd.apple.installer+xml"}, + }, } tests := []struct { @@ -162,7 +184,10 @@ func Test_GetAppProviders(t *testing.T) { Message: "", }, Providers: []*registrypb.ProviderInfo{ - {Address: "JSON format"}, + { + Address: "text appprovider addr", + MimeTypes: []string{"text/json", "text/xml"}, + }, }, }, }, @@ -176,7 +201,10 @@ func Test_GetAppProviders(t *testing.T) { Message: "", }, Providers: []*registrypb.ProviderInfo{ - {Address: "Apple Installer Package"}, + { + Address: "misc appprovider addr", + MimeTypes: []string{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.presentation", "application/vnd.apple.installer+xml"}, + }, }, }, }, @@ -230,7 +258,7 @@ func Test_GetAppProviders(t *testing.T) { }, } - rr, err := static.New(map[string]interface{}{"Rules": rules}) + rr, err := static.New(map[string]interface{}{"providers": providers}) if err != nil { t.Errorf("could not create registry error = %v", err) return @@ -260,11 +288,11 @@ func Test_GetAppProviders(t *testing.T) { func TestNew(t *testing.T) { tests := []struct { - name string - m map[string]interface{} - rules map[string]interface{} - want svc - wantErr interface{} + name string + m map[string]interface{} + providers map[string]interface{} + want svc + wantErr interface{} }{ { name: "no error", @@ -279,7 +307,7 @@ func TestNew(t *testing.T) { { name: "empty", m: map[string]interface{}{}, - wantErr: "error: not found: appregistrysvc: driver not found: ", + wantErr: nil, }, { name: "extra not existing field in setting", diff --git a/pkg/app/app.go b/pkg/app/app.go index 8e461235b9..7516465bad 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -28,8 +28,11 @@ import ( // Registry is the interface that application registries implement // for discovering application providers type Registry interface { - FindProvider(ctx context.Context, mimeType string) (*registry.ProviderInfo, error) + FindProviders(ctx context.Context, mimeType string) ([]*registry.ProviderInfo, error) ListProviders(ctx context.Context) ([]*registry.ProviderInfo, error) + AddProvider(ctx context.Context, p *registry.ProviderInfo) error + GetDefaultProviderForMimeType(ctx context.Context, mimeType string) (*registry.ProviderInfo, error) + SetDefaultProviderForMimeType(ctx context.Context, mimeType string, p *registry.ProviderInfo) error } // Provider is the interface that application providers implement diff --git a/pkg/app/registry/static/static.go b/pkg/app/registry/static/static.go index bfde41d7ef..e2472ea548 100644 --- a/pkg/app/registry/static/static.go +++ b/pkg/app/registry/static/static.go @@ -35,13 +35,16 @@ func init() { } type config struct { - Rules map[string]string `mapstructure:"rules"` + Providers map[string]*registrypb.ProviderInfo `mapstructure:"providers"` } func (c *config) init() { - if len(c.Rules) == 0 { - c.Rules = map[string]string{ - "text/plain": sharedconf.GetGatewaySVC(""), + if len(c.Providers) == 0 { + c.Providers = map[string]*registrypb.ProviderInfo{ + sharedconf.GetGatewaySVC(""): ®istrypb.ProviderInfo{ + Address: sharedconf.GetGatewaySVC(""), + MimeTypes: []string{"text/plain"}, + }, } } } @@ -54,34 +57,49 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } +type mimeTypeIndex struct { + defaultApp string + apps []string +} + type reg struct { - rules map[string]string + providers map[string]*registrypb.ProviderInfo + mimetypes map[string]mimeTypeIndex // map the mime type to the addresses of the corresponding providers } +// New returns an implementation of the app.Registry interface. func New(m map[string]interface{}) (app.Registry, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - return ®{rules: c.Rules}, nil -} -func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { - var providers = make([]*registrypb.ProviderInfo, 0, len(b.rules)) - for _, address := range b.rules { - providers = append(providers, ®istrypb.ProviderInfo{ - Address: address, - }) + newReg := reg{ + providers: c.Providers, + mimetypes: make(map[string]mimeTypeIndex), } - return providers, nil + + for addr, p := range c.Providers { + if p != nil { + for _, m := range p.MimeTypes { + idx, ok := newReg.mimetypes[m] + if ok { + idx.apps = append(idx.apps, addr) + } else { + newReg.mimetypes[m] = mimeTypeIndex{apps: []string{addr}} + } + } + } + } + return &newReg, nil } -func (b *reg) FindProvider(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) { - // find the longest match +func (b *reg) FindProviders(ctx context.Context, mimeType string) ([]*registrypb.ProviderInfo, error) { + // find longest match var match string - for prefix := range b.rules { + for prefix := range b.mimetypes { if strings.HasPrefix(mimeType, prefix) && len(prefix) > len(match) { match = prefix } @@ -91,8 +109,63 @@ func (b *reg) FindProvider(ctx context.Context, mimeType string) (*registrypb.Pr return nil, errtypes.NotFound("application provider not found for mime type " + mimeType) } - p := ®istrypb.ProviderInfo{ - Address: b.rules[match], + var providers = make([]*registrypb.ProviderInfo, 0, len(b.mimetypes[match].apps)) + for _, p := range b.mimetypes[match].apps { + providers = append(providers, b.providers[p]) + } + return providers, nil +} + +func (b *reg) AddProvider(ctx context.Context, p *registrypb.ProviderInfo) error { + b.providers[p.Address] = p + + for _, m := range p.MimeTypes { + idx, ok := b.mimetypes[m] + if ok { + idx.apps = append(idx.apps, p.Address) + } else { + b.mimetypes[m] = mimeTypeIndex{apps: []string{p.Address}} + } + } + return nil +} + +func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { + var providers = make([]*registrypb.ProviderInfo, 0, len(b.providers)) + for _, p := range b.providers { + providers = append(providers, p) + } + return providers, nil +} + +func (b *reg) SetDefaultProviderForMimeType(ctx context.Context, mimeType string, p *registrypb.ProviderInfo) error { + idx, ok := b.mimetypes[mimeType] + if ok { + idx.defaultApp = p.Address + // Add to list of apps if not present + var present bool + for _, pr := range idx.apps { + if pr == p.Address { + present = true + break + } + } + if !present { + idx.apps = append(idx.apps, p.Address) + } + } else { + b.mimetypes[mimeType] = mimeTypeIndex{apps: []string{p.Address}, defaultApp: p.Address} + } + return nil +} + +func (b *reg) GetDefaultProviderForMimeType(ctx context.Context, mimeType string) (*registrypb.ProviderInfo, error) { + idx, ok := b.mimetypes[mimeType] + if ok { + p, ok := b.providers[idx.defaultApp] + if ok { + return p, nil + } } - return p, nil + return nil, errtypes.NotFound("default application provider not set for mime type " + mimeType) } From 56d9e86fe9e744d077411d03ffe3e179c705dd87 Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 15 Jun 2021 18:08:55 +0200 Subject: [PATCH 04/20] Forward requests from stubs to pkg --- .../grpc/services/appregistry/appregistry.go | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 419cb03757..49418e15a3 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -115,9 +115,17 @@ func (s *svc) GetAppProviders(ctx context.Context, req *registrypb.GetAppProvide } func (s *svc) AddAppProvider(ctx context.Context, req *registrypb.AddAppProviderRequest) (*registrypb.AddAppProviderResponse, error) { - return ®istrypb.AddAppProviderResponse{ - Status: status.NewUnimplemented(ctx, errtypes.NotSupported("AddAppProvider not implemented"), "AddAppProvider not implemented"), - }, nil + err := s.reg.AddProvider(ctx, req.Provider) + if err != nil { + return ®istrypb.AddAppProviderResponse{ + Status: status.NewInternal(ctx, err, "error adding the app provider"), + }, nil + } + + res := ®istrypb.AddAppProviderResponse{ + Status: status.NewOK(ctx), + } + return res, nil } func (s *svc) ListAppProviders(ctx context.Context, req *registrypb.ListAppProvidersRequest) (*registrypb.ListAppProvidersResponse, error) { @@ -136,13 +144,30 @@ func (s *svc) ListAppProviders(ctx context.Context, req *registrypb.ListAppProvi } func (s *svc) GetDefaultAppProviderForMimeType(ctx context.Context, req *registrypb.GetDefaultAppProviderForMimeTypeRequest) (*registrypb.GetDefaultAppProviderForMimeTypeResponse, error) { - return ®istrypb.GetDefaultAppProviderForMimeTypeResponse{ - Status: status.NewUnimplemented(ctx, errtypes.NotSupported("GetDefaultAppProviderForMimeType not implemented"), "GetDefaultAppProviderForMimeType not implemented"), - }, nil + provider, err := s.reg.GetDefaultProviderForMimeType(ctx, req.MimeType) + if err != nil { + return ®istrypb.GetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewInternal(ctx, err, "error getting the default app provider for the mimetype"), + }, nil + } + + res := ®istrypb.GetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewOK(ctx), + Provider: provider, + } + return res, nil } func (s *svc) SetDefaultAppProviderForMimeType(ctx context.Context, req *registrypb.SetDefaultAppProviderForMimeTypeRequest) (*registrypb.SetDefaultAppProviderForMimeTypeResponse, error) { - return ®istrypb.SetDefaultAppProviderForMimeTypeResponse{ - Status: status.NewUnimplemented(ctx, errtypes.NotSupported("SetDefaultAppProviderForMimeType not implemented"), "SetDefaultAppProviderForMimeType not implemented"), - }, nil + err := s.reg.SetDefaultProviderForMimeType(ctx, req.MimeType, req.Provider) + if err != nil { + return ®istrypb.SetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewInternal(ctx, err, "error setting the default app provider for the mimetype"), + }, nil + } + + res := ®istrypb.SetDefaultAppProviderForMimeTypeResponse{ + Status: status.NewOK(ctx), + } + return res, nil } From 47ebec4803f630628318cbc871bcdf0df1ee5c6a Mon Sep 17 00:00:00 2001 From: Ishank Arora Date: Tue, 15 Jun 2021 18:58:07 +0200 Subject: [PATCH 05/20] Move wopi-specific code to its driver --- .../grpc/services/appprovider/appprovider.go | 192 +------------- .../services/appprovider/appprovider_test.go | 35 +-- pkg/app/app.go | 5 +- pkg/app/provider/demo/demo.go | 11 +- pkg/app/provider/loader/loader.go | 1 + pkg/app/provider/wopi/wopi.go | 237 ++++++++++++++++++ 6 files changed, 277 insertions(+), 204 deletions(-) create mode 100644 pkg/app/provider/wopi/wopi.go diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 06e41bb7cf..4cc2e02014 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -19,28 +19,16 @@ package appprovider import ( - "bytes" "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "path" - "strings" - "time" providerpb "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" - "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/cs3org/reva/pkg/rhttp" - "github.com/cs3org/reva/pkg/user" "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -50,16 +38,12 @@ func init() { type service struct { provider app.Provider - client *http.Client conf *config } type config struct { - Driver string `mapstructure:"driver"` - Drivers map[string]map[string]interface{} `mapstructure:"drivers"` - IopSecret string `mapstructure:"iopsecret" docs:";The iopsecret used to connect to the wopiserver."` - WopiURL string `mapstructure:"wopiurl" docs:";The wopiserver's URL."` - WopiBrURL string `mapstructure:"wopibridgeurl" docs:";The wopibridge's URL."` + Driver string `mapstructure:"driver"` + Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } func (c *config) init() { @@ -92,9 +76,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, provider: provider, - client: rhttp.GetHTTPClient( - rhttp.Timeout(5 * time.Second), - ), } return service, nil @@ -119,174 +100,21 @@ func getProvider(c *config) (app.Provider, error) { return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func (s *service) getWopiAppEndpoints(ctx context.Context) (map[string]interface{}, error) { - // TODO this query will eventually be served by Reva. - // For the time being it is a remnant of the CERNBox-specific WOPI server, which justifies the /cbox path in the URL. - wopiurl, err := url.Parse(s.conf.WopiURL) - if err != nil { - return nil, err - } - wopiurl.Path = path.Join(wopiurl.Path, "/wopi/cbox/endpoints") - appsReq, err := rhttp.NewRequest(ctx, "GET", wopiurl.String(), nil) - if err != nil { - return nil, err - } - appsRes, err := s.client.Do(appsReq) - if err != nil { - return nil, err - } - defer appsRes.Body.Close() - if appsRes.StatusCode != http.StatusOK { - return nil, errtypes.InternalError(fmt.Sprintf("Request to WOPI server returned %d", appsRes.StatusCode)) - } - appsBody, err := ioutil.ReadAll(appsRes.Body) - if err != nil { - return nil, err - } - - appsURLMap := make(map[string]interface{}) - err = json.Unmarshal(appsBody, &appsURLMap) - if err != nil { - return nil, err - } - - log := appctx.GetLogger(ctx) - log.Info().Msg(fmt.Sprintf("Successfully retrieved %d WOPI app endpoints", len(appsURLMap))) - return appsURLMap, nil -} - func (s *service) OpenInApp(ctx context.Context, req *providerpb.OpenInAppRequest) (*providerpb.OpenInAppResponse, error) { - - log := appctx.GetLogger(ctx) - - wopiurl, err := url.Parse(s.conf.WopiURL) - if err != nil { - return nil, err - } - wopiurl.Path = path.Join(wopiurl.Path, "/wopi/iop/open") - httpReq, err := rhttp.NewRequest(ctx, "GET", wopiurl.String(), nil) - if err != nil { - return nil, err - } - - q := httpReq.URL.Query() - q.Add("fileid", req.ResourceInfo.GetId().OpaqueId) - q.Add("endpoint", req.ResourceInfo.GetId().StorageId) - q.Add("viewmode", req.ViewMode.String()) - // TODO the folder URL should be resolved as e.g. `'https://cernbox.cern.ch/index.php/apps/files/?dir=' + filepath.Dir(req.Ref.GetPath())` - // or should be deprecated/removed altogether, needs discussion and decision. - q.Add("folderurl", "undefined") - u, ok := user.ContextGetUser(ctx) - if ok { - q.Add("username", u.Username) - } - // else defaults to "Anonymous Guest" - - if s.conf.IopSecret == "" { - s.conf.IopSecret = os.Getenv("REVA_APPPROVIDER_IOPSECRET") - } - - httpReq.Header.Set("Authorization", "Bearer "+s.conf.IopSecret) - httpReq.Header.Set("TokenHeader", req.AccessToken) - - httpReq.URL.RawQuery = q.Encode() - - openRes, err := s.client.Do(httpReq) - + appURL, err := s.provider.GetAppURL(ctx, req.ResourceInfo, req.ViewMode, req.App, req.AccessToken) if err != nil { + err := errors.Wrap(err, "appprovider: error calling GetAppURL") res := &providerpb.OpenInAppResponse{ - Status: status.NewInternal(ctx, err, "appprovider: error performing open request to WOPI"), + Status: status.NewInternal(ctx, err, "error getting app URL"), } return res, nil } - defer openRes.Body.Close() - - if openRes.StatusCode != http.StatusOK { - res := &providerpb.OpenInAppResponse{ - Status: status.NewInvalid(ctx, fmt.Sprintf("appprovider: error performing open request to WOPI, status code: %d", openRes.StatusCode)), - } - return res, nil - } - - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(openRes.Body) - if err != nil { - return nil, err - } - openResBody := buf.String() - - var viewmode string - if req.ViewMode == providerpb.OpenInAppRequest_VIEW_MODE_READ_WRITE { - viewmode = "edit" - } else { - viewmode = "view" - } - - var appProviderURL string - if req.App == "" { - // Default behavior: work out the application URL to be used for this file - // TODO call this e.g. once a day or a week, and cache the content in a shared map protected by a multi-reader Lock - appsURLMap, err := s.getWopiAppEndpoints(ctx) - if err != nil { - res := &providerpb.OpenInAppResponse{ - Status: status.NewInternal(ctx, err, "appprovider: getWopiAppEndpoints failed"), - } - return res, nil - } - viewOptions := appsURLMap[path.Ext(req.ResourceInfo.GetPath())] - viewOptionsMap, ok := viewOptions.(map[string]interface{}) - if !ok { - res := &providerpb.OpenInAppResponse{ - Status: status.NewInvalid(ctx, "Incorrect parsing of the App URLs map from the WOPI server"), - } - return res, nil - } - - appProviderURL = fmt.Sprintf("%v", viewOptionsMap[viewmode]) - if strings.Contains(appProviderURL, "?") { - appProviderURL += "&" - } else { - appProviderURL += "?" - } - appProviderURL = fmt.Sprintf("%sWOPISrc=%s", appProviderURL, openResBody) - } else { - // User specified the application to use, generate the URL out of that - // TODO map the given req.App to the URL via config. For now assume it's a URL! - appProviderURL = fmt.Sprintf("%sWOPISrc=%s", req.App, openResBody) - } - - // In case of applications served by the WOPI bridge, resolve the URL and go to the app - // Note that URL matching is performed via string matching, not via IP resolution: may need to fix this - if len(s.conf.WopiBrURL) > 0 && strings.Contains(appProviderURL, s.conf.WopiBrURL) { - httpClient := rhttp.GetHTTPClient( - rhttp.Context(ctx), - rhttp.Timeout(time.Duration(5*int64(time.Second))), - ) - httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { - // do not follow a redirect - return http.ErrUseLastResponse - } - - bridgeReq, err := rhttp.NewRequest(ctx, "GET", appProviderURL, nil) - if err != nil { - return nil, err - } - bridgeRes, err := httpClient.Do(bridgeReq) - if err != nil { - return nil, err - } - defer bridgeRes.Body.Close() - if bridgeRes.StatusCode != http.StatusFound { - return nil, errtypes.InternalError(fmt.Sprintf("Request to WOPI bridge returned %d", bridgeRes.StatusCode)) - } - appProviderURL = bridgeRes.Header.Get("Location") + res := &providerpb.OpenInAppResponse{ + Status: status.NewOK(ctx), + AppUrl: appURL, } + return res, nil - log.Info().Msg(fmt.Sprintf("Returning app provider URL %s", appProviderURL)) - return &providerpb.OpenInAppResponse{ - Status: status.NewOK(ctx), - AppUrl: appProviderURL, - }, nil } func (s *service) OpenFileInAppProvider(ctx context.Context, req *providerpb.OpenFileInAppProviderRequest) (*providerpb.OpenFileInAppProviderResponse, error) { diff --git a/internal/grpc/services/appprovider/appprovider_test.go b/internal/grpc/services/appprovider/appprovider_test.go index 8950a20a1b..4d76286565 100644 --- a/internal/grpc/services/appprovider/appprovider_test.go +++ b/internal/grpc/services/appprovider/appprovider_test.go @@ -34,29 +34,36 @@ func Test_parseConfig(t *testing.T) { wantErr interface{} }{ { - name: "all configurations set", + name: "all configurations set for demo driver", m: map[string]interface{}{ - "Driver": "demo", - "Drivers": map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, - "IopSecret": "very-secret", - "WopiURL": "https://my.wopi:9871", + "Driver": "demo", + "Drivers": map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, }, want: &config{ - Driver: "demo", - Drivers: map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, - IopSecret: "very-secret", - WopiURL: "https://my.wopi:9871", + Driver: "demo", + Drivers: map[string]map[string]interface{}{"demo": map[string]interface{}{"a": "b", "c": "d"}}, + }, + wantErr: nil, + }, + { + name: "all configurations set for wopi driver", + m: map[string]interface{}{ + "Driver": "wopi", + "Drivers": map[string]map[string]interface{}{"wopi": map[string]interface{}{"iop_secret": "very-secret", "wopi_url": "https://my.wopi:9871"}}, + }, + want: &config{ + Driver: "wopi", + Drivers: map[string]map[string]interface{}{"wopi": map[string]interface{}{"iop_secret": "very-secret", "wopi_url": "https://my.wopi:9871"}}, }, wantErr: nil, }, { name: "wrong type of setting", - m: map[string]interface{}{"Driver": 123, "IopSecret": 456}, + m: map[string]interface{}{"Driver": 123, "NonExistentField": 456}, want: nil, wantErr: &mapstructure.Error{ Errors: []string{ "'driver' expected type 'string', got unconvertible type 'int', value: '123'", - "'iopsecret' expected type 'string', got unconvertible type 'int', value: '456'", }, }, }, @@ -64,10 +71,8 @@ func Test_parseConfig(t *testing.T) { name: "undefined settings type", m: map[string]interface{}{"Not-Defined": 123}, want: &config{ - Driver: "demo", - Drivers: map[string]map[string]interface{}(nil), - IopSecret: "", - WopiURL: "", + Driver: "demo", + Drivers: map[string]map[string]interface{}(nil), }, wantErr: nil, }, diff --git a/pkg/app/app.go b/pkg/app/app.go index 7516465bad..efe5faf676 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -21,6 +21,7 @@ package app import ( "context" + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" ) @@ -36,7 +37,7 @@ type Registry interface { } // Provider is the interface that application providers implement -// for providing the iframe location to a iframe UI Provider +// for providing the URL of the app which will serve the requested resource. type Provider interface { - GetIFrame(ctx context.Context, resID *provider.Reference, token string) (string, error) + GetAppURL(ctx context.Context, resource *provider.ResourceInfo, viewMode appprovider.OpenInAppRequest_ViewMode, app, token string) (string, error) } diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go index 5be990ffd5..9fcaadbcb9 100644 --- a/pkg/app/provider/demo/demo.go +++ b/pkg/app/provider/demo/demo.go @@ -24,7 +24,8 @@ import ( "github.com/cs3org/reva/pkg/app" - providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/app/provider/registry" "github.com/mitchellh/mapstructure" ) @@ -33,12 +34,12 @@ func init() { registry.Register("demo", New) } -type provider struct { +type demoProvider struct { iframeUIProvider string } -func (p *provider) GetIFrame(ctx context.Context, resID *providerpb.Reference, token string) (string, error) { - msg := fmt.Sprintf("