From 4ec0eb320e093f39a0da421abc3a33955348e56b Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Wed, 1 Nov 2023 14:20:31 +0000 Subject: [PATCH] Dynamic storage provider add aliases (#4307) * Extract regex expansion into utils package * Add alias resolution for storage ids * Linting * Add changelog * Remove stray log --- .../unreleased/storage-registry-aliases.md | 7 ++ pkg/storage/registry/dynamic/dynamic.go | 66 +++++++++++++------ pkg/storage/registry/static/static.go | 29 +------- pkg/storage/registry/utils/utils.go | 49 ++++++++++++++ 4 files changed, 105 insertions(+), 46 deletions(-) create mode 100644 changelog/unreleased/storage-registry-aliases.md create mode 100644 pkg/storage/registry/utils/utils.go diff --git a/changelog/unreleased/storage-registry-aliases.md b/changelog/unreleased/storage-registry-aliases.md new file mode 100644 index 0000000000..4f4ee6f268 --- /dev/null +++ b/changelog/unreleased/storage-registry-aliases.md @@ -0,0 +1,7 @@ +Bugfix: Dynamic storage registry storage_id aliases + +Fixes the bug where the dynamic storage registry would not be able to +resolve storage ids like `eoshome-a`, as those are aliased and need to +be resolved into the proper storage-id (`eoshome-i01`). + +https://github.com/cs3org/reva/pull/4307 diff --git a/pkg/storage/registry/dynamic/dynamic.go b/pkg/storage/registry/dynamic/dynamic.go index 32053250a4..f87c527eca 100644 --- a/pkg/storage/registry/dynamic/dynamic.go +++ b/pkg/storage/registry/dynamic/dynamic.go @@ -32,6 +32,7 @@ import ( "github.com/cs3org/reva/pkg/storage/registry/dynamic/rewriter" "github.com/cs3org/reva/pkg/storage/registry/dynamic/routingtree" "github.com/cs3org/reva/pkg/storage/registry/registry" + "github.com/cs3org/reva/pkg/storage/registry/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -42,16 +43,18 @@ func init() { } type dynamic struct { - c *config - log *zerolog.Logger - r map[string]string - rt *routingtree.RoutingTree - ur *rewriter.UserRewriter + c *config + log *zerolog.Logger + aliases map[string]string + r map[string]string + rt *routingtree.RoutingTree + ur *rewriter.UserRewriter } type config struct { - Rules map[string]string `docs:"nil;A map from mountID to provider address" mapstructure:"rules"` - Rewrites map[string]string `docs:"nil;A map from a path to an template alias to use when resolving" mapstructure:"rewrites"` + Rules map[string]string `docs:"nil;A map from mountID to provider address" mapstructure:"rules"` + Rewrites map[string]string `docs:"nil;A map from a path to an template alias to use when resolving" mapstructure:"rewrites"` + IDAliases map[string]string `docs:"nil;A map containing storageID aliases, can contain simple brackets" mapstructure:"aliases"` HomePath string `mapstructure:"home_path"` DBUsername string `mapstructure:"db_username"` DBPassword string `mapstructure:"db_password"` @@ -76,11 +79,14 @@ func New(ctx context.Context, m map[string]interface{}) (storage.Registry, error return nil, errors.Wrap(err, "error initializing routing tree") } + aliases := initAliases(c.IDAliases) + d := &dynamic{ - c: &c, - log: &annotatedLog, - r: c.Rules, - rt: rt, + c: &c, + log: &annotatedLog, + r: c.Rules, + aliases: aliases, + rt: rt, ur: &rewriter.UserRewriter{ Tpls: c.Rewrites, }, @@ -114,6 +120,19 @@ func initRoutingTree(dbUsername, dbPassword, dbHost string, dbPort int, dbName s return routingtree.New(rs), nil } +func initAliases(aliasesConfig map[string]string) map[string]string { + aliases := make(map[string]string) + + for aliasRegex, storageID := range aliasesConfig { + expandedAliases := utils.GenerateRegexCombinations(aliasRegex) + for _, alias := range expandedAliases { + aliases[alias] = storageID + } + } + + return aliases +} + // ListProviders lists all available storage providers. func (d *dynamic) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) { providers := make([]*registrypb.ProviderInfo, 0, len(d.r)) @@ -149,17 +168,24 @@ func (d *dynamic) GetHome(ctx context.Context) (*registrypb.ProviderInfo, error) func (d *dynamic) FindProviders(ctx context.Context, ref *provider.Reference) ([]*registrypb.ProviderInfo, error) { l := d.log.With().Interface("ref", ref).Logger() - if ref.ResourceId != nil { - if ref.ResourceId.StorageId != "" { - if address, ok := d.r[ref.ResourceId.StorageId]; ok { - return []*registrypb.ProviderInfo{{ - ProviderId: ref.ResourceId.StorageId, - Address: address, - }}, nil - } + l.Trace().Msg("Finding providers") + + if ref.ResourceId != nil && ref.ResourceId.StorageId != "" { + storageID := ref.ResourceId.StorageId + + // Resolve aliased storageIDs (like eoshome-a into eoshome-i01) + if i, ok := d.aliases[storageID]; ok { + storageID = i + } - return nil, errtypes.NotFound("storage provider not found for ref " + ref.String()) + if address, ok := d.r[storageID]; ok { + return []*registrypb.ProviderInfo{{ + ProviderId: ref.ResourceId.StorageId, + Address: address, + }}, nil } + + return nil, errtypes.NotFound("storage provider not found for ref " + ref.String()) } providerAlias := d.ur.GetAlias(ctx, ref.Path) diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index 02d2889a37..f27fa976c1 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -32,6 +32,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/registry/registry" + "github.com/cs3org/reva/pkg/storage/registry/utils" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" @@ -41,8 +42,6 @@ func init() { registry.Register("static", New) } -var bracketRegex = regexp.MustCompile(`\[(.*?)\]`) - type rule struct { Mapping string `mapstructure:"mapping"` Address string `mapstructure:"address"` @@ -104,7 +103,7 @@ func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, er providers := []*registrypb.ProviderInfo{} for k, v := range b.c.Rules { if addr := getProviderAddr(ctx, v); addr != "" { - combs := generateRegexCombinations(k) + combs := utils.GenerateRegexCombinations(k) for _, c := range combs { providers = append(providers, ®istrypb.ProviderInfo{ ProviderPath: c, @@ -182,7 +181,7 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re } // Check if the current rule forms a part of a reference spread across storage providers. if strings.HasPrefix(prefix, fn) { - combs := generateRegexCombinations(prefix) + combs := utils.GenerateRegexCombinations(prefix) for _, c := range combs { shardedMatches = append(shardedMatches, ®istrypb.ProviderInfo{ ProviderPath: c, @@ -203,25 +202,3 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re return nil, errtypes.NotFound("storage provider not found for ref " + ref.String()) } - -func generateRegexCombinations(rex string) []string { - m := bracketRegex.FindString(rex) - r := strings.Trim(strings.Trim(m, "["), "]") - if r == "" { - return []string{rex} - } - var combinations []string - for i := 0; i < len(r); i++ { - if i < len(r)-2 && r[i+1] == '-' { - for j := r[i]; j <= r[i+2]; j++ { - p := strings.Replace(rex, m, string(j), 1) - combinations = append(combinations, generateRegexCombinations(p)...) - } - i += 2 - } else { - p := strings.Replace(rex, m, string(r[i]), 1) - combinations = append(combinations, generateRegexCombinations(p)...) - } - } - return combinations -} diff --git a/pkg/storage/registry/utils/utils.go b/pkg/storage/registry/utils/utils.go new file mode 100644 index 0000000000..a9589c2055 --- /dev/null +++ b/pkg/storage/registry/utils/utils.go @@ -0,0 +1,49 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package utils contains utilities for storage registries +package utils + +import ( + "regexp" + "strings" +) + +// GenerateRegexCombinations expands bracket regexes. +func GenerateRegexCombinations(rex string) []string { + var bracketRegex = regexp.MustCompile(`\[(.*?)\]`) + m := bracketRegex.FindString(rex) + r := strings.Trim(strings.Trim(m, "["), "]") + if r == "" { + return []string{rex} + } + var combinations []string + for i := 0; i < len(r); i++ { + if i < len(r)-2 && r[i+1] == '-' { + for j := r[i]; j <= r[i+2]; j++ { + p := strings.Replace(rex, m, string(j), 1) + combinations = append(combinations, GenerateRegexCombinations(p)...) + } + i += 2 + } else { + p := strings.Replace(rex, m, string(r[i]), 1) + combinations = append(combinations, GenerateRegexCombinations(p)...) + } + } + return combinations +}