Dynamic storage provider add aliases (#4307)

* Extract regex expansion into utils package

* Add alias resolution for storage ids

* Linting

* Add changelog

* Remove stray log

changelog/unreleased/storage-registry-aliases.md

@@ -0,0 +1,7 @@
+Bugfix: Dynamic storage registry storage_id aliases
+
+Fixes the bug where the dynamic storage registry would not be able to
+resolve storage ids like `eoshome-a`, as those are aliased and need to
+be resolved into the proper storage-id (`eoshome-i01`).
+
+https://github.com/cs3org/reva/pull/4307

pkg/storage/registry/dynamic/dynamic.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cs3org/reva/pkg/storage/registry/dynamic/rewriter"
 	"github.com/cs3org/reva/pkg/storage/registry/dynamic/routingtree"
 	"github.com/cs3org/reva/pkg/storage/registry/registry"
+	"github.com/cs3org/reva/pkg/storage/registry/utils"
 	"github.com/cs3org/reva/pkg/utils/cfg"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -42,16 +43,18 @@ func init() {
 }
 
 type dynamic struct {
-	c   *config
-	log *zerolog.Logger
-	r   map[string]string
-	rt  *routingtree.RoutingTree
-	ur  *rewriter.UserRewriter
+	c       *config
+	log     *zerolog.Logger
+	aliases map[string]string
+	r       map[string]string
+	rt      *routingtree.RoutingTree
+	ur      *rewriter.UserRewriter
 }
 
 type config struct {
-	Rules      map[string]string `docs:"nil;A map from mountID to provider address"                       mapstructure:"rules"`
-	Rewrites   map[string]string `docs:"nil;A map from a path to an template alias to use when resolving" mapstructure:"rewrites"`
+	Rules      map[string]string `docs:"nil;A map from mountID to provider address"                          mapstructure:"rules"`
+	Rewrites   map[string]string `docs:"nil;A map from a path to an template alias to use when resolving"    mapstructure:"rewrites"`
+	IDAliases  map[string]string `docs:"nil;A map containing storageID aliases, can contain simple brackets" mapstructure:"aliases"`
 	HomePath   string            `mapstructure:"home_path"`
 	DBUsername string            `mapstructure:"db_username"`
 	DBPassword string            `mapstructure:"db_password"`
@@ -76,11 +79,14 @@ func New(ctx context.Context, m map[string]interface{}) (storage.Registry, error
 		return nil, errors.Wrap(err, "error initializing routing tree")
 	}
 
+	aliases := initAliases(c.IDAliases)
+
 	d := &dynamic{
-		c:   &c,
-		log: &annotatedLog,
-		r:   c.Rules,
-		rt:  rt,
+		c:       &c,
+		log:     &annotatedLog,
+		r:       c.Rules,
+		aliases: aliases,
+		rt:      rt,
 		ur: &rewriter.UserRewriter{
 			Tpls: c.Rewrites,
 		},
@@ -114,6 +120,19 @@ func initRoutingTree(dbUsername, dbPassword, dbHost string, dbPort int, dbName s
 	return routingtree.New(rs), nil
 }
 
+func initAliases(aliasesConfig map[string]string) map[string]string {
+	aliases := make(map[string]string)
+
+	for aliasRegex, storageID := range aliasesConfig {
+		expandedAliases := utils.GenerateRegexCombinations(aliasRegex)
+		for _, alias := range expandedAliases {
+			aliases[alias] = storageID
+		}
+	}
+
+	return aliases
+}
+
 // ListProviders lists all available storage providers.
 func (d *dynamic) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, error) {
 	providers := make([]*registrypb.ProviderInfo, 0, len(d.r))
@@ -149,17 +168,24 @@ func (d *dynamic) GetHome(ctx context.Context) (*registrypb.ProviderInfo, error)
 func (d *dynamic) FindProviders(ctx context.Context, ref *provider.Reference) ([]*registrypb.ProviderInfo, error) {
 	l := d.log.With().Interface("ref", ref).Logger()
 
-	if ref.ResourceId != nil {
-		if ref.ResourceId.StorageId != "" {
-			if address, ok := d.r[ref.ResourceId.StorageId]; ok {
-				return []*registrypb.ProviderInfo{{
-					ProviderId: ref.ResourceId.StorageId,
-					Address:    address,
-				}}, nil
-			}
+	l.Trace().Msg("Finding providers")
+
+	if ref.ResourceId != nil && ref.ResourceId.StorageId != "" {
+		storageID := ref.ResourceId.StorageId
+
+		// Resolve aliased storageIDs (like eoshome-a into eoshome-i01)
+		if i, ok := d.aliases[storageID]; ok {
+			storageID = i
+		}
 
-			return nil, errtypes.NotFound("storage provider not found for ref " + ref.String())
+		if address, ok := d.r[storageID]; ok {
+			return []*registrypb.ProviderInfo{{
+				ProviderId: ref.ResourceId.StorageId,
+				Address:    address,
+			}}, nil
 		}
+
+		return nil, errtypes.NotFound("storage provider not found for ref " + ref.String())
 	}
 
 	providerAlias := d.ur.GetAlias(ctx, ref.Path)

pkg/storage/registry/static/static.go

@@ -32,6 +32,7 @@ import (
 	"github.com/cs3org/reva/pkg/sharedconf"
 	"github.com/cs3org/reva/pkg/storage"
 	"github.com/cs3org/reva/pkg/storage/registry/registry"
+	"github.com/cs3org/reva/pkg/storage/registry/utils"
 	"github.com/cs3org/reva/pkg/storage/utils/templates"
 	"github.com/cs3org/reva/pkg/utils/cfg"
 	"github.com/pkg/errors"
@@ -41,8 +42,6 @@ func init() {
 	registry.Register("static", New)
 }
 
-var bracketRegex = regexp.MustCompile(`\[(.*?)\]`)
-
 type rule struct {
 	Mapping string            `mapstructure:"mapping"`
 	Address string            `mapstructure:"address"`
@@ -104,7 +103,7 @@ func (b *reg) ListProviders(ctx context.Context) ([]*registrypb.ProviderInfo, er
 	providers := []*registrypb.ProviderInfo{}
 	for k, v := range b.c.Rules {
 		if addr := getProviderAddr(ctx, v); addr != "" {
-			combs := generateRegexCombinations(k)
+			combs := utils.GenerateRegexCombinations(k)
 			for _, c := range combs {
 				providers = append(providers, &registrypb.ProviderInfo{
 					ProviderPath: c,
@@ -182,7 +181,7 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re
 			}
 			// Check if the current rule forms a part of a reference spread across storage providers.
 			if strings.HasPrefix(prefix, fn) {
-				combs := generateRegexCombinations(prefix)
+				combs := utils.GenerateRegexCombinations(prefix)
 				for _, c := range combs {
 					shardedMatches = append(shardedMatches, &registrypb.ProviderInfo{
 						ProviderPath: c,
@@ -203,25 +202,3 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re
 
 	return nil, errtypes.NotFound("storage provider not found for ref " + ref.String())
 }
-
-func generateRegexCombinations(rex string) []string {
-	m := bracketRegex.FindString(rex)
-	r := strings.Trim(strings.Trim(m, "["), "]")
-	if r == "" {
-		return []string{rex}
-	}
-	var combinations []string
-	for i := 0; i < len(r); i++ {
-		if i < len(r)-2 && r[i+1] == '-' {
-			for j := r[i]; j <= r[i+2]; j++ {
-				p := strings.Replace(rex, m, string(j), 1)
-				combinations = append(combinations, generateRegexCombinations(p)...)
-			}
-			i += 2
-		} else {
-			p := strings.Replace(rex, m, string(r[i]), 1)
-			combinations = append(combinations, generateRegexCombinations(p)...)
-		}
-	}
-	return combinations
-}

pkg/storage/registry/utils/utils.go

@@ -0,0 +1,49 @@
+// Copyright 2018-2023 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+// Package utils contains utilities for storage registries
+package utils
+
+import (
+	"regexp"
+	"strings"
+)
+
+// GenerateRegexCombinations expands bracket regexes.
+func GenerateRegexCombinations(rex string) []string {
+	var bracketRegex = regexp.MustCompile(`\[(.*?)\]`)
+	m := bracketRegex.FindString(rex)
+	r := strings.Trim(strings.Trim(m, "["), "]")
+	if r == "" {
+		return []string{rex}
+	}
+	var combinations []string
+	for i := 0; i < len(r); i++ {
+		if i < len(r)-2 && r[i+1] == '-' {
+			for j := r[i]; j <= r[i+2]; j++ {
+				p := strings.Replace(rex, m, string(j), 1)
+				combinations = append(combinations, GenerateRegexCombinations(p)...)
+			}
+			i += 2
+		} else {
+			p := strings.Replace(rex, m, string(r[i]), 1)
+			combinations = append(combinations, GenerateRegexCombinations(p)...)
+		}
+	}
+	return combinations
+}