Cache list responses in EOS in case of virtual views

pkg/storage/utils/eosfs/config.go

@@ -111,4 +111,12 @@ type Config struct {
 	// URI of the EOS MGM grpc server
 	// Default is empty
 	GrpcURI string `mapstructure:"master_grpc_uri"`
+
+	// The size of the cache used to store resource infos
+	// Default is 1000000
+	ResourceInfoCacheSize int `mapstructure:"resource_info_cache_size"`
+
+	// The TTL of the resource info cache in seconds
+	// Default is 86400 seconds (1 day)
+	ResourceInfoCacheTTL int `mapstructure:"resource_info_cache_ttl"`
 }

pkg/storage/utils/eosfs/eosfs.go

@@ -25,10 +25,12 @@ import (
 	"io"
 	"net/url"
 	"os"
+	"path/filepath"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 	"sync"
 
 	grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1"
@@ -51,6 +53,7 @@ import (
 	"github.com/cs3org/reva/pkg/storage/utils/templates"
 	"github.com/cs3org/reva/pkg/user"
 	"github.com/pkg/errors"
+	"github.com/bluele/gcache"
 )
 
 const (
@@ -118,6 +121,14 @@ func (c *Config) init() {
 		c.UserLayout = "{{.Username}}" // TODO set better layout
 	}
 
+	if c.ResourceInfoCacheSize == 0 {
+		c.ResourceInfoCacheSize = 1000000
+	}
+
+	if c.ResourceInfoCacheTTL == 0 {
+		c.ResourceInfoCacheTTL = 86400
+	}
+
 	c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
 }
 
@@ -128,6 +139,8 @@ type eosfs struct {
 	singleUserUID string
 	singleUserGID string
 	userIDCache   sync.Map
+	resourceInfoCache      gcache.Cache
+	resourceInfoCacheTTL   time.Duration
 }
 
 // NewEOSFS returns a storage.FS interface implementation that connects to an EOS instance
@@ -179,6 +192,8 @@ func NewEOSFS(c *Config) (storage.FS, error) {
 		conf:         c,
 		chunkHandler: chunking.NewChunkHandler(c.CacheDirectory),
 		userIDCache:  sync.Map{},
+		resourceInfoCache: gcache.New(c.ResourceInfoCacheSize).LFU().Build(),
+		resourceInfoCacheTTL: time.Duration(c.ResourceInfoCacheTTL),
 	}
 
 	return eosfs, nil
@@ -643,6 +658,17 @@ func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*p
 	}
 
 	fn := fs.wrap(ctx, p)
+	if !fs.conf.EnableHome {
+		// EOS file systems are usually sharded with a /username-first-letter/username/resource-path layout
+		// So if the list request is for the first element of this path, it is likely that we're generating
+		// a virtual view. In that case, use cached values as this can be take a long time.
+		rel, err := filepath.Rel(fs.conf.Namespace, fn)
+		if err != nil && rel == filepath.Dir(fn) {
+			if infoIf, err := fs.resourceInfoCache.Get(fn); err == nil {
+				return infoIf.([]*provider.ResourceInfo), nil
+		}
+	}
+}
 
 	eosFileInfos, err := fs.c.List(ctx, uid, gid, fn)
 	if err != nil {
@@ -664,6 +690,7 @@ func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*p
 			finfos = append(finfos, finfo)
 		}
 	}
+	_ = fs.resourceInfoCache.SetWithExpire(fn, finfos, time.Second*fs.resourceInfoCacheTTL)
 
 	return finfos, nil
 }