feat: redis dedup backend

THIRD-PARTY-LICENSES.md

@@ -176,7 +176,7 @@ github.com/go-pkgz/expirable-cache|https://github.com/go-pkgz/expirable-cache/bl
 github.com/go-playground/locales|https://github.com/go-playground/locales/blob/v0.14.1/LICENSE|MIT
 github.com/go-playground/universal-translator|https://github.com/go-playground/universal-translator/blob/v0.18.1/LICENSE|MIT
 github.com/go-playground/validator/v10|https://github.com/go-playground/validator/blob/v10.15.1/LICENSE|MIT
-github.com/go-redis/redis/v8|https://github.com/go-redis/redis/blob/v8.11.5/LICENSE|BSD-2-Clause
+github.com/go-redis/redis/v9|https://github.com/redis/go-redis/blob/v9.3.0/LICENSE|BSD-2-Clause
 github.com/gobwas/glob|https://github.com/gobwas/glob/blob/v0.2.3/LICENSE|MIT
 github.com/goccy/go-json|https://github.com/goccy/go-json/blob/v0.10.2/LICENSE|MIT
 github.com/gofrs/uuid|https://github.com/gofrs/uuid/blob/v4.4.0/LICENSE|MIT

go.mod

@@ -44,6 +44,7 @@ require (
 )
 
 require (
+	github.com/alicebob/miniredis/v2 v2.30.4
 	github.com/aquasecurity/trivy v0.46.1
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.23.0
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.6
@@ -57,6 +58,7 @@ require (
 	github.com/notaryproject/notation-go v1.0.0
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230117141039-067a0f5b0e25
 	github.com/project-zot/mockoidc v0.0.0-20230307111146-f607b4b5fb97
+	github.com/redis/go-redis/v9 v9.3.0
 	github.com/sigstore/cosign/v2 v2.2.0
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/zitadel/oidc v1.13.5
@@ -93,6 +95,7 @@ require (
 	github.com/Masterminds/squirrel v1.5.4 // indirect
 	github.com/Microsoft/hcsshim v0.12.0-rc.0 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
+	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
 	github.com/aquasecurity/defsec v0.93.1 // indirect
 	github.com/aquasecurity/table v1.8.0 // indirect
@@ -209,6 +212,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yuin/gopher-lua v1.1.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect

go.sum

@@ -363,7 +363,9 @@ github.com/alibabacloud-go/tea-utils v1.4.4/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5
 github.com/alibabacloud-go/tea-xml v1.1.2 h1:oLxa7JUXm2EDFzMg+7oRsYc+kutgCVwm+bZlhhmvW5M=
 github.com/alibabacloud-go/tea-xml v1.1.2/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis/v2 v2.30.4 h1:8S4/o1/KoUArAGbGwPxcwf0krlzceva2XVOSchFS7Eo=
+github.com/alicebob/miniredis/v2 v2.30.4/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg=
 github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw=
 github.com/aliyun/credentials-go v1.2.3 h1:Vmodnr52Rz1mcbwn0kzMhLRKb6soizewuKXdfZiNemU=
 github.com/aliyun/credentials-go v1.2.3/go.mod h1:/KowD1cfGSLrLsH28Jr8W+xwoId0ywIy5lNzDz6O1vw=
@@ -510,6 +512,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM
 github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A=
 github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE=
 github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
@@ -1472,6 +1476,8 @@ github.com/puzpuzpuz/xsync/v2 v2.4.1 h1:aGdE1C/HaR/QC6YAFdtZXi60Df8/qBIrs8PKrzkI
 github.com/puzpuzpuz/xsync/v2 v2.4.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
+github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -1717,6 +1723,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
+github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY=
@@ -1988,6 +1995,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/storage/cache.go

@@ -64,6 +64,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache
 		{
 			return cache.NewDynamoDBCache(parameters, log), nil
 		}
+	case "redis":
+		{
+			return cache.NewRedisCache(parameters, log), nil
+		}
 	default:
 		{
 			return nil, errors.ErrBadConfig

pkg/storage/cache/redis.go

@@ -0,0 +1,196 @@
+package cache
+
+import (
+	"context"
+	goerrors "errors"
+	"path/filepath"
+	"strings"
+
+	godigest "github.com/opencontainers/go-digest"
+	"github.com/redis/go-redis/v9"
+
+	"zotregistry.io/zot/errors"
+	zlog "zotregistry.io/zot/pkg/log"
+	"zotregistry.io/zot/pkg/storage/constants"
+)
+
+type RedisDriver struct {
+	rootDir     string
+	db          redis.UniversalClient
+	log         zlog.Logger
+	useRelPaths bool // whether or not to use relative paths, should be true for filesystem and false for s3
+}
+
+type RedisDriverParameters struct {
+	RootDir     string
+	Url         string // https://github.com/redis/redis-specifications/blob/master/uri/redis.txt
+	UseRelPaths bool
+}
+
+func NewRedisCache(parameters interface{}, log zlog.Logger) Cache {
+	properParameters, ok := parameters.(RedisDriverParameters)
+	if !ok {
+		panic("Failed type assertion")
+	}
+
+	connOpts, err := redis.ParseURL(properParameters.Url)
+	if err != nil {
+		log.Error().Err(err).Str("directory", properParameters.Url).Msg("unable to connect to redis")
+	}
+	cacheDB := redis.NewClient(connOpts)
+
+	if _, err := cacheDB.Ping(context.Background()).Result(); err != nil {
+		log.Error().Err(err).Msg("unable to ping redis cache")
+		return nil
+	}
+
+	return &RedisDriver{
+		db:          cacheDB,
+		log:         log,
+		rootDir:     properParameters.RootDir,
+		useRelPaths: properParameters.UseRelPaths,
+	}
+}
+
+func join(xs ...string) string {
+	return "zot:" + strings.Join(xs, ":")
+}
+
+func (d *RedisDriver) UsesRelativePaths() bool {
+	return d.useRelPaths
+}
+
+func (d *RedisDriver) Name() string {
+	return "redis"
+}
+
+func (d *RedisDriver) PutBlob(digest godigest.Digest, path string) error {
+	ctx := context.TODO()
+	if path == "" {
+		d.log.Error().Err(errors.ErrEmptyValue).Str("digest", digest.String()).Msg("empty path provided")
+		return errors.ErrEmptyValue
+	}
+
+	// use only relative (to rootDir) paths on blobs
+	var err error
+	if d.useRelPaths {
+		path, err = filepath.Rel(d.rootDir, path)
+		if err != nil {
+			d.log.Error().Err(err).Str("path", path).Msg("unable to get relative path")
+		}
+	}
+	if len(path) == 0 {
+		return errors.ErrEmptyValue
+	}
+	// see if the blob digest exists.
+	exists, err := d.db.HExists(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
+	if err != nil {
+		return err
+	}
+	if _, err := d.db.TxPipelined(ctx, func(tx redis.Pipeliner) error {
+		if !exists {
+			// add the key value pair [digest, path] to blobs:origin if not exist already. the path becomes the canonical blob
+			// we do this in a transaction to make sure that if something is in the set, then it is guaranteed to always have a path
+			// note that there is a race, but the worst case is that a different origin path that is still valid is used.
+			if err := tx.HSet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String(), path).Err(); err != nil {
+				d.log.Error().Err(err).Str("hset", join(constants.BlobsCache, constants.OriginalBucket)).Str("value", path).Msg("unable to put record")
+				return err
+			}
+		}
+		// add path to the set of paths which the digest represents
+		if err := d.db.SAdd(ctx, join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()), path).Err(); err != nil {
+			d.log.Error().Err(err).Str("sadd", join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())).Str("value", path).Msg("unable to put record")
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *RedisDriver) GetBlob(digest godigest.Digest) (string, error) {
+	ctx := context.TODO()
+	path, err := d.db.HGet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
+	if err != nil {
+		if goerrors.Is(err, redis.Nil) {
+			return "", errors.ErrCacheMiss
+		}
+		d.log.Error().Err(err).Str("hget", join(constants.BlobsCache, constants.OriginalBucket)).Str("digest", digest.String()).Msg("unable to get record")
+		return "", err
+	}
+	return path, nil
+}
+
+func (d *RedisDriver) HasBlob(digest godigest.Digest, blob string) bool {
+	ctx := context.TODO()
+	// see if we are in the set
+	exists, err := d.db.SIsMember(ctx, join(constants.BlobsCache, constants.DuplicatesBucket, digest.String()), blob).Result()
+	if err != nil {
+		d.log.Error().Err(err).Str("sismember", join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())).Str("digest", digest.String()).Msg("unable to get record")
+		return false
+	}
+	if !exists {
+		return false
+	}
+	// see if the path entry exists. is this actually needed? i guess it doesn't really hurt (it is fast)
+	exists, err = d.db.HExists(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
+	d.log.Error().Err(err).Str("hexists", join(constants.BlobsCache, constants.OriginalBucket)).Str("digest", digest.String()).Msg("unable to get record")
+	if err != nil {
+		return false
+	}
+	if !exists {
+		return false
+	}
+	return true
+}
+
+func (d *RedisDriver) DeleteBlob(digest godigest.Digest, path string) error {
+	ctx := context.TODO()
+
+	// use only relative (to rootDir) paths on blobs
+	var err error
+	if d.useRelPaths {
+		path, err = filepath.Rel(d.rootDir, path)
+		if err != nil {
+			d.log.Error().Err(err).Str("path", path).Msg("unable to get relative path")
+		}
+	}
+
+	pathSet := join(constants.BlobsCache, constants.DuplicatesBucket, digest.String())
+
+	// delete path from the set of paths which the digest represents
+	_, err = d.db.SRem(ctx, pathSet, path).Result()
+	if err != nil {
+		d.log.Error().Err(err).Str("srem", pathSet).Str("value", path).Msg("unable to delete record")
+		return err
+	}
+	currentPath, err := d.GetBlob(digest)
+	if err != nil {
+		return err
+	}
+	if currentPath != path {
+		// nothing we need to do, return nil yay
+		return nil
+	}
+	// we need to set a new path
+	newPath, err := d.db.SRandMember(ctx, pathSet).Result()
+	if err != nil {
+		if goerrors.Is(err, redis.Nil) {
+			_, err := d.db.HDel(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String()).Result()
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+		d.log.Error().Err(err).Str("srandmember", pathSet).Msg("unable to get new path")
+		return err
+	}
+	if _, err := d.db.HSet(ctx, join(constants.BlobsCache, constants.OriginalBucket), digest.String(), newPath).Result(); err != nil {
+		d.log.Error().Err(err).Str("hset", join(constants.BlobsCache, constants.OriginalBucket)).Str("value", newPath).Msg("unable to put record")
+		return err
+	}
+
+	return nil
+}