From 882d010cc0558a917f8e1fe521b2f088c770fa2f Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sat, 8 Jun 2024 13:57:39 -0400 Subject: [PATCH] feat: add memcache cache --- README.md | 10 ++++-- docs/configuration.md | 24 ++++++++++++- go.mod | 1 + go.sum | 2 ++ internal/caches/cache.go | 28 +++++++++++++++ internal/caches/memcache.go | 68 +++++++++++++++++++++++++++++++++++-- internal/caches/redis.go | 46 +++++++++++++------------ test_config.yml | 5 +-- 8 files changed, 154 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6ec9cdc0..271b28ea 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ The following features are currently available: * Provide a uniform ZXY mapping interface for incoming requests. * Proxy map tiles to ZXY, WMS, TMS, or WMTS backed map layers -* Cache map tiles in disk, memory, s3, redis ... +* Cache map tiles in disk, memory, s3, redis, or memcache * Generic support for any content type * Incoming authentication using a static key or JWT * Configurable timeout, logging, and error handling rules @@ -323,7 +323,13 @@ The following are the known incompatibilities with tilestache configurations: * No `dirs` parameter - Files are currently stored in a flat structure rather than creating separate directories * No `gzip` parameter - Might be added in the future * The `path` parameter must be supplied as a file path, not a URI -* Redis cache supports a wider variety of configuration options. It's recommended but not required that you consider utilizing a Cluster or Ring deployment if you previously used a single server. +* Memcache cache: + * No `revision` parameter - Put the revision inside the key prefix + * The `key prefix` parameter is replaced with `keyprefix` + * The `servers` array is now an array of objects containing `host` and `port` instead of an array of strings with those combined +* Redis cache: + * Supports a wider variety of configuration options. It's recommended but not required that you consider utilizing a Cluster or Ring deployment if you previously used a single server. + * The `key prefix` parameter is replaced with `keyprefix` * S3 cache: * No `use_locks` parameter - Caches are currently lockless * No `reduced_redundancy` parameter - Instead use the more flexible `storageclass` parameter with the "REDUCED_REDUNDANCY" option diff --git a/docs/configuration.md b/docs/configuration.md index 1fde7369..287c887d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -116,7 +116,29 @@ Example: ### Memcache -TODO. Not yet implemented. +Cache tiles using memcache. + +Name should be "memcache" + +Configuration options: + +| Parameter | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| host | String | No | 127.0.0.1 | The host of the memcache server. A convenience equivalent to supplying `servers` with a single entry. Do not supply both this and `servers` | +| port | int | No | 6379 | The port of the memcache server. A convenience equivalent to supplying `servers` with a single entry. Do not supply both this and `servers` | +| keyprefix | string | No | None | A prefix to use for keys stored in cache. Helps avoid collisions when multiple applications use the same memcache | +| ttl | uint32 | No | 1 day | How long cache entries should persist for in seconds. Cannot be disabled. | +| servers | Array of `host` and `port` | No | host and port | The list of servers to connect to supplied as an array of objects, each with a host and key parameter. This should only have a single entry when operating in standalone mode. If this is unspecified it uses the standalone `host` and `port` parameters as a default, therefore this shouldn't be specified at the same time as those | + +Example: + +```yaml +cache: + name: memcache + host: 127.0.0.1 + port: 11211 +``` + ### Memory diff --git a/go.mod b/go.mod index 20babf69..3159097c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22.3 require ( github.com/aws/aws-sdk-go v1.53.14 + github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 github.com/go-redis/cache/v9 v9.0.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index b71c58e7..15f41bba 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/aws/aws-sdk-go v1.53.14 h1:SzhkC2Pzag0iRW8WBb80RzKdGXDydJR9LAMs2GyKJ2M= github.com/aws/aws-sdk-go v1.53.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous= +github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/internal/caches/cache.go b/internal/caches/cache.go index 28819f22..9118adab 100644 --- a/internal/caches/cache.go +++ b/internal/caches/cache.go @@ -16,6 +16,7 @@ package caches import ( "fmt" + "strconv" "github.com/Michad/tilegroxy/internal" "github.com/Michad/tilegroxy/internal/config" @@ -77,8 +78,35 @@ func ConstructCache(rawConfig map[string]interface{}, errorMessages *config.Erro return nil, err } return ConstructRedis(&config, errorMessages) + } else if rawConfig["name"] == "memcache" { + var config MemcacheConfig + err := mapstructure.Decode(rawConfig, &config) + if err != nil { + return nil, err + } + return ConstructMemcache(&config, errorMessages) } name := fmt.Sprintf("%#v", rawConfig["name"]) return nil, fmt.Errorf(errorMessages.InvalidParam, "cache.name", name) } + +// Utility type used in a couple caches +type HostAndPort struct { + Host string + Port uint16 +} + +func (hp HostAndPort) String() string { + return hp.Host + ":" + strconv.Itoa(int(hp.Port)) +} + +func HostAndPortArrayToStringArray(servers []HostAndPort) []string { + addrs := make([]string, len(servers)) + + for i, addr := range servers { + addrs[i] = addr.String() + } + + return addrs +} diff --git a/internal/caches/memcache.go b/internal/caches/memcache.go index 7b21e8db..4709d062 100644 --- a/internal/caches/memcache.go +++ b/internal/caches/memcache.go @@ -14,15 +14,77 @@ package caches -import "github.com/Michad/tilegroxy/internal" +import ( + "fmt" + + "github.com/Michad/tilegroxy/internal" + "github.com/Michad/tilegroxy/internal/config" + "github.com/bradfitz/gomemcache/memcache" +) + +type MemcacheConfig struct { + HostAndPort `mapstructure:",squash"` + Servers []HostAndPort //The list of servers to use. + KeyPrefix string //Prefix to keynames stored in cache + Ttl uint32 //Cache expiration in seconds. Max of 30 days. Default to 1 day +} + +const ( + memcacheDefaultHost = "127.0.0.1" + memcacheDefaultPort = 11211 + memcacheDefaultTtl = 60 * 60 * 24 + memcacheMaxTtl = 30 * 60 * 60 * 24 +) type Memcache struct { + *MemcacheConfig + client *memcache.Client +} + +func ConstructMemcache(config *MemcacheConfig, errorMessages *config.ErrorMessages) (*Memcache, error) { + if config.Servers == nil || len(config.Servers) == 0 { + if config.Host == "" { + config.Host = memcacheDefaultHost + } + if config.Port == 0 { + config.Port = memcacheDefaultPort + } + + config.Servers = []HostAndPort{{config.Host, config.Port}} + } else { + if config.Host != "" { + return nil, fmt.Errorf(errorMessages.ParamsMutuallyExclusive, "config.memcache.host", "config.memcache.servers") + } + } + + if config.Ttl == 0 { + config.Ttl = memcacheDefaultTtl + } + if config.Ttl > memcacheMaxTtl { + config.Ttl = memcacheMaxTtl + } + + addrs := HostAndPortArrayToStringArray(config.Servers) + mc := memcache.New(addrs...) + + err := mc.Ping() + + return &Memcache{config, mc}, err + } func (c Memcache) Lookup(t internal.TileRequest) (*internal.Image, error) { - return nil, nil + it, err := c.client.Get(c.KeyPrefix + t.String()) + + if err != nil { + return nil, err + } + + result := internal.Image(it.Value) + + return &result, nil } func (c Memcache) Save(t internal.TileRequest, img *internal.Image) error { - return nil + return c.client.Set(&memcache.Item{Key: c.KeyPrefix + t.String(), Value: *img, Expiration: int32(c.Ttl)}) } diff --git a/internal/caches/redis.go b/internal/caches/redis.go index af6b2ca4..3bd98760 100644 --- a/internal/caches/redis.go +++ b/internal/caches/redis.go @@ -28,11 +28,6 @@ import ( "github.com/redis/go-redis/v9" ) -type RedisServer struct { - Host string - Port uint16 -} - const ( ModeStandalone = "standalone" ModeCluster = "cluster" @@ -42,16 +37,23 @@ const ( var AllModes = []string{ModeStandalone, ModeCluster, ModeRing} type RedisConfig struct { - RedisServer //Host and Port for a single server. A convenience equivalent to supplying Servers with a single entry - Db int //Database number, defaults to 0 - KeyPrefix string //Prefix to keynames stored in cache - Username string //Username to use to authenticate - Password string //Password to use to authenticate - Mode string //Controls operating mode. One of AllModes. Defaults to standalone - Ttl uint32 //Cache expiration in seconds. Default to 1 day - Servers []RedisServer //The list of servers to use. + HostAndPort `mapstructure:",squash"` //Host and Port for a single server. A convenience equivalent to supplying Servers with a single entry + Db int //Database number, defaults to 0 + KeyPrefix string //Prefix to keynames stored in cache + Username string //Username to use to authenticate + Password string //Password to use to authenticate + Mode string //Controls operating mode. One of AllModes. Defaults to standalone + Ttl uint32 //Cache expiration in seconds. Max of 1 year. Default to 1 day + Servers []HostAndPort //The list of servers to use. } +const ( + redisDefaultHost = "127.0.0.1" + redisDefaultPort = 6379 + redisDefaultTtl = 60 * 60 * 24 + redisMaxTtl = 60 * 60 * 24 * 365 +) + type Redis struct { *RedisConfig cache *cache.Cache @@ -70,20 +72,24 @@ func ConstructRedis(config *RedisConfig, errorMessages *config.ErrorMessages) (* if config.Servers == nil || len(config.Servers) == 0 { if config.Host == "" { - config.Host = "127.0.0.1" + config.Host = redisDefaultHost } if config.Port == 0 { - config.Port = 6379 + config.Port = redisDefaultPort } - config.Servers = []RedisServer{{config.Host, config.Port}} + config.Servers = []HostAndPort{{config.Host, config.Port}} } else { if config.Host != "" { return nil, fmt.Errorf(errorMessages.ParamsMutuallyExclusive, "config.redis.host", "config.redis.servers") } } + if config.Ttl == 0 { - config.Ttl = 60 * 60 * 24 + config.Ttl = redisDefaultTtl + } + if config.Ttl > redisMaxTtl { + config.Ttl = redisMaxTtl } if config.Mode == ModeCluster { @@ -91,11 +97,7 @@ func ConstructRedis(config *RedisConfig, errorMessages *config.ErrorMessages) (* return nil, fmt.Errorf(errorMessages.ParamsMutuallyExclusive, "cache.redis.db", "cache.redis.cluster") } - addrs := make([]string, len(config.Servers)) - - for _, addr := range config.Servers { - addrs = append(addrs, addr.Host+":"+strconv.Itoa(int(addr.Port))) - } + addrs := HostAndPortArrayToStringArray(config.Servers) client := redis.NewClusterClient(&redis.ClusterOptions{ Addrs: addrs, diff --git a/test_config.yml b/test_config.yml index 50f41ba4..13033f3d 100644 --- a/test_config.yml +++ b/test_config.yml @@ -36,8 +36,9 @@ authentication: # region: us-east-1 # profile: tilegroxy cache: - name: redis - mode: standalone + name: memcache + host: 127.0.0.1 + port: 11211 layers: - id: test