From 592f4c096b299e480e72a4f0c889d3e77d4f60be Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 4 Dec 2022 11:36:45 +0000 Subject: [PATCH 1/5] test --- Dockerfile | 12 +++---- internal/resolvers/default/initialize.go | 4 +-- pkg/cache/db.go | 40 +++++++++++++++++++++++- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5eace5dd..e8198f78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,10 @@ -FROM ubuntu:22.04 AS build -ENV GOVER=1.19.2 +FROM golang:1.19-alpine AS build ADD . /src -RUN apt update && apt -y install libvips-dev wget build-essential -RUN wget -qO- https://go.dev/dl/go$GOVER.linux-amd64.tar.gz | tar -C /src -xzf - -RUN cd /src/cmd/api && /src/go/bin/go build +RUN apk add --no-cache build-base pkgconfig vips-dev +RUN cd /src/cmd/api && go build -FROM ubuntu:22.04 +FROM alpine:latest WORKDIR /app COPY --from=build /src/cmd/api/api /app/ -RUN apt update && apt install -y ca-certificates libvips && apt clean +RUN apk add --no-cache ca-certificates vips CMD ["./api"] diff --git a/internal/resolvers/default/initialize.go b/internal/resolvers/default/initialize.go index 5511dd1f..d6937a02 100644 --- a/internal/resolvers/default/initialize.go +++ b/internal/resolvers/default/initialize.go @@ -28,11 +28,11 @@ var defaultTooltip = template.Must(template.New("default_tooltip").Parse(default func Initialize(ctx context.Context, cfg config.APIConfig, pool db.Pool, router *chi.Mux, helixClient *helix.Client) { defaultLinkResolver := New(ctx, cfg, pool, helixClient) - cached := stampede.Handler(512, 10*time.Second) + // cached := stampede.Handler(512, 10*time.Second) imageCached := stampede.Handler(256, 2*time.Second) generatedValuesCached := stampede.Handler(256, 2*time.Second) - router.With(cached).Get("/link_resolver/{url}", defaultLinkResolver.HandleRequest) + router.Get("/link_resolver/{url}", defaultLinkResolver.HandleRequest) router.With(imageCached).Get("/thumbnail/{url}", defaultLinkResolver.HandleThumbnailRequest) router.With(generatedValuesCached).Get("/generated/{url}", defaultLinkResolver.HandleGeneratedValueRequest) } diff --git a/pkg/cache/db.go b/pkg/cache/db.go index b07d98a5..1ee5843c 100644 --- a/pkg/cache/db.go +++ b/pkg/cache/db.go @@ -3,6 +3,7 @@ package cache import ( "context" "net/http" + "sync" "time" "github.com/Chatterino/api/internal/db" @@ -49,6 +50,9 @@ type PostgreSQLCache struct { pool db.Pool dependentCaches []DependentCache + + requestsMutex sync.Mutex + requests map[string][]chan *Response } // TODO: Make the "internal error" tooltip an actual tooltip @@ -189,7 +193,40 @@ func (c *PostgreSQLCache) Get(ctx context.Context, key string, r *http.Request) cacheMisses.Inc() log.Debugw("DB Get cache miss", "cacheKey", cacheKey) - return c.load(ctx, key, r) + responseChannel := make(chan *Response) + + c.requestsMutex.Lock() + + c.requests[key] = append(c.requests[key], responseChannel) + + first := len(c.requests[key]) == 1 + + c.requestsMutex.Unlock() + + if first { + log.Debugw("DB Get cache miss!!", "cacheKey", cacheKey) + go func() { + response, err := c.load(ctx, key, r) + if err != nil { + log.Warnw("Error loading DB request", "error", err) + response = &Response{ + Payload: []byte(`{"status":500,"message":"Internal server error (PSQL) loading cache"}`), + StatusCode: 500, + ContentType: "application/json", + } + } + c.requestsMutex.Lock() + for _, ch := range c.requests[key] { + ch <- response + } + delete(c.requests, key) + }() + } + + // If key is not in cache, sign up as a listener and ensure loader is only called once + // Wait for loader to complete, then return value from loader + response := <-responseChannel + return response, nil } func (c *PostgreSQLCache) GetOnly(ctx context.Context, key string) *Response { @@ -280,6 +317,7 @@ func NewPostgreSQLCache(ctx context.Context, cfg config.APIConfig, pool db.Pool, loader: loader, cacheDuration: cacheDuration, pool: pool, + requests: make(map[string][]chan *Response), } } From 903b7b0dda9d9b73a582aabacc4bcdb65ab155f3 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 4 Dec 2022 12:42:07 +0100 Subject: [PATCH 2/5] unlock mutex :tf: --- pkg/cache/db.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cache/db.go b/pkg/cache/db.go index 1ee5843c..993048a4 100644 --- a/pkg/cache/db.go +++ b/pkg/cache/db.go @@ -220,11 +220,13 @@ func (c *PostgreSQLCache) Get(ctx context.Context, key string, r *http.Request) ch <- response } delete(c.requests, key) + c.requestsMutex.Unlock() }() } // If key is not in cache, sign up as a listener and ensure loader is only called once // Wait for loader to complete, then return value from loader + log.Debugw("DB Waiting for response channel", "cacheKey", cacheKey) response := <-responseChannel return response, nil } From 6cccba47d9205c644dfb92a810cdbd08025018ba Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 4 Dec 2022 13:01:21 +0100 Subject: [PATCH 3/5] Remove commented out code --- internal/resolvers/default/initialize.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/resolvers/default/initialize.go b/internal/resolvers/default/initialize.go index d6937a02..9d3a7f82 100644 --- a/internal/resolvers/default/initialize.go +++ b/internal/resolvers/default/initialize.go @@ -28,7 +28,6 @@ var defaultTooltip = template.Must(template.New("default_tooltip").Parse(default func Initialize(ctx context.Context, cfg config.APIConfig, pool db.Pool, router *chi.Mux, helixClient *helix.Client) { defaultLinkResolver := New(ctx, cfg, pool, helixClient) - // cached := stampede.Handler(512, 10*time.Second) imageCached := stampede.Handler(256, 2*time.Second) generatedValuesCached := stampede.Handler(256, 2*time.Second) From 05f944066d74269f10c1d3a2b42ef5a66065591f Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 4 Dec 2022 13:03:15 +0100 Subject: [PATCH 4/5] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2fd1479..24ea8c11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Dev: Improve Livestreamfails tests. (#297, #301) - Dev: Improve default resolver tests. (#300) - Dev: Resolve imgur.io links. (#365) +- Dev: Don't use `stampede` for link resolver links. (#394) ## 1.2.3 From e18d4c4966774118cd8c5577565549982e011ff9 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 4 Dec 2022 13:15:54 +0100 Subject: [PATCH 5/5] wrap response so we retain the error --- pkg/cache/db.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/cache/db.go b/pkg/cache/db.go index 993048a4..ab037636 100644 --- a/pkg/cache/db.go +++ b/pkg/cache/db.go @@ -34,6 +34,11 @@ var ( ) ) +type wrappedResponse struct { + response *Response + err error +} + func init() { prometheus.MustRegister(cacheHits) prometheus.MustRegister(cacheMisses) @@ -52,7 +57,7 @@ type PostgreSQLCache struct { dependentCaches []DependentCache requestsMutex sync.Mutex - requests map[string][]chan *Response + requests map[string][]chan wrappedResponse } // TODO: Make the "internal error" tooltip an actual tooltip @@ -191,9 +196,10 @@ func (c *PostgreSQLCache) Get(ctx context.Context, key string, r *http.Request) return cacheResponse, nil } + // If key is not in cache, sign up as a listener and ensure loader is only called once cacheMisses.Inc() log.Debugw("DB Get cache miss", "cacheKey", cacheKey) - responseChannel := make(chan *Response) + responseChannel := make(chan wrappedResponse) c.requestsMutex.Lock() @@ -204,31 +210,25 @@ func (c *PostgreSQLCache) Get(ctx context.Context, key string, r *http.Request) c.requestsMutex.Unlock() if first { - log.Debugw("DB Get cache miss!!", "cacheKey", cacheKey) go func() { response, err := c.load(ctx, key, r) - if err != nil { - log.Warnw("Error loading DB request", "error", err) - response = &Response{ - Payload: []byte(`{"status":500,"message":"Internal server error (PSQL) loading cache"}`), - StatusCode: 500, - ContentType: "application/json", - } + + r := wrappedResponse{ + response, + err, } c.requestsMutex.Lock() for _, ch := range c.requests[key] { - ch <- response + ch <- r } delete(c.requests, key) c.requestsMutex.Unlock() }() } - // If key is not in cache, sign up as a listener and ensure loader is only called once // Wait for loader to complete, then return value from loader - log.Debugw("DB Waiting for response channel", "cacheKey", cacheKey) response := <-responseChannel - return response, nil + return response.response, response.err } func (c *PostgreSQLCache) GetOnly(ctx context.Context, key string) *Response { @@ -319,7 +319,7 @@ func NewPostgreSQLCache(ctx context.Context, cfg config.APIConfig, pool db.Pool, loader: loader, cacheDuration: cacheDuration, pool: pool, - requests: make(map[string][]chan *Response), + requests: make(map[string][]chan wrappedResponse), } }