From 1480f77177bfe3ab73bc3bfef3013f505b5bb1d5 Mon Sep 17 00:00:00 2001 From: pajlada Date: Sat, 3 Jul 2021 15:42:16 +0200 Subject: [PATCH] Use Helix for loading Emote Set info (#175) Co-authored-by: zneix --- CHANGELOG.md | 1 + README.md | 11 +- cmd/api/link_resolver_test.go | 4 +- cmd/api/main.go | 18 +- cmd/api/twitchemotes.go | 163 ------------------ internal/resolvers/default/resolver.go | 9 +- internal/resolvers/twitch/resolver.go | 28 +-- internal/routes/twitchemotes/route.go | 138 +++++++++++++++ internal/twitchapiclient/cache.go | 33 ++++ internal/twitchapiclient/client.go | 40 +++++ .../token_refresh.go | 2 +- 11 files changed, 248 insertions(+), 199 deletions(-) delete mode 100644 cmd/api/twitchemotes.go create mode 100644 internal/routes/twitchemotes/route.go create mode 100644 internal/twitchapiclient/cache.go create mode 100644 internal/twitchapiclient/client.go rename internal/{resolvers/twitch => twitchapiclient}/token_refresh.go (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfa11ff..476304a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Added link preview support for 7tv emote links. (#155) - Skip lilliput if image is below maxThumbnailSize. (#184) +- Dev: Change Emote Set backend from `twitchemotes.com` to the Twitch Helix API. (#175) ## 1.2.0 diff --git a/README.md b/README.md index 2a2fdf23..3f2779c3 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ Go web service that serves as a cache to APIs that each Chatterino client could use. -Emote data is served cached from [twitchemotes.com](https://twitchemotes.com/). - ## Routes -`/twitchemotes/set/:setID/` + +`twitchemotes/set/:setID` Returns information about a given twitch emote set. Example response: + ``` { "channel_name": "forsen", // twitch user name @@ -21,6 +21,7 @@ Returns information about a given twitch emote set. Example response: `link_resolver/:url` Resolves a url into a preview tooltip. Example response: + ``` { "status": 200, // status code returned from the page @@ -33,23 +34,27 @@ Resolves a url into a preview tooltip. Example response: `health/uptime` Returns API service's uptime. Example response: + ``` 928h2m53.795354922s ``` `health/memory` Returns information about memory usage. Example response: + ``` Alloc=505 MiB, TotalAlloc=17418866 MiB, Sys=3070 MiB, NumGC=111245 ``` `health/combined` Returns both uptime and information about memory usage. Example response: + ``` Uptime: 928h5m7.937821282s - Memory: Alloc=510 MiB, TotalAlloc=17419213 MiB, Sys=3070 MiB, NumGC=111246 ``` ## Using your self-hosted version + If you host your own version of this API, you can modify which url Chatterino2 uses to resolve links and to resolve twitch emote sets. [Change link resolver](https://wiki.chatterino.com/Environment%20Variables/#chatterino2_link_resolver_url) [Change Twitch emote resolver](https://wiki.chatterino.com/Environment%20Variables/#chatterino2_twitch_emote_set_resolver_url) diff --git a/cmd/api/link_resolver_test.go b/cmd/api/link_resolver_test.go index de766244..e34db67e 100644 --- a/cmd/api/link_resolver_test.go +++ b/cmd/api/link_resolver_test.go @@ -17,7 +17,7 @@ import ( func TestResolveTwitchClip(t *testing.T) { router := chi.NewRouter() cfg := config.New() - defaultresolver.Initialize(router, cfg) + defaultresolver.Initialize(router, cfg, nil) ts := httptest.NewServer(router) defer ts.Close() fmt.Println(ts.URL) @@ -45,7 +45,7 @@ func TestResolveTwitchClip(t *testing.T) { func TestResolveTwitchClip2(t *testing.T) { router := chi.NewRouter() cfg := config.New() - defaultresolver.Initialize(router, cfg) + defaultresolver.Initialize(router, cfg, nil) ts := httptest.NewServer(router) defer ts.Close() const url = `https%3A%2F%2Ftwitch.tv%2Fpajlada%2Fclip%2FGorgeousAntsyPizzaSaltBae` diff --git a/cmd/api/main.go b/cmd/api/main.go index 71af0c26..dea79669 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -7,10 +7,14 @@ import ( "time" defaultresolver "github.com/Chatterino/api/internal/resolvers/default" + "github.com/Chatterino/api/internal/routes/twitchemotes" + "github.com/Chatterino/api/internal/twitchapiclient" + "github.com/Chatterino/api/pkg/cache" "github.com/Chatterino/api/pkg/config" "github.com/Chatterino/api/pkg/resolver" "github.com/Chatterino/api/pkg/thumbnail" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" ) var ( @@ -65,9 +69,19 @@ func main() { router := chi.NewRouter() - handleTwitchEmotes(router) + // Strip trailing slashes from API requests + router.Use(middleware.StripSlashes) + + var helixUsernameCache *cache.Cache + + helixClient, helixUsernameCache, err := twitchapiclient.New(cfg) + if err != nil { + log.Printf("[Twitch] %s\n", err.Error()) + } + + twitchemotes.Initialize(router, helixClient, helixUsernameCache) handleHealth(router) - defaultresolver.Initialize(router, cfg) + defaultresolver.Initialize(router, cfg, helixClient) listen(cfg.BindAddress, mountRouter(router)) } diff --git a/cmd/api/twitchemotes.go b/cmd/api/twitchemotes.go deleted file mode 100644 index f45f73ac..00000000 --- a/cmd/api/twitchemotes.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" - - "github.com/Chatterino/api/pkg/cache" - "github.com/Chatterino/api/pkg/utils" - "github.com/go-chi/chi/v5" -) - -type TwitchEmotesError struct { - Status int - Error error -} - -type TwitchEmotesErrorResponse struct { - Status int - Message string -} - -type EmoteSet struct { - ChannelName string `json:"channel_name"` - ChannelID string `json:"channel_id"` - Type string `json:"type"` - Tier int `json:"tier"` - Custom bool `json:"custom"` -} - -var ( - errInvalidEmoteID = errors.New("invalid emote id") - - customEmoteSets map[string][]byte = make(map[string][]byte) - - twitchemotesCache = cache.New("twitchemotes", doTwitchemotesRequest, time.Duration(30)*time.Minute) -) - -func addEmoteSet(emoteSetID, channelName, channelID, setType string) { - b, err := json.Marshal(&EmoteSet{ - ChannelName: channelName, - ChannelID: channelID, - Type: setType, - Custom: true, - }) - if err != nil { - panic(err) - } - customEmoteSets[emoteSetID] = b -} - -func init() { - addEmoteSet("13985", "evohistorical2015", "129284508", "sub") -} - -func setsHandler(w http.ResponseWriter, r *http.Request) { - // TODO: Implement multiset-fetcher and in future version of Chatterino which sends a list of sets instead of one per request - w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte("{\"error\": \"not implemented\"}")) - if err != nil { - log.Println("Error writing response:", err) - } -} - -func setHandler(w http.ResponseWriter, r *http.Request) { - setID := chi.URLParam(r, "setID") - w.Header().Set("Content-Type", "application/json") - - // 1. Check our "custom" responses - if v, ok := customEmoteSets[setID]; ok { - _, err := w.Write(v) - if err != nil { - log.Println("Error writing response:", err) - } - return - } - - // 2. Cache a request from twitchemotes.com - response := twitchemotesCache.Get(setID, nil) - - switch v := response.(type) { - case []byte: - _, err := w.Write(v) - if err != nil { - log.Println("Error writing response:", err) - } - - case *TwitchEmotesError: - w.WriteHeader(v.Status) - data, err := json.Marshal(&TwitchEmotesErrorResponse{ - Status: v.Status, - Message: v.Error.Error(), - }) - if err != nil { - log.Println("Error marshalling twitch emotes error response:", err) - } - _, err = w.Write(data) - if err != nil { - log.Println("Error writing response:", err) - } - } -} - -func doTwitchemotesRequest(setID string, r *http.Request) (interface{}, time.Duration, error) { - url := fmt.Sprintf("https://api.twitchemotes.com/api/v4/sets?id=%s", setID) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return &TwitchEmotesError{ - Error: err, - Status: 500, - }, 0, nil - } - - req.Header.Set("User-Agent", "chatterino-api-cache/1.0 link-resolver") - - resp, err := httpClient.Do(req) - if err != nil { - return &TwitchEmotesError{ - Error: err, - Status: 500, - }, 0, nil - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return &TwitchEmotesError{ - Error: err, - Status: 500, - }, 0, nil - } - var emoteSets []EmoteSet - err = json.Unmarshal(body, &emoteSets) - if err != nil { - return &TwitchEmotesError{ - Error: err, - Status: 500, - }, 0, nil - } - - if len(emoteSets) == 0 { - return &TwitchEmotesError{ - Error: errInvalidEmoteID, - Status: 404, - }, 0, nil - } - - if len(emoteSets) > 1 { - log.Println("Unhandled long emote set for emote set", setID) - } - - return utils.MarshalNoDur(&emoteSets[0]) -} - -func handleTwitchEmotes(router *chi.Mux) { - router.Get("/twitchemotes/set/{setID}/", setHandler) - - router.Get("/twitchemotes/sets/", setsHandler) -} diff --git a/internal/resolvers/default/resolver.go b/internal/resolvers/default/resolver.go index 65fd8edd..9c755d85 100644 --- a/internal/resolvers/default/resolver.go +++ b/internal/resolvers/default/resolver.go @@ -24,6 +24,7 @@ import ( "github.com/Chatterino/api/pkg/thumbnail" "github.com/Chatterino/api/pkg/utils" "github.com/go-chi/chi/v5" + "github.com/nicklaw5/helix" ) const ( @@ -87,7 +88,7 @@ func (dr *R) HandleThumbnailRequest(w http.ResponseWriter, r *http.Request) { } } -func New(cfg config.APIConfig) *R { +func New(cfg config.APIConfig, helixClient *helix.Client) *R { r := &R{ cfg: cfg, } @@ -103,7 +104,7 @@ func New(cfg config.APIConfig) *R { r.customResolvers = append(r.customResolvers, livestreamfails.New(cfg)...) r.customResolvers = append(r.customResolvers, oembed.New(cfg)...) r.customResolvers = append(r.customResolvers, supinic.New(cfg)...) - r.customResolvers = append(r.customResolvers, twitch.New(cfg)...) + r.customResolvers = append(r.customResolvers, twitch.New(cfg, helixClient)...) r.customResolvers = append(r.customResolvers, twitter.New(cfg)...) r.customResolvers = append(r.customResolvers, wikipedia.New(cfg)...) r.customResolvers = append(r.customResolvers, youtube.New(cfg)...) @@ -112,8 +113,8 @@ func New(cfg config.APIConfig) *R { return r } -func Initialize(router *chi.Mux, cfg config.APIConfig) { - defaultLinkResolver := New(cfg) +func Initialize(router *chi.Mux, cfg config.APIConfig, helixClient *helix.Client) { + defaultLinkResolver := New(cfg, helixClient) router.Get("/link_resolver/{url}", defaultLinkResolver.HandleRequest) router.Get("/thumbnail/{url}", defaultLinkResolver.HandleThumbnailRequest) diff --git a/internal/resolvers/twitch/resolver.go b/internal/resolvers/twitch/resolver.go index b783066b..d0165bfc 100644 --- a/internal/resolvers/twitch/resolver.go +++ b/internal/resolvers/twitch/resolver.go @@ -39,33 +39,13 @@ var ( helixAPI TwitchAPIClient ) -func New(cfg config.APIConfig) (resolvers []resolver.CustomURLManager) { - if cfg.TwitchClientID == "" { - log.Println("[Config] twitch_client_id is missing, won't do special responses for Twitch clips") +func New(cfg config.APIConfig, helixClient *helix.Client) (resolvers []resolver.CustomURLManager) { + if helixClient == nil { + log.Println("[Config] No Helix Client passed to New - won't do special responses for Twitch clips") return } - if cfg.TwitchClientSecret == "" { - log.Println("[Config] twitch_client_secret is missing, won't do special responses for Twitch clips") - return - } - - var err error - - helixAPI, err = helix.NewClient(&helix.Options{ - ClientID: cfg.TwitchClientID, - ClientSecret: cfg.TwitchClientSecret, - }) - - if err != nil { - log.Fatalf("[Helix] Error initializing API client: %s", err.Error()) - } - - waitForFirstAppAccessToken := make(chan struct{}) - - // Initialize methods responsible for refreshing oauth - go initAppAccessToken(helixAPI.(*helix.Client), waitForFirstAppAccessToken) - <-waitForFirstAppAccessToken + helixAPI = helixClient resolvers = append(resolvers, resolver.CustomURLManager{ Check: check, diff --git a/internal/routes/twitchemotes/route.go b/internal/routes/twitchemotes/route.go new file mode 100644 index 00000000..ec38541a --- /dev/null +++ b/internal/routes/twitchemotes/route.go @@ -0,0 +1,138 @@ +package twitchemotes + +import ( + "encoding/json" + "errors" + "log" + "net/http" + "time" + + "github.com/Chatterino/api/pkg/cache" + "github.com/Chatterino/api/pkg/utils" + "github.com/go-chi/chi/v5" + "github.com/nicklaw5/helix" +) + +var ( + errInvalidEmoteID = errors.New("invalid emote id") + errUnableToHandle = errors.New("unable to handle twitchemotes requests") + + twitchemotesCache = cache.New("twitchemotes", doTwitchemotesRequest, time.Duration(30)*time.Minute) + + helixAPI *helix.Client + helixUsernameCache *cache.Cache +) + +type TwitchEmotesError struct { + Status int + Error error +} + +type TwitchEmotesErrorResponse struct { + Status int + Message string +} + +type EmoteSet struct { + ChannelName string `json:"channel_name"` + ChannelID string `json:"channel_id"` + Type string `json:"type"` + Tier int `json:"tier"` + Custom bool `json:"custom"` +} + +func doTwitchemotesRequest(setID string, r *http.Request) (interface{}, time.Duration, error) { + if helixAPI == nil || helixUsernameCache == nil { + return &TwitchEmotesError{ + Error: errUnableToHandle, + Status: 500, + }, 0, nil + } + + params := &helix.GetEmoteSetsParams{ + EmoteSetIDs: []string{ + setID, + }, + } + resp, err := helixAPI.GetEmoteSets(params) + if err != nil { + return &TwitchEmotesError{ + Error: err, + Status: 500, + }, 0, nil + } + + if len(resp.Data.Emotes) == 0 { + return &TwitchEmotesError{ + Error: errInvalidEmoteID, + Status: 404, + }, 0, nil + } + + emote := resp.Data.Emotes[0] + + var ok bool + var username string + + // For Emote Sets 0 (global) and 19194 (prime emotes), the Owner ID returns 0 + // 0 is not a valid Twitch User ID, so hardcode the username to Twitch + if emote.OwnerID == "0" { + username = "Twitch" + } else { + // Load username from Helix + if username, ok = helixUsernameCache.Get(emote.OwnerID, nil).(string); !ok { + return &TwitchEmotesError{ + Error: errInvalidEmoteID, + Status: 404, + }, 0, nil + } + } + + emoteSet := EmoteSet{ + ChannelName: username, + ChannelID: emote.OwnerID, + } + + return utils.MarshalNoDur(&emoteSet) +} + +func setHandler(w http.ResponseWriter, r *http.Request) { + setID := chi.URLParam(r, "setID") + w.Header().Set("Content-Type", "application/json") + + response := twitchemotesCache.Get(setID, nil) + + switch v := response.(type) { + case []byte: + _, err := w.Write(v) + if err != nil { + log.Println("Error writing response:", err) + } + + case *TwitchEmotesError: + w.WriteHeader(v.Status) + data, err := json.Marshal(&TwitchEmotesErrorResponse{ + Status: v.Status, + Message: v.Error.Error(), + }) + if err != nil { + log.Println("Error marshalling twitch emotes error response:", err) + } + _, err = w.Write(data) + if err != nil { + log.Println("Error writing response:", err) + } + } +} + +// Initialize servers the /twitchemotes/set/{setID} route +// In newer versions of Chatterino this data is fetched client-side instead. +// To support older versions of Chattterino that relied on this API we will keep this API functional for some time longer. +func Initialize(router *chi.Mux, helixClient *helix.Client, usernameCache *cache.Cache) error { + helixAPI = helixClient + helixUsernameCache = usernameCache + + router.Get("/twitchemotes/set/{setID}", setHandler) + + return nil +} diff --git a/internal/twitchapiclient/cache.go b/internal/twitchapiclient/cache.go new file mode 100644 index 00000000..13f3bacc --- /dev/null +++ b/internal/twitchapiclient/cache.go @@ -0,0 +1,33 @@ +package twitchapiclient + +import ( + "errors" + "net/http" + "time" + + "github.com/Chatterino/api/pkg/cache" + "github.com/nicklaw5/helix" +) + +func loadUsername(helixClient *helix.Client) func(key string, r *http.Request) (interface{}, time.Duration, error) { + return func(twitchUserID string, r *http.Request) (interface{}, time.Duration, error) { + params := &helix.UsersParams{ + IDs: []string{ + twitchUserID, + }, + } + + response, err := helixClient.GetUsers(params) + if err != nil { + return nil, cache.NoSpecialDur, err + } + + if len(response.Data.Users) != 1 { + return nil, cache.NoSpecialDur, errors.New("no user with this ID found") + } + + user := response.Data.Users[0] + + return user.Login, cache.NoSpecialDur, nil + } +} diff --git a/internal/twitchapiclient/client.go b/internal/twitchapiclient/client.go new file mode 100644 index 00000000..8154f52e --- /dev/null +++ b/internal/twitchapiclient/client.go @@ -0,0 +1,40 @@ +package twitchapiclient + +import ( + "errors" + "time" + + "github.com/Chatterino/api/pkg/cache" + "github.com/Chatterino/api/pkg/config" + "github.com/nicklaw5/helix" +) + +// New returns a helix.Client that has requested an AppAccessToken and will keep it refreshed every 24h +func New(cfg config.APIConfig) (*helix.Client, *cache.Cache, error) { + if cfg.TwitchClientID == "" { + return nil, nil, errors.New("twitch-client-id is missing, can't make Twitch requests") + } + + if cfg.TwitchClientSecret == "" { + return nil, nil, errors.New("twitch-client-secret is missing, can't make Twitch requests") + } + + apiClient, err := helix.NewClient(&helix.Options{ + ClientID: cfg.TwitchClientID, + ClientSecret: cfg.TwitchClientSecret, + }) + + if err != nil { + return nil, nil, err + } + + waitForFirstAppAccessToken := make(chan struct{}) + + // Initialize methods responsible for refreshing oauth + go initAppAccessToken(apiClient, waitForFirstAppAccessToken) + <-waitForFirstAppAccessToken + + usernameCache := cache.New("helix:username", loadUsername(apiClient), 1*time.Hour) + + return apiClient, usernameCache, nil +} diff --git a/internal/resolvers/twitch/token_refresh.go b/internal/twitchapiclient/token_refresh.go similarity index 98% rename from internal/resolvers/twitch/token_refresh.go rename to internal/twitchapiclient/token_refresh.go index bb1bdcbb..af677b98 100644 --- a/internal/resolvers/twitch/token_refresh.go +++ b/internal/twitchapiclient/token_refresh.go @@ -1,4 +1,4 @@ -package twitch +package twitchapiclient import ( "log"