From 816ac5ea038b867fedbc4f077b77cf418483eec4 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Sun, 2 May 2021 12:22:07 +0200 Subject: [PATCH 01/18] feat: initial oEmbed implementation --- go.mod | 1 + go.sum | 4 ++ internal/resolvers/default/resolver.go | 2 + internal/resolvers/oembed/load.go | 72 ++++++++++++++++++++++++++ internal/resolvers/oembed/model.go | 8 +++ internal/resolvers/oembed/resolver.go | 64 +++++++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100644 internal/resolvers/oembed/load.go create mode 100644 internal/resolvers/oembed/model.go create mode 100644 internal/resolvers/oembed/resolver.go diff --git a/go.mod b/go.mod index ffc26662..21413277 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/PuerkitoBio/goquery v1.6.1 github.com/discord/lilliput v0.0.0-20210410064651-6e127f25858d + github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c github.com/frankban/quicktest v1.12.1 github.com/go-chi/chi/v5 v5.0.3 github.com/golang/mock v1.5.0 diff --git a/go.sum b/go.sum index 1265edb6..52719bdf 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= +github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= @@ -55,6 +57,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/discord/lilliput v0.0.0-20210410064651-6e127f25858d h1:jWQgeT6mu5HOHTYkG38bK3gEmCDPTl93mtXmFeSvFmY= github.com/discord/lilliput v0.0.0-20210410064651-6e127f25858d/go.mod h1:0euuUBAD72MAYRm2ElLaG1h0nBR+CgpfnKc/U6y/uE8= +github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c h1:MEV1LrQtCBGacXajlT4CSuYWbZuLl/qaZVqwoOmwAbU= +github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c/go.mod h1:DjlDZiZGRRKbiJZmiEiiXozsBQAQzHmxwHKFeXifL2g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= diff --git a/internal/resolvers/default/resolver.go b/internal/resolvers/default/resolver.go index ee83b6cd..4594989b 100644 --- a/internal/resolvers/default/resolver.go +++ b/internal/resolvers/default/resolver.go @@ -11,6 +11,7 @@ import ( "github.com/Chatterino/api/internal/resolvers/frankerfacez" "github.com/Chatterino/api/internal/resolvers/imgur" "github.com/Chatterino/api/internal/resolvers/livestreamfails" + "github.com/Chatterino/api/internal/resolvers/oembed" "github.com/Chatterino/api/internal/resolvers/supinic" "github.com/Chatterino/api/internal/resolvers/twitch" "github.com/Chatterino/api/internal/resolvers/twitter" @@ -82,6 +83,7 @@ func New(baseURL string) *R { r.customResolvers = append(r.customResolvers, imgur.New()...) r.customResolvers = append(r.customResolvers, wikipedia.New()...) r.customResolvers = append(r.customResolvers, livestreamfails.New()...) + r.customResolvers = append(r.customResolvers, oembed.New()...) return r } diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go new file mode 100644 index 00000000..363fef0f --- /dev/null +++ b/internal/resolvers/oembed/load.go @@ -0,0 +1,72 @@ +package oembed + +import ( + "bytes" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Chatterino/api/pkg/cache" + "github.com/Chatterino/api/pkg/humanize" + "github.com/Chatterino/api/pkg/resolver" + "github.com/dyatlov/go-oembed/oembed" +) + +func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { + item := oEmbed.FindItem(fullURL) + + data, err := item.FetchOembed(oembed.Options{URL: fullURL}) + + if err != nil { + return &resolver.Response{ + Status: http.StatusInternalServerError, + Message: "oEmbed error: " + resolver.CleanResponse(err.Error()), + }, cache.NoSpecialDur, nil + } + + if data.Status >= 300 { + fmt.Printf("[oEmbed] Skipping url %s because status code is %d\n", fullURL, data.Status) + return &resolver.Response{ + Status: data.Status, + Message: fmt.Sprintf("oEmbed status code: %d", data.Status), + }, cache.NoSpecialDur, nil + } + + infoTooltipData := oEmbedData{data, fullURL} + + infoTooltipData.Title = humanize.Title(infoTooltipData.Title) + infoTooltipData.Description = humanize.Description(infoTooltipData.Description) + infoTooltipData.FullURL = fullURL + + // Build a tooltip using the tooltip template (see tooltipTemplate) with the data we massaged above + var tooltip bytes.Buffer + if err := oEmbedTemplate.Execute(&tooltip, infoTooltipData); err != nil { + return &resolver.Response{ + Status: http.StatusInternalServerError, + Message: "oEmbed template error " + resolver.CleanResponse(err.Error()), + }, cache.NoSpecialDur, nil + } + + resolverResponse := resolver.Response{ + Status: 200, + Tooltip: url.PathEscape(tooltip.String()), + } + + if infoTooltipData.Type == "photo" { + resolverResponse.Thumbnail = infoTooltipData.URL + } + + if infoTooltipData.ThumbnailURL != "" { + + // Some thumbnail URLs, like Streamable's returns // with no schema. + if strings.HasPrefix(infoTooltipData.ThumbnailURL, "//") { + infoTooltipData.ThumbnailURL = "https:" + infoTooltipData.ThumbnailURL + } + + resolverResponse.Thumbnail = infoTooltipData.ThumbnailURL + } + + return &resolverResponse, cache.NoSpecialDur, nil +} diff --git a/internal/resolvers/oembed/model.go b/internal/resolvers/oembed/model.go new file mode 100644 index 00000000..2bdb7746 --- /dev/null +++ b/internal/resolvers/oembed/model.go @@ -0,0 +1,8 @@ +package oembed + +import "github.com/dyatlov/go-oembed/oembed" + +type oEmbedData struct { + *oembed.Info + FullURL string +} diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go new file mode 100644 index 00000000..a48cdee8 --- /dev/null +++ b/internal/resolvers/oembed/resolver.go @@ -0,0 +1,64 @@ +package oembed + +import ( + "bytes" + "encoding/json" + "html/template" + "io/ioutil" + "log" + "net/url" + "time" + + "github.com/Chatterino/api/pkg/cache" + "github.com/Chatterino/api/pkg/resolver" + "github.com/Chatterino/api/pkg/utils" + "github.com/dyatlov/go-oembed/oembed" +) + +const ( + oEmbedTooltipString = `
+{{.ProviderName}}{{ if .Title }} - {{.Title}}{{ end }}
+{{ if .Description }}{{.Description}}{{ end }} +{{ if .AuthorName }}
Author: {{.AuthorName}}{{ end }} +
URL: {{.FullURL}} +
` +) + +var ( + oEmbedTemplate = template.Must(template.New("oEmbedTemplateTooltip").Parse(oEmbedTooltipString)) + + oEmbedCache = cache.New("oEmbed", load, 1*time.Hour) + + oEmbed = oembed.NewOembed() + + excludedHosts = [...]string{"reddit.com", "spotify.com", "kickstarter.com"} +) + +func New() (resolvers []resolver.CustomURLManager) { + data, err := ioutil.ReadFile("./providers.json") + + if err != nil { + log.Println("No providers.json file found, won't do oEmbed parsing") + return + } + + oEmbed.ParseProviders(bytes.NewReader(data)) + + resolvers = append(resolvers, resolver.CustomURLManager{ + Check: func(url *url.URL) bool { + for _, host := range excludedHosts { + if utils.IsSubdomainOf(url, host) { + return false + } + } + + return oEmbed.FindItem(url.String()) != nil + }, + Run: func(url *url.URL) ([]byte, error) { + apiResponse := oEmbedCache.Get(url.String(), nil) + return json.Marshal(apiResponse) + }, + }) + + return +} From 25748b925fbf87623fe3b4a865cf89e28067273f Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 14:19:10 +0200 Subject: [PATCH 02/18] Provide a 'default' list of providers --- data/oembed/README.md | 13 + data/oembed/providers.json | 503 +++++++++++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 data/oembed/README.md create mode 100644 data/oembed/providers.json diff --git a/data/oembed/README.md b/data/oembed/README.md new file mode 100644 index 00000000..115714f4 --- /dev/null +++ b/data/oembed/README.md @@ -0,0 +1,13 @@ +# providers.json + +Original source: https://oembed.com/providers.json (2021-05-09) + +## Removed providers: + +- Kickstarter because its oembed provides less data than using opengrah values +- Reddit because its oembed provides less data than using opengraph values +- Spotify because its oembed provides less data than using opengraph values +- Twitter because we have a rich resolver already +- YouTube because we have a rich resolver already + +And a few more that can be diffed between https://oembed.com/providers.json and our providers.json file diff --git a/data/oembed/providers.json b/data/oembed/providers.json new file mode 100644 index 00000000..bfabb912 --- /dev/null +++ b/data/oembed/providers.json @@ -0,0 +1,503 @@ +[ + { + "provider_name": "CircuitLab", + "provider_url": "https://www.circuitlab.com/", + "endpoints": [ + { + "schemes": ["https://www.circuitlab.com/circuit/*"], + "url": "https://www.circuitlab.com/circuit/oembed/", + "discovery": true + } + ] + }, + { + "provider_name": "Clyp", + "provider_url": "http://clyp.it/", + "endpoints": [ + { + "schemes": ["http://clyp.it/*", "http://clyp.it/playlist/*"], + "url": "http://api.clyp.it/oembed/", + "discovery": true + } + ] + }, + { + "provider_name": "CodePen", + "provider_url": "https://codepen.io", + "endpoints": [ + { + "schemes": ["http://codepen.io/*", "https://codepen.io/*"], + "url": "https://codepen.io/api/oembed" + } + ] + }, + { + "provider_name": "CollegeHumor", + "provider_url": "http://www.collegehumor.com/", + "endpoints": [ + { + "schemes": ["http://www.collegehumor.com/video/*"], + "url": "http://www.collegehumor.com/oembed.{format}", + "discovery": true + } + ] + }, + { + "provider_name": "Dailymotion", + "provider_url": "https://www.dailymotion.com", + "endpoints": [ + { + "schemes": ["https://www.dailymotion.com/video/*"], + "url": "https://www.dailymotion.com/services/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "Deviantart.com", + "provider_url": "http://www.deviantart.com", + "endpoints": [ + { + "schemes": [ + "http://*.deviantart.com/art/*", + "http://*.deviantart.com/*#/d*", + "http://fav.me/*", + "http://sta.sh/*", + "https://*.deviantart.com/art/*", + "https://*.deviantart.com/*/art/*", + "https://sta.sh/*\",", + "https://*.deviantart.com/*#/d*\"" + ], + "url": "http://backend.deviantart.com/oembed" + } + ] + }, + { + "provider_name": "Facebook", + "provider_url": "https://www.facebook.com/", + "endpoints": [ + { + "schemes": [ + "https://www.facebook.com/*/posts/*", + "https://www.facebook.com/*/activity/*", + "https://www.facebook.com/*/photos/*", + "https://www.facebook.com/photo.php?fbid=*", + "https://www.facebook.com/photos/*", + "https://www.facebook.com/permalink.php?story_fbid=*", + "https://www.facebook.com/media/set?set=*", + "https://www.facebook.com/questions/*", + "https://www.facebook.com/notes/*/*/*" + ], + "url": "https://graph.facebook.com/v9.0/oembed_post", + "discovery": false + }, + { + "schemes": [ + "https://www.facebook.com/*/videos/*", + "https://www.facebook.com/video.php?id=*", + "https://www.facebook.com/video.php?v=*" + ], + "url": "https://graph.facebook.com/v9.0/oembed_video", + "discovery": false + }, + { + "schemes": ["https://www.facebook.com/*"], + "url": "https://graph.facebook.com/v9.0/oembed_page", + "discovery": false + } + ] + }, + { + "provider_name": "Flickr", + "provider_url": "https://www.flickr.com/", + "endpoints": [ + { + "schemes": [ + "http://*.flickr.com/photos/*", + "http://flic.kr/p/*", + "https://*.flickr.com/photos/*", + "https://flic.kr/p/*" + ], + "url": "https://www.flickr.com/services/oembed/", + "discovery": true + } + ] + }, + { + "provider_name": "FOX SPORTS Australia", + "provider_url": "http://www.foxsports.com.au", + "endpoints": [ + { + "schemes": [ + "http://fiso.foxsports.com.au/isomorphic-widget/*", + "https://fiso.foxsports.com.au/isomorphic-widget/*" + ], + "url": "https://fiso.foxsports.com.au/oembed" + } + ] + }, + { + "provider_name": "FunnyOrDie", + "provider_url": "http://www.funnyordie.com/", + "endpoints": [ + { + "schemes": ["http://www.funnyordie.com/videos/*"], + "url": "http://www.funnyordie.com/oembed.{format}" + } + ] + }, + { + "provider_name": "Getty Images", + "provider_url": "http://www.gettyimages.com/", + "endpoints": [ + { + "schemes": ["http://gty.im/*"], + "url": "http://embed.gettyimages.com/oembed", + "formats": ["json"] + } + ] + }, + { + "provider_name": "Gfycat", + "provider_url": "https://gfycat.com/", + "endpoints": [ + { + "schemes": [ + "http://gfycat.com/*", + "http://www.gfycat.com/*", + "https://gfycat.com/*", + "https://www.gfycat.com/*" + ], + "url": "https://api.gfycat.com/v1/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "GIPHY", + "provider_url": "https://giphy.com", + "endpoints": [ + { + "schemes": [ + "https://giphy.com/gifs/*", + "https://giphy.com/clips/*", + "http://gph.is/*", + "https://media.giphy.com/media/*/giphy.gif" + ], + "url": "https://giphy.com/services/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "Gyazo", + "provider_url": "https://gyazo.com", + "endpoints": [ + { + "schemes": ["https://gyazo.com/*"], + "url": "https://api.gyazo.com/api/oembed", + "formats": ["json"] + } + ] + }, + { + "provider_name": "Hulu", + "provider_url": "http://www.hulu.com/", + "endpoints": [ + { + "schemes": ["http://www.hulu.com/watch/*"], + "url": "http://www.hulu.com/api/oembed.{format}" + } + ] + }, + { + "provider_name": "iFixit", + "provider_url": "http://www.iFixit.com", + "endpoints": [ + { + "schemes": ["http://www.ifixit.com/Guide/View/*"], + "url": "http://www.ifixit.com/Embed" + } + ] + }, + { + "provider_name": "IFTTT", + "provider_url": "http://www.ifttt.com/", + "endpoints": [ + { + "schemes": ["http://ifttt.com/recipes/*"], + "url": "http://www.ifttt.com/oembed/", + "discovery": true + } + ] + }, + { + "provider_name": "Instagram", + "provider_url": "https://instagram.com", + "endpoints": [ + { + "schemes": [ + "http://instagram.com/*/p/*,", + "http://www.instagram.com/*/p/*,", + "https://instagram.com/*/p/*,", + "https://www.instagram.com/*/p/*,", + "http://instagram.com/p/*", + "http://instagr.am/p/*", + "http://www.instagram.com/p/*", + "http://www.instagr.am/p/*", + "https://instagram.com/p/*", + "https://instagr.am/p/*", + "https://www.instagram.com/p/*", + "https://www.instagr.am/p/*", + "http://instagram.com/tv/*", + "http://instagr.am/tv/*", + "http://www.instagram.com/tv/*", + "http://www.instagr.am/tv/*", + "https://instagram.com/tv/*", + "https://instagr.am/tv/*", + "https://www.instagram.com/tv/*", + "https://www.instagr.am/tv/*" + ], + "url": "https://graph.facebook.com/v9.0/instagram_oembed", + "formats": ["json"] + } + ] + }, + { + "provider_name": "Livestream", + "provider_url": "https://livestream.com/", + "endpoints": [ + { + "schemes": [ + "https://livestream.com/accounts/*/events/*", + "https://livestream.com/accounts/*/events/*/videos/*", + "https://livestream.com/*/events/*", + "https://livestream.com/*/events/*/videos/*", + "https://livestream.com/*/*", + "https://livestream.com/*/*/videos/*" + ], + "url": "https://livestream.com/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "LottieFiles", + "provider_url": "https://lottiefiles.com/", + "endpoints": [ + { + "schemes": ["https://lottiefiles.com/*", "https://*.lottiefiles.com/*"], + "url": "https://embed.lottiefiles.com/oembed", + "discovery": true, + "formats": ["json"] + } + ] + }, + { + "provider_name": "Microsoft Stream", + "provider_url": "https://stream.microsoft.com", + "endpoints": [ + { + "schemes": [ + "https://*.microsoftstream.com/video/*", + "https://*.microsoftstream.com/channel/*" + ], + "url": "https://web.microsoftstream.com/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "Nasjonalbiblioteket", + "provider_url": "https://www.nb.no/", + "endpoints": [ + { + "schemes": ["https://www.nb.no/items/*"], + "url": "https://api.nb.no/catalog/v1/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "On Aol", + "provider_url": "http://on.aol.com/", + "endpoints": [ + { + "schemes": ["http://on.aol.com/video/*"], + "url": "http://on.aol.com/api" + } + ] + }, + { + "provider_name": "RoosterTeeth", + "provider_url": "https://roosterteeth.com", + "endpoints": [ + { + "schemes": ["https://roosterteeth.com/*"], + "url": "https://roosterteeth.com/oembed", + "formats": ["json"], + "discovery": true + } + ] + }, + { + "provider_name": "Runkit", + "provider_url": "https://runkit.com", + "endpoints": [ + { + "schemes": [ + "http://embed.runkit.com/*,", + "https://embed.runkit.com/*," + ], + "url": "https://embed.runkit.com/oembed", + "formats": ["json"] + } + ] + }, + { + "provider_name": "SoundCloud", + "provider_url": "http://soundcloud.com/", + "endpoints": [ + { + "schemes": [ + "http://soundcloud.com/*", + "https://soundcloud.com/*", + "https://soundcloud.app.goog.gl/*" + ], + "url": "https://soundcloud.com/oembed" + } + ] + }, + { + "provider_name": "Stanford Digital Repository", + "provider_url": "https://purl.stanford.edu/", + "endpoints": [ + { + "schemes": ["https://purl.stanford.edu/*"], + "url": "https://purl.stanford.edu/embed.{format}", + "discovery": true + } + ] + }, + { + "provider_name": "Streamable", + "provider_url": "https://streamable.com/", + "endpoints": [ + { + "schemes": ["http://streamable.com/*", "https://streamable.com/*"], + "url": "https://api.streamable.com/oembed.json", + "discovery": true + } + ] + }, + { + "provider_name": "TED", + "provider_url": "https://www.ted.com", + "endpoints": [ + { + "schemes": [ + "http://ted.com/talks/*", + "https://ted.com/talks/*", + "https://www.ted.com/talks/*" + ], + "url": "https://www.ted.com/services/v1/oembed.{format}", + "discovery": true + } + ] + }, + { + "provider_name": "The New York Times", + "provider_url": "https://www.nytimes.com", + "endpoints": [ + { + "schemes": [ + "https://www.nytimes.com/svc/oembed", + "https://nytimes.com/*", + "https://*.nytimes.com/*" + ], + "url": "https://www.nytimes.com/svc/oembed/json/", + "discovery": true + } + ] + }, + { + "provider_name": "TikTok", + "provider_url": "http://www.tiktok.com/", + "endpoints": [ + { + "schemes": ["https://www.tiktok.com/*/video/*"], + "url": "https://www.tiktok.com/oembed" + } + ] + }, + { + "provider_name": "Tumblr", + "provider_url": "https://www.tumblr.com", + "endpoints": [ + { + "schemes": ["https://*.tumblr.com/post/*"], + "url": "https://www.tumblr.com/oembed/1.0" + } + ] + }, + { + "provider_name": "University of Cambridge Map", + "provider_url": "https://map.cam.ac.uk", + "endpoints": [ + { + "schemes": ["https://map.cam.ac.uk/*"], + "url": "https://map.cam.ac.uk/oembed/" + } + ] + }, + { + "provider_name": "Vimeo", + "provider_url": "https://vimeo.com/", + "endpoints": [ + { + "schemes": [ + "https://vimeo.com/*", + "https://vimeo.com/album/*/video/*", + "https://vimeo.com/channels/*/*", + "https://vimeo.com/groups/*/videos/*", + "https://vimeo.com/ondemand/*/*", + "https://player.vimeo.com/video/*" + ], + "url": "https://vimeo.com/api/oembed.{format}", + "discovery": true + } + ] + }, + { + "provider_name": "Wolfram Cloud", + "provider_url": "https://www.wolframcloud.com", + "endpoints": [ + { + "schemes": ["https://*.wolframcloud.com/*"], + "url": "https://www.wolframcloud.com/oembed", + "discovery": true + } + ] + }, + { + "provider_name": "WordPress.com", + "provider_url": "http://wordpress.com/", + "endpoints": [ + { + "url": "http://public-api.wordpress.com/oembed/", + "discovery": true + } + ] + }, + { + "provider_name": "YFrog", + "provider_url": "http://yfrog.com/", + "endpoints": [ + { + "schemes": ["http://*.yfrog.com/*", "http://yfrog.us/*"], + "url": "http://www.yfrog.com/api/oembed", + "formats": ["json"] + } + ] + } +] From 58dac2a4f0901d6a2ed49f7ae44929c51f67f48e Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 14:22:17 +0200 Subject: [PATCH 03/18] Add the option to change path where to find the providers.json file --- internal/resolvers/oembed/resolver.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go index a48cdee8..a6fe3a82 100644 --- a/internal/resolvers/oembed/resolver.go +++ b/internal/resolvers/oembed/resolver.go @@ -35,7 +35,14 @@ var ( ) func New() (resolvers []resolver.CustomURLManager) { - data, err := ioutil.ReadFile("./providers.json") + providersPath := "./providers.json" + + if providersPathEnv, exists := utils.LookupEnv("OEMBED_PROVIDERS_PATH"); exists { + log.Println("[oEmbed] Overriding path of providers.json to", providersPathEnv) + providersPath = providersPathEnv + } + + data, err := ioutil.ReadFile(providersPath) if err != nil { log.Println("No providers.json file found, won't do oEmbed parsing") From 7d2365b22621f0921d41a2a72766eb45785ca449 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 14:24:03 +0200 Subject: [PATCH 04/18] Since we provide our own `providers.json`, remove `excludedHosts` logic --- internal/resolvers/oembed/resolver.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go index a6fe3a82..4bad0d7e 100644 --- a/internal/resolvers/oembed/resolver.go +++ b/internal/resolvers/oembed/resolver.go @@ -30,8 +30,6 @@ var ( oEmbedCache = cache.New("oEmbed", load, 1*time.Hour) oEmbed = oembed.NewOembed() - - excludedHosts = [...]string{"reddit.com", "spotify.com", "kickstarter.com"} ) func New() (resolvers []resolver.CustomURLManager) { @@ -53,12 +51,6 @@ func New() (resolvers []resolver.CustomURLManager) { resolvers = append(resolvers, resolver.CustomURLManager{ Check: func(url *url.URL) bool { - for _, host := range excludedHosts { - if utils.IsSubdomainOf(url, host) { - return false - } - } - return oEmbed.FindItem(url.String()) != nil }, Run: func(url *url.URL) ([]byte, error) { From a7279f60e19d75308e9feca8c01bddd04464069a Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 15:31:54 +0200 Subject: [PATCH 05/18] remove some bad oembed providers --- data/oembed/providers.json | 90 -------------------------------------- 1 file changed, 90 deletions(-) diff --git a/data/oembed/providers.json b/data/oembed/providers.json index bfabb912..e7a7c8be 100644 --- a/data/oembed/providers.json +++ b/data/oembed/providers.json @@ -1,26 +1,4 @@ [ - { - "provider_name": "CircuitLab", - "provider_url": "https://www.circuitlab.com/", - "endpoints": [ - { - "schemes": ["https://www.circuitlab.com/circuit/*"], - "url": "https://www.circuitlab.com/circuit/oembed/", - "discovery": true - } - ] - }, - { - "provider_name": "Clyp", - "provider_url": "http://clyp.it/", - "endpoints": [ - { - "schemes": ["http://clyp.it/*", "http://clyp.it/playlist/*"], - "url": "http://api.clyp.it/oembed/", - "discovery": true - } - ] - }, { "provider_name": "CodePen", "provider_url": "https://codepen.io", @@ -31,17 +9,6 @@ } ] }, - { - "provider_name": "CollegeHumor", - "provider_url": "http://www.collegehumor.com/", - "endpoints": [ - { - "schemes": ["http://www.collegehumor.com/video/*"], - "url": "http://www.collegehumor.com/oembed.{format}", - "discovery": true - } - ] - }, { "provider_name": "Dailymotion", "provider_url": "https://www.dailymotion.com", @@ -136,16 +103,6 @@ } ] }, - { - "provider_name": "FunnyOrDie", - "provider_url": "http://www.funnyordie.com/", - "endpoints": [ - { - "schemes": ["http://www.funnyordie.com/videos/*"], - "url": "http://www.funnyordie.com/oembed.{format}" - } - ] - }, { "provider_name": "Getty Images", "provider_url": "http://www.gettyimages.com/", @@ -157,22 +114,6 @@ } ] }, - { - "provider_name": "Gfycat", - "provider_url": "https://gfycat.com/", - "endpoints": [ - { - "schemes": [ - "http://gfycat.com/*", - "http://www.gfycat.com/*", - "https://gfycat.com/*", - "https://www.gfycat.com/*" - ], - "url": "https://api.gfycat.com/v1/oembed", - "discovery": true - } - ] - }, { "provider_name": "GIPHY", "provider_url": "https://giphy.com", @@ -200,37 +141,6 @@ } ] }, - { - "provider_name": "Hulu", - "provider_url": "http://www.hulu.com/", - "endpoints": [ - { - "schemes": ["http://www.hulu.com/watch/*"], - "url": "http://www.hulu.com/api/oembed.{format}" - } - ] - }, - { - "provider_name": "iFixit", - "provider_url": "http://www.iFixit.com", - "endpoints": [ - { - "schemes": ["http://www.ifixit.com/Guide/View/*"], - "url": "http://www.ifixit.com/Embed" - } - ] - }, - { - "provider_name": "IFTTT", - "provider_url": "http://www.ifttt.com/", - "endpoints": [ - { - "schemes": ["http://ifttt.com/recipes/*"], - "url": "http://www.ifttt.com/oembed/", - "discovery": true - } - ] - }, { "provider_name": "Instagram", "provider_url": "https://instagram.com", From ea8cce9619868e8c160683b27c8218cb2726a736 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 15:36:30 +0200 Subject: [PATCH 06/18] Add support for rich facebook/instagram oEmbed using the `CHATTERINO_API_OEMBED_FACEBOOK_APP_ID` and `CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET` environment variables --- internal/resolvers/oembed/facebook.go | 74 +++++++++++++++++++++++++++ internal/resolvers/oembed/load.go | 18 ++++++- internal/resolvers/oembed/resolver.go | 8 +++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 internal/resolvers/oembed/facebook.go diff --git a/internal/resolvers/oembed/facebook.go b/internal/resolvers/oembed/facebook.go new file mode 100644 index 00000000..9bdea07b --- /dev/null +++ b/internal/resolvers/oembed/facebook.go @@ -0,0 +1,74 @@ +package oembed + +import ( + "encoding/json" + "io/ioutil" + "log" + "net/http" + "net/url" + + "github.com/Chatterino/api/pkg/utils" +) + +var ( + facebookAppAccessToken string +) + +func loadFacebookCredentials() (appID string, appSecret string, exists bool) { + if appID, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_ID"); !exists { + return + } + + if appSecret, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_SECRET"); !exists { + return + } + + return +} + +func initFacebookAppAccessToken(appID, appSecret string) error { + u, err := url.Parse("https://graph.facebook.com/oauth/access_token") + if err != nil { + return err + } + queryVariables := url.Values{} + + queryVariables.Set("client_id", appID) + queryVariables.Set("client_secret", appSecret) + queryVariables.Set("grant_type", "client_credentials") + + u.RawQuery = queryVariables.Encode() + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + type dataxd struct { + AccessToken string `json:"access_token"` + } + + d := &dataxd{} + + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Println("[oEmbed] error loading app access token", err) + return err + } + + err = json.Unmarshal(bytes, &d) + if err != nil { + return err + } + + facebookAppAccessToken = d.AccessToken + + return nil +} diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index 363fef0f..783a3363 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -3,6 +3,7 @@ package oembed import ( "bytes" "fmt" + "log" "net/http" "net/url" "strings" @@ -15,9 +16,22 @@ import ( ) func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { + extraOpts := url.Values{} + item := oEmbed.FindItem(fullURL) - data, err := item.FetchOembed(oembed.Options{URL: fullURL}) + if item.ProviderName == "Facebook" || item.ProviderName == "Instagram" { + // Add facebook token if it exists + if facebookAppAccessToken != "" { + extraOpts.Set("access_token", facebookAppAccessToken) + extraOpts.Set("omitscript", "true") + } + } + + data, err := item.FetchOembed(oembed.Options{ + URL: fullURL, + ExtraOpts: extraOpts, + }) if err != nil { return &resolver.Response{ @@ -27,7 +41,7 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { } if data.Status >= 300 { - fmt.Printf("[oEmbed] Skipping url %s because status code is %d\n", fullURL, data.Status) + log.Printf("[oEmbed] Skipping url %s because status code is %d\n", fullURL, data.Status) return &resolver.Response{ Status: data.Status, Message: fmt.Sprintf("oEmbed status code: %d", data.Status), diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go index 4bad0d7e..3be59900 100644 --- a/internal/resolvers/oembed/resolver.go +++ b/internal/resolvers/oembed/resolver.go @@ -47,6 +47,14 @@ func New() (resolvers []resolver.CustomURLManager) { return } + if facebookAppID, facebookAppSecret, exists := loadFacebookCredentials(); exists { + if err := initFacebookAppAccessToken(facebookAppID, facebookAppSecret); err != nil { + log.Println("[oEmbed] error loading facebook app access token", err) + } else { + log.Println("[oEmbed] Extra rich info loading enabled for Instagram and Facebook") + } + } + oEmbed.ParseProviders(bytes.NewReader(data)) resolvers = append(resolvers, resolver.CustomURLManager{ From d9d8a46763007859955afbea1b6cb1934be6b0dc Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 15:40:12 +0200 Subject: [PATCH 07/18] Add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 918ad3f8..fffc42ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Support for oEmbed resolving for websites customizable with the `resolvers.json` file. See `data/oembed/resolvers.json`. (#139) - Breaking: Environment variable `CHATTERINO_API_CACHE_TWITCH_CLIENT_ID` was renamed to `CHATTERINO_API_TWITCH_CLIENT_ID`. (#144) - Dev, Breaking: Replaced `dankeroni/gotwitch` with `nicklaw5/helix`. This change requires you to add new environment variable: `CHATTERINO_API_TWITCH_CLIENT_SECRET` - it's a client secret generated for your Twitch application. From b8d9a7af9170fc63d840b478b47373872c5d2bc7 Mon Sep 17 00:00:00 2001 From: Rasmus Karlsson Date: Sun, 9 May 2021 15:42:44 +0200 Subject: [PATCH 08/18] Add readme page for the oEmbed resolver package --- internal/resolvers/oembed/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 internal/resolvers/oembed/README.md diff --git a/internal/resolvers/oembed/README.md b/internal/resolvers/oembed/README.md new file mode 100644 index 00000000..969e30bb --- /dev/null +++ b/internal/resolvers/oembed/README.md @@ -0,0 +1,7 @@ +# oEmbed + +The oEmbed resolver requires a `resolvers.json` to be loadable from the application. + +The `CHATTERINO_API_OEMBED_PROVIDERS_PATH` environment variable can be set to change where the file is loaded from, and if no environment variable is set it tries to load the file from `./resolvers.json`. + +If the Facebook and Instagram resolvers are part of your `resolvers.json` file, you can specify the `CHATTERINO_API_OEMBED_FACEBOOK_APP_ID` and `CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET` environment variables to grant Authorization for those requests, giving you rich data for Facebook and Instagram posts. From 59d5b1e52b5fdc4f230240491352a3cf77a0220f Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 17:43:12 +0200 Subject: [PATCH 09/18] Update internal/resolvers/oembed/load.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł --- internal/resolvers/oembed/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index 783a3363..c8175f2a 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -64,7 +64,7 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { } resolverResponse := resolver.Response{ - Status: 200, + Status: http.StatusOK, Tooltip: url.PathEscape(tooltip.String()), } From d46cacd74c0153a89a4f08013e84ea5713488526 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 17:43:30 +0200 Subject: [PATCH 10/18] Update internal/resolvers/oembed/load.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł --- internal/resolvers/oembed/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index c8175f2a..0d5ae856 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -40,7 +40,7 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { }, cache.NoSpecialDur, nil } - if data.Status >= 300 { + if data.Status < http.StatusOK || data.Status > http.StatusMultipleChoices { log.Printf("[oEmbed] Skipping url %s because status code is %d\n", fullURL, data.Status) return &resolver.Response{ Status: data.Status, From e554ca7461ffceff362d2ecd7315960dc9331c92 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 17:43:56 +0200 Subject: [PATCH 11/18] Update internal/resolvers/oembed/load.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Paweł --- internal/resolvers/oembed/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index 0d5ae856..060aa4aa 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -59,7 +59,7 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { if err := oEmbedTemplate.Execute(&tooltip, infoTooltipData); err != nil { return &resolver.Response{ Status: http.StatusInternalServerError, - Message: "oEmbed template error " + resolver.CleanResponse(err.Error()), + Message: "oEmbed template error: " + resolver.CleanResponse(err.Error()), }, cache.NoSpecialDur, nil } From 7bca0ff066eecd0111fe1194888a27e2628bafdb Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 18:47:58 +0200 Subject: [PATCH 12/18] fix: resolve requested changes --- data/oembed/README.md | 6 +++--- internal/resolvers/oembed/facebook.go | 10 +++------- internal/resolvers/oembed/load.go | 12 ++++++------ internal/resolvers/oembed/model.go | 6 +++++- internal/resolvers/oembed/resolver.go | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/data/oembed/README.md b/data/oembed/README.md index 115714f4..4301d77b 100644 --- a/data/oembed/README.md +++ b/data/oembed/README.md @@ -1,8 +1,8 @@ # providers.json -Original source: https://oembed.com/providers.json (2021-05-09) +Original source: (2021-05-09) -## Removed providers: +## Removed providers - Kickstarter because its oembed provides less data than using opengrah values - Reddit because its oembed provides less data than using opengraph values @@ -10,4 +10,4 @@ Original source: https://oembed.com/providers.json (2021-05-09) - Twitter because we have a rich resolver already - YouTube because we have a rich resolver already -And a few more that can be diffed between https://oembed.com/providers.json and our providers.json file +And a few more that can be diffed between and our [providers.json](data/provider.json) file diff --git a/internal/resolvers/oembed/facebook.go b/internal/resolvers/oembed/facebook.go index 9bdea07b..2f7c00f3 100644 --- a/internal/resolvers/oembed/facebook.go +++ b/internal/resolvers/oembed/facebook.go @@ -15,11 +15,11 @@ var ( ) func loadFacebookCredentials() (appID string, appSecret string, exists bool) { - if appID, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_ID"); !exists { + if appID, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_ID"); !exists { return } - if appSecret, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_SECRET"); !exists { + if appSecret, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET"); !exists { return } @@ -51,11 +51,7 @@ func initFacebookAppAccessToken(appID, appSecret string) error { defer resp.Body.Close() - type dataxd struct { - AccessToken string `json:"access_token"` - } - - d := &dataxd{} + d := &facebookTokenResponse{} bytes, err := ioutil.ReadAll(resp.Body) if err != nil { diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index 060aa4aa..50dffe24 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -15,10 +15,10 @@ import ( "github.com/dyatlov/go-oembed/oembed" ) -func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { +func load(requestedURL string, r *http.Request) (interface{}, time.Duration, error) { extraOpts := url.Values{} - item := oEmbed.FindItem(fullURL) + item := oEmbed.FindItem(requestedURL) if item.ProviderName == "Facebook" || item.ProviderName == "Instagram" { // Add facebook token if it exists @@ -29,7 +29,7 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { } data, err := item.FetchOembed(oembed.Options{ - URL: fullURL, + URL: requestedURL, ExtraOpts: extraOpts, }) @@ -41,18 +41,18 @@ func load(fullURL string, r *http.Request) (interface{}, time.Duration, error) { } if data.Status < http.StatusOK || data.Status > http.StatusMultipleChoices { - log.Printf("[oEmbed] Skipping url %s because status code is %d\n", fullURL, data.Status) + log.Printf("[oEmbed] Skipping url %s because status code is %d\n", requestedURL, data.Status) return &resolver.Response{ Status: data.Status, Message: fmt.Sprintf("oEmbed status code: %d", data.Status), }, cache.NoSpecialDur, nil } - infoTooltipData := oEmbedData{data, fullURL} + infoTooltipData := oEmbedData{data, requestedURL} infoTooltipData.Title = humanize.Title(infoTooltipData.Title) infoTooltipData.Description = humanize.Description(infoTooltipData.Description) - infoTooltipData.FullURL = fullURL + infoTooltipData.RequestedURL = requestedURL // Build a tooltip using the tooltip template (see tooltipTemplate) with the data we massaged above var tooltip bytes.Buffer diff --git a/internal/resolvers/oembed/model.go b/internal/resolvers/oembed/model.go index 2bdb7746..f158ba51 100644 --- a/internal/resolvers/oembed/model.go +++ b/internal/resolvers/oembed/model.go @@ -4,5 +4,9 @@ import "github.com/dyatlov/go-oembed/oembed" type oEmbedData struct { *oembed.Info - FullURL string + RequestedURL string +} + +type facebookTokenResponse struct { + AccessToken string `json:"access_token"` } diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go index 3be59900..b39f4662 100644 --- a/internal/resolvers/oembed/resolver.go +++ b/internal/resolvers/oembed/resolver.go @@ -43,7 +43,7 @@ func New() (resolvers []resolver.CustomURLManager) { data, err := ioutil.ReadFile(providersPath) if err != nil { - log.Println("No providers.json file found, won't do oEmbed parsing") + log.Println("[oEmbed] No providers.json file found, won't do oEmbed parsing") return } From 78d621dd02e48bf5371236e389e7ef39b7e9c47e Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 18:48:59 +0200 Subject: [PATCH 13/18] fix: resolve requested changes Does it need to be this verbose? --- internal/resolvers/oembed/load.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/resolvers/oembed/load.go b/internal/resolvers/oembed/load.go index 50dffe24..d5e56bd2 100644 --- a/internal/resolvers/oembed/load.go +++ b/internal/resolvers/oembed/load.go @@ -36,7 +36,7 @@ func load(requestedURL string, r *http.Request) (interface{}, time.Duration, err if err != nil { return &resolver.Response{ Status: http.StatusInternalServerError, - Message: "oEmbed error: " + resolver.CleanResponse(err.Error()), + Message: "Something went wrong loading this oEmbed.\noEmbed error: " + resolver.CleanResponse(err.Error()), }, cache.NoSpecialDur, nil } @@ -44,7 +44,7 @@ func load(requestedURL string, r *http.Request) (interface{}, time.Duration, err log.Printf("[oEmbed] Skipping url %s because status code is %d\n", requestedURL, data.Status) return &resolver.Response{ Status: data.Status, - Message: fmt.Sprintf("oEmbed status code: %d", data.Status), + Message: fmt.Sprintf("This oEmbed couldn't be loaded in.\noEmbed status code: %d", data.Status), }, cache.NoSpecialDur, nil } From 8444928c77be276324a20d12e0e051e24dccb893 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 18:51:08 +0200 Subject: [PATCH 14/18] docs: Refer users to the README for oEmbed --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fffc42ac..3b58b071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Support for oEmbed resolving for websites customizable with the `resolvers.json` file. See `data/oembed/resolvers.json`. (#139) +- Added support for customizable oEmbed resolving for websites with the `resolvers.json` file. See [`data/oembed/resolvers.json`](data/oembed/resolvers.json). Three new environment variables can be set. See [`internal/resolvers/oembed/README.md`](internal/resolvers/oembed/README.md) (#139) - Breaking: Environment variable `CHATTERINO_API_CACHE_TWITCH_CLIENT_ID` was renamed to `CHATTERINO_API_TWITCH_CLIENT_ID`. (#144) - Dev, Breaking: Replaced `dankeroni/gotwitch` with `nicklaw5/helix`. This change requires you to add new environment variable: `CHATTERINO_API_TWITCH_CLIENT_SECRET` - it's a client secret generated for your Twitch application. From 4882c0f7aef50653d983f73856f2c4813fc59997 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 18:51:23 +0200 Subject: [PATCH 15/18] docs: Add apiKeys explanation. --- docs/apikeys.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/apikeys.md b/docs/apikeys.md index 3964d9c5..d459a78f 100644 --- a/docs/apikeys.md +++ b/docs/apikeys.md @@ -41,3 +41,13 @@ 2. Select "Anonymous usage without user authorization" 3. Fill in the rest of the information 4. Copy the Client ID value (which is what we need) + +## Facebook & Instagram + +1. Head here: https://developers.facebook.com/ +2. Tap on "My Apps" in the top right corner. +3. Click `Create App` and select `Consumer`. +4. Fill the provided information, and you will get a security check. If you don't see the security check, disable your ad-blocker and try the security check again. +5. In "Add Products to Your App", under "oEmbed" click on `Set up`. +6. Fill in "Privacy Policy URL" and "User Data Collection" with relevant information. +7. In the left menu, click on "Settings" and select "Basic". Copy both "App ID" and "App Secret". From 4079882f0ea562762e83d14490b1d98d06694dfc Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 18:53:27 +0200 Subject: [PATCH 16/18] xd --- internal/resolvers/oembed/resolver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/resolvers/oembed/resolver.go b/internal/resolvers/oembed/resolver.go index b39f4662..ccc1b918 100644 --- a/internal/resolvers/oembed/resolver.go +++ b/internal/resolvers/oembed/resolver.go @@ -20,7 +20,7 @@ const ( {{.ProviderName}}{{ if .Title }} - {{.Title}}{{ end }}
{{ if .Description }}{{.Description}}{{ end }} {{ if .AuthorName }}
Author: {{.AuthorName}}{{ end }} -
URL: {{.FullURL}} +
URL: {{.RequestedURL}} ` ) From 542e9a7acfbf86535e2040b883700c6e9d72716e Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 19:34:25 +0200 Subject: [PATCH 17/18] fix: Log missing environment variables --- internal/resolvers/oembed/facebook.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/resolvers/oembed/facebook.go b/internal/resolvers/oembed/facebook.go index 2f7c00f3..e6d4fa1a 100644 --- a/internal/resolvers/oembed/facebook.go +++ b/internal/resolvers/oembed/facebook.go @@ -16,10 +16,12 @@ var ( func loadFacebookCredentials() (appID string, appSecret string, exists bool) { if appID, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_ID"); !exists { + log.Println("No CHATTERINO_API_OEMBED_FACEBOOK_APP_ID specified, won't do special responses for Facebook or Instagram oEmbed") return } if appSecret, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET"); !exists { + log.Println("No CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET specified, won't do special responses for Facebook or Instagram oEmbed") return } From a06f3a4e4c3e1b72777b0612730e021fd6f23df2 Mon Sep 17 00:00:00 2001 From: Karar Al-Remahy Date: Mon, 17 May 2021 20:59:32 +0200 Subject: [PATCH 18/18] fix: woops --- internal/resolvers/oembed/facebook.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/resolvers/oembed/facebook.go b/internal/resolvers/oembed/facebook.go index e6d4fa1a..db65b00b 100644 --- a/internal/resolvers/oembed/facebook.go +++ b/internal/resolvers/oembed/facebook.go @@ -15,12 +15,12 @@ var ( ) func loadFacebookCredentials() (appID string, appSecret string, exists bool) { - if appID, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_ID"); !exists { + if appID, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_ID"); !exists { log.Println("No CHATTERINO_API_OEMBED_FACEBOOK_APP_ID specified, won't do special responses for Facebook or Instagram oEmbed") return } - if appSecret, exists = utils.LookupEnv("CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET"); !exists { + if appSecret, exists = utils.LookupEnv("OEMBED_FACEBOOK_APP_SECRET"); !exists { log.Println("No CHATTERINO_API_OEMBED_FACEBOOK_APP_SECRET specified, won't do special responses for Facebook or Instagram oEmbed") return }