Ignore websites based on their hosts

CHANGELOG.md

@@ -9,6 +9,7 @@
 - PDF: Generate customized tooltips for PDF files. (#374, #377)
 - Twitter: Generate thumbnails with all images of a tweet. (#373)
 - YouTube: Added support for 'YouTube shorts' URLs. (#299)
+- Minor: Add ability to opt out hostnames from the API. (#405)
 - Fix: SevenTV emotes now resolve correctly. (#281, #288, #307)
 - Fix: YouTube videos are no longer resolved as channels. (#284)
 - Fix: Default resolver no longer crashes when provided url is broken. (#310)

internal/resolvers/default/initialize.go

@@ -26,7 +26,10 @@ const (
 var defaultTooltip = template.Must(template.New("default_tooltip").Parse(defaultTooltipString))
 
 func Initialize(ctx context.Context, cfg config.APIConfig, pool db.Pool, router *chi.Mux, helixClient *helix.Client) {
-	defaultLinkResolver := New(ctx, cfg, pool, helixClient)
+	// Ignored hosts can be added here at request of the hoster
+	ignoredHosts := map[string]struct{}{}
+
+	defaultLinkResolver := New(ctx, cfg, pool, helixClient, ignoredHosts)
 
 	imageCached := stampede.Handler(256, 2*time.Second)
 	generatedValuesCached := stampede.Handler(256, 2*time.Second)

internal/resolvers/default/link_resolver.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/Chatterino/api/internal/db"
@@ -31,11 +32,22 @@ import (
 type LinkResolver struct {
 	customResolvers []resolver.Resolver
 
+	ignoredHosts map[string]struct{}
+
 	linkCache      cache.Cache
 	thumbnailCache cache.Cache
 	generatedCache cache.DependentCache
 }
 
+func (r *LinkResolver) shouldIgnore(u *url.URL) bool {
+	if _, ok := r.ignoredHosts[strings.ToLower(u.Hostname())]; ok {
+		// Ignoring url because host is ignored
+		return true
+	}
+
+	return false
+}
+
 func (r *LinkResolver) HandleRequest(w http.ResponseWriter, req *http.Request) {
 	ctx := req.Context()
 	log := logger.FromContext(ctx)
@@ -70,6 +82,16 @@ func (r *LinkResolver) HandleRequest(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	if r.shouldIgnore(requestUrl) {
+		_, err = resolver.WriteForbiddenURL(w)
+		if err != nil {
+			log.Errorw("Error writing response",
+				"error", err,
+			)
+		}
+		return
+	}
+
 	for _, m := range r.customResolvers {
 		if ctx, result := m.Check(ctx, requestUrl); result {
 			log.Debugw("Run url on custom resolver",
@@ -208,7 +230,7 @@ func (r *LinkResolver) HandleGeneratedValueRequest(w http.ResponseWriter, req *h
 	}
 }
 
-func New(ctx context.Context, cfg config.APIConfig, pool db.Pool, helixClient *helix.Client) *LinkResolver {
+func New(ctx context.Context, cfg config.APIConfig, pool db.Pool, helixClient *helix.Client, ignoredHosts map[string]struct{}) *LinkResolver {
 	generatedCache := cache.NewPostgreSQLDependentCache(ctx, cfg, pool, cache.NewPrefixKeyProvider("default:dependent"))
 
 	customResolvers := []resolver.Resolver{}
@@ -253,6 +275,8 @@ func New(ctx context.Context, cfg config.APIConfig, pool db.Pool, helixClient *h
 	r := &LinkResolver{
 		customResolvers: customResolvers,
 
+		ignoredHosts: ignoredHosts,
+
 		linkCache:      linkCache,
 		thumbnailCache: thumbnailCache,
 		generatedCache: generatedCache,

internal/resolvers/default/link_resolver_test.go

@@ -64,7 +64,11 @@ func TestLinkResolver(t *testing.T) {
 
 	router := chi.NewRouter()
 
-	r := New(ctx, cfg, pool, nil)
+	ignoredHosts := map[string]struct{}{
+		"ignoredhost.com": {},
+	}
+
+	r := New(ctx, cfg, pool, nil, ignoredHosts)
 
 	router.Get("/link_resolver/{url}", r.HandleRequest)
 	router.Get("/thumbnail/{url}", r.HandleThumbnailRequest)
@@ -200,6 +204,24 @@ func TestLinkResolver(t *testing.T) {
 					Message: `Could not fetch link info: Invalid URL`,
 				},
 			},
+			{
+				inputReq:     newLinkResolverRequest(t, ctx, "GET", "https://ignoredhost.com/forsen", nil),
+				inputLinkKey: ts.URL,
+				expected: resolver.Response{
+					Status:  http.StatusForbidden,
+					Link:    "",
+					Message: `Link forbidden`,
+				},
+			},
+			{
+				inputReq:     newLinkResolverRequest(t, ctx, "GET", "https://IgnoredHost.com/forsen", nil),
+				inputLinkKey: ts.URL,
+				expected: resolver.Response{
+					Status:  http.StatusForbidden,
+					Link:    "",
+					Message: `Link forbidden`,
+				},
+			},
 		}
 
 		for _, test := range tests {

pkg/resolver/static_responses.go

@@ -17,6 +17,8 @@ var (
 
 	InvalidURLBytes = []byte(`{"status":400,"message":"Could not fetch link info: Invalid URL"}`)
 
+	ForbiddenURLBytes = []byte(`{"status":403,"message":"Link forbidden"}`)
+
 	// Dynamically created based on config
 	ResponseTooLarge []byte
 )
@@ -33,6 +35,12 @@ func WriteInvalidURL(w http.ResponseWriter) (int, error) {
 	return w.Write(InvalidURLBytes)
 }
 
+func WriteForbiddenURL(w http.ResponseWriter) (int, error) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusForbidden)
+	return w.Write(ForbiddenURLBytes)
+}
+
 func InitializeStaticResponses(ctx context.Context, cfg config.APIConfig) {
 	log := logger.FromContext(ctx)