From a3b23000bcb386d26c3e8cffbd85fed6d7204ea9 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Mon, 29 Sep 2014 11:46:06 -0700 Subject: [PATCH 1/2] Simplify and expand bot rules Blocks all bots instead of just named bots, and blocks thumbnails since they're all dynamically generated and bots can really slow the site down --- static/robots.txt | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/static/robots.txt b/static/robots.txt index bd80be4c..eab72c60 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,29 +1,3 @@ -User-Agent: Googlebot -Disallow: /data/ - -User-Agent: Googlebot-Image -Disallow: /data/ - -User-Agent: Baiduspider -Disallow: /data/ - -User-Agent: Bingbot -Disallow: /data/ - -User-Agent: YandexBot -Disallow: /data/ - -User-Agent: Blekkobot -Disallow: /data/ - -User-Agent: discoverybot -Disallow: /data/ - -User-Agent: ia_archiver -Disallow: /data/ - -User-Agent: msnbot -Disallow: /data/ - -User-Agent: Yahoo! Slurp +User-Agent: * +Disallow: /*thumbnail.jpg$ Disallow: /data/ From 6ffbf48eb488ef89277856eea713cbcbd8db5859 Mon Sep 17 00:00:00 2001 From: Jeremy Echols Date: Wed, 1 Oct 2014 09:59:25 -0700 Subject: [PATCH 2/2] Stop spidering our tiles!! --- static/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/static/robots.txt b/static/robots.txt index eab72c60..65010e4c 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,3 +1,4 @@ User-Agent: * Disallow: /*thumbnail.jpg$ Disallow: /data/ +Disallow: /images/tiles/