From 44df4891a594b75e19e1816a66b3c411bfbd964e Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 13 Dec 2024 09:27:47 +0100 Subject: [PATCH] ws: Prevent search engine indexing with robots.txt Fixes #17401 --- src/ws/cockpithandlers.c | 11 +++++++++++ test/verify/check-connection | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/src/ws/cockpithandlers.c b/src/ws/cockpithandlers.c index c75c3db5a2d3..2a17d34ae01d 100644 --- a/src/ws/cockpithandlers.c +++ b/src/ws/cockpithandlers.c @@ -689,6 +689,17 @@ cockpit_handler_default (CockpitWebServer *server, path = cockpit_web_response_get_path (response); g_return_val_if_fail (path != NULL, FALSE); + /* robots.txt is unauthorized and works on any directory */ + if g_str_has_suffix (path, "/robots.txt") + { + g_autoptr(GHashTable) out_headers = cockpit_web_server_new_table (); + g_hash_table_insert (out_headers, g_strdup ("Content-Type"), g_strdup ("text/plain")); + const char *body ="User-agent: *\nDisallow: /\n"; + g_autoptr(GBytes) content = g_bytes_new_static (body, strlen (body)); + cockpit_web_response_content (response, out_headers, content, NULL); + return TRUE; + } + resource = g_str_has_prefix (path, "/cockpit/") || g_str_has_prefix (path, "/cockpit+") || g_str_equal (path, "/cockpit"); diff --git a/test/verify/check-connection b/test/verify/check-connection index c0a919f3ea78..f00f91f09351 100755 --- a/test/verify/check-connection +++ b/test/verify/check-connection @@ -782,6 +782,12 @@ class TestConnection(testlib.MachineCase): self.assertIn("HTTP/1.1 200", out) self.assertNotIn("", out) + # robots.txt from any directory and with/without TLS, and without auth + expected = "User-agent: *\nDisallow: /\n" + self.assertEqual(m.execute("curl http://localhost:9000/robots.txt"), expected) + self.assertEqual(m.execute("curl http://localhost:9000/somedir/robots.txt"), expected) + self.assertEqual(m.execute("curl -k https://localhost:9000/robots.txt"), expected) + @testlib.nondestructive def testHeadRequest(self): m = self.machine