ws: Prevent search engine indexing with robots.txt

Fixes #17401
cockpit-project · Dec 13, 2024 · 44df489 · 44df489
1 parent ab2d7be
commit 44df489
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/src/ws/cockpithandlers.c b/src/ws/cockpithandlers.c
@@ -689,6 +689,17 @@ cockpit_handler_default (CockpitWebServer *server,
   path = cockpit_web_response_get_path (response);
   g_return_val_if_fail (path != NULL, FALSE);
 
+  /* robots.txt is unauthorized and works on any directory */
+  if g_str_has_suffix (path, "/robots.txt")
+    {
+      g_autoptr(GHashTable) out_headers = cockpit_web_server_new_table ();
+      g_hash_table_insert (out_headers, g_strdup ("Content-Type"), g_strdup ("text/plain"));
+      const char *body ="User-agent: *\nDisallow: /\n";
+      g_autoptr(GBytes) content = g_bytes_new_static (body, strlen (body));
+      cockpit_web_response_content (response, out_headers, content, NULL);
+      return TRUE;
+    }
+
   resource = g_str_has_prefix (path, "/cockpit/") ||
              g_str_has_prefix (path, "/cockpit+") ||
              g_str_equal (path, "/cockpit");

diff --git a/test/verify/check-connection b/test/verify/check-connection
@@ -782,6 +782,12 @@ class TestConnection(testlib.MachineCase):
         self.assertIn("HTTP/1.1 200", out)
         self.assertNotIn("<html>", out)
 
+        # robots.txt from any directory and with/without TLS, and without auth
+        expected = "User-agent: *\nDisallow: /\n"
+        self.assertEqual(m.execute("curl http://localhost:9000/robots.txt"), expected)
+        self.assertEqual(m.execute("curl http://localhost:9000/somedir/robots.txt"), expected)
+        self.assertEqual(m.execute("curl -k https://localhost:9000/robots.txt"), expected)
+
     @testlib.nondestructive
     def testHeadRequest(self):
         m = self.machine