Fix the /robots.txt response

Instead of newlines it was returning the text \n.
Content type was set to application/json instead of text/plain.

cubedash/_pages.py

@@ -8,7 +8,7 @@
 from datacube.model import DatasetType, Range
 from datacube.scripts.dataset import build_dataset_info
 from dateutil import tz
-from flask import abort, redirect, request, url_for
+from flask import abort, make_response, redirect, request, url_for
 from werkzeug.datastructures import MultiDict
 from werkzeug.exceptions import HTTPException
 
@@ -680,9 +680,11 @@ def about_page():
 
 @app.route("/robots.txt")
 def robots_txt():
-    return utils.as_json(
-        flask.current_app.config.get("ROBOTS_TXT", _ROBOTS_TXT_DEFAULT)
+    resp = make_response(
+        flask.current_app.config.get("ROBOTS_TXT", _ROBOTS_TXT_DEFAULT), 200
     )
+    resp.headers["Content-Type"] = "text/plain"
+    return resp
 
 
 @app.route("/")

integration_tests/test_page_loads.py

@@ -966,7 +966,14 @@ def test_get_robots(client: FlaskClient):
     Check that robots.txt is correctly served from root
     """
     text, rv = get_text_response(client, "/robots.txt")
-    assert "User-Agent: *" in text
+    assert "User-Agent:" in text
+
+    num_lines = len(text.split("\n"))
+    assert num_lines > 1, "robots.txt should have multiple lines"
+
+    assert (
+        rv.headers["Content-Type"] == "text/plain"
+    ), "robots.txt content-type should be text/plain"
 
 
 def test_all_give_404s(client: FlaskClient):