diff --git a/cubedash/_pages.py b/cubedash/_pages.py index 87ea3594e..bb97760e3 100644 --- a/cubedash/_pages.py +++ b/cubedash/_pages.py @@ -51,6 +51,10 @@ _DEFAULT_ARRIVALS_DAYS: int = app.config.get("CUBEDASH_DEFAULT_ARRIVALS_DAY_COUNT", 14) +_ROBOTS_TXT_DEFAULT = ( + "User-Agent: *\nAllow: /\nDisallow: /products/*/*\nDisallow: /stac/**" +) + # Add server timings to http headers. if app.config.get("CUBEDASH_SHOW_PERF_TIMES", False): _monitoring.init_app_monitoring() @@ -674,6 +678,13 @@ def about_page(): ) +@app.route("/robots.txt") +def robots_txt(): + return utils.as_json( + flask.current_app.config.get("ROBOTS_TXT", _ROBOTS_TXT_DEFAULT) + ) + + @app.route("/") def default_redirect(): """Redirect to default starting page.""" diff --git a/integration_tests/test_page_loads.py b/integration_tests/test_page_loads.py index 0ddc95f9b..ff37e907d 100644 --- a/integration_tests/test_page_loads.py +++ b/integration_tests/test_page_loads.py @@ -961,6 +961,14 @@ def check_doc_start_has_hint(hint: str, url: str): ) +def test_get_robots(client: FlaskClient): + """ + Check that robots.txt is correctly served from root + """ + text, rv = get_text_response(client, "/robots.txt") + assert "User-Agent: *" in text + + def test_all_give_404s(client: FlaskClient): """ We should get 404 messages, not exceptions, for missing things.