janitor: automatically refresh branches.json

For SNAPSHOTS the targets and architectures likely change over time.
Automatically create the overview file to lower manual maintenance.

Signed-off-by: Paul Spooren <mail@aparcar.org>

asu/api.py

@@ -59,6 +59,11 @@ def api_latest():
     return redirect("/json/v1/latest.json")
 
 
+@bp.route("/overview")
+def api_v1_overview():
+    return redirect("/json/v1/overview.json")
+
+
 def validate_packages(req):
     if req.get("packages_versions") and not req.get("packages"):
         req["packages"] = req["packages_versions"].keys()
@@ -235,11 +240,6 @@ def return_job_v1(job):
 
 
 # legacy offering /api/overview
-@bp.route("/overview")
-def api_v1_overview():
-    return jsonify(current_app.config["OVERVIEW"])
-
-
 def api_v1_build_get(request_hash):
     job = get_queue().fetch_job(request_hash)
     if not job:

asu/asu.py

@@ -95,40 +95,6 @@ def store_path(path="index.html"):
             filter(lambda b: b.get("enabled"), app.config["BRANCHES"].values()),
         )
     )
-    latest = list(
-        map(
-            lambda b: b["versions"][0],
-            filter(
-                lambda b: b.get("enabled"),
-                app.config["BRANCHES"].values(),
-            ),
-        )
-    )
-
-    app.config["OVERVIEW"] = {
-        "latest": latest,
-        "branches": branches,
-        "server": {
-            "version": __version__,
-            "contact": "mail@aparcar.org",
-            "allow_defaults": app.config["ALLOW_DEFAULTS"],
-        },
-    }
-
-    # legacy
-    (app.config["JSON_PATH"] / "branches.json").write_text(
-        json.dumps(list(branches.values()))
-    )
-
-    # tdb
-    (app.config["JSON_PATH"] / "latest.json").write_text(json.dumps({"latest": latest}))
-
-    (app.config["JSON_PATH"] / "overview.json").write_text(
-        json.dumps(
-            app.config["OVERVIEW"],
-            indent=2,
-        )
-    )
 
     @app.route("/")
     def overview():

asu/janitor.py

@@ -13,6 +13,9 @@
 
 from asu.common import get_redis, is_modified
 
+from asu import __version__
+
+
 bp = Blueprint("janitor", __name__)
 
 
@@ -374,6 +377,64 @@ def update_target_profiles(branch: dict, version: str, target: str) -> str:
     return metadata["arch_packages"]
 
 
+def update_meta_json():
+    latest = list(
+        map(
+            lambda b: b["versions"][0],
+            filter(
+                lambda b: b.get("enabled"),
+                current_app.config["BRANCHES"].values(),
+            ),
+        )
+    )
+
+    branches = dict(
+        map(
+            lambda b: (
+                b["name"],
+                {
+                    **b,
+                    "targets": dict(
+                        map(
+                            lambda a: (a[0].decode(), a[1].decode()),
+                            get_redis().hgetall(f"architecture:{b['name']}").items(),
+                        )
+                    ),
+                },
+            ),
+            filter(
+                lambda b: b.get("enabled"),
+                current_app.config["BRANCHES"].values(),
+            ),
+        )
+    )
+
+    current_app.config["OVERVIEW"] = {
+        "latest": latest,
+        "branches": branches,
+        "server": {
+            "version": __version__,
+            "contact": "mail@aparcar.org",
+            "allow_defaults": current_app.config["ALLOW_DEFAULTS"],
+        },
+    }
+
+    (current_app.config["JSON_PATH"] / "overview.json").write_text(
+        json.dumps(
+            current_app.config["OVERVIEW"],
+            indent=2,
+        )
+    )
+
+    (current_app.config["JSON_PATH"] / "branches.json").write_text(
+        json.dumps(list(branches.values()))
+    )
+
+    (current_app.config["JSON_PATH"] / "latest.json").write_text(
+        json.dumps({"latest": latest})
+    )
+
+
 @bp.cli.command("update")
 @click.option("-i", "--interval", default=10, type=int)
 def update(interval):
@@ -397,6 +458,8 @@ def update(interval):
             current_app.logger.info(f"Update {branch['name']}")
             update_branch(branch)
 
+        update_meta_json()
+
         if interval > 0:
             current_app.logger.info(f"Next reload in { interval } minutes")
             sleep(interval * 60)

tests/test_api.py

@@ -1,11 +1,5 @@
 import pytest
 
-
-def test_api_version(client, app):
-    response = client.get("/api/branches")
-    assert response.status == "200 OK"
-
-
 def test_api_build(client, upstream):
     response = client.post(
         "/api/v1/build",

tests/test_factory.py

@@ -12,16 +12,8 @@ def test_other(app):
     assert app.config["UPSTREAM_URL"] == "http://localhost:8001"
 
 
-def test_json_path_latest(client):
-    response = client.get("/json/latest.json")
-    assert "19.07.7" in response.json["latest"]
-    assert "21.02.0" in response.json["latest"]
-    assert "SNAPSHOT" in response.json["latest"]
-    assert response.status == "200 OK"
-
-
 def test_json_path_branches(client):
-    response = client.get("/json/branches.json")
+    response = client.get("/json/v1/branches.json")
     assert "19.07" == response.json[3]["name"]
     assert "SNAPSHOT" == response.json[0]["name"]
     assert response.status == "200 OK"

tests/test_janitor.py

@@ -33,6 +33,15 @@ def test_update_branch(app, upstream):
     assert (app.config["JSON_PATH"] / "snapshots/overview.json").is_file()
 
 
+def test_update_meta_json(app):
+    with app.app_context():
+        update_meta_json()
+    latest_json = json.loads((app.config["JSON_PATH"] / "latest.json").read_text())
+    assert "19.07.7" in latest_json["latest"]
+    assert "21.02.0" in latest_json["latest"]
+    assert "SNAPSHOT" in latest_json["latest"]
+
+
 def test_parse_packages_file(app, upstream):
     url = (
         app.config["UPSTREAM_URL"]