Refactor product filtering and loading for generic scripts

Create a common function to list and load product configurations for generic scripts. Generic scripts are scripts that handle multiple product based on some identifier (URL, coordinates...). This make them more simple to read and maintain, while making the way they work much more consistent. This is only the first step in that process, the end goal being: - providing utilities to simplify the maintenance of existing scripts / development of new scripts (while keeping those script independent), - being able to change globaly how things work (such as #51)
endoflife-date · May 13, 2023 · aa4aa8f · aa4aa8f
1 parent 3b40b72
commit aa4aa8f
Show file tree

Hide file tree

Showing 7 changed files with 97 additions and 147 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@ website
 vendor
 .bundle
 pyrightconfig.json
+__pycache__
diff --git a/src/common/endoflife.py b/src/common/endoflife.py
@@ -0,0 +1,27 @@
+import frontmatter
+
+from glob import glob
+from os import path
+
+
+def list_products(method, products_filter=None, pathname = "website/products"):
+    """Return a list of products that are using the same given update method.
+    """
+    products_with_method = {}
+
+    for product_file in glob(f"{pathname}/*.md"):
+        product_name = path.splitext(path.basename(product_file))[0]
+        if products_filter and product_name != products_filter:
+            continue
+
+        with open(product_file, "r") as f:
+            data = frontmatter.load(f)
+            if "auto" in data:
+                configs = list(filter(
+                    lambda config: method in config.keys(),
+                    data["auto"]
+                ))
+                if len(configs) > 0:
+                    products_with_method[product_name] = configs
+
+    return products_with_method
diff --git a/src/distrowatch.py b/src/distrowatch.py
@@ -1,15 +1,13 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
 from bs4 import BeautifulSoup
 from liquid import Template
+from common import endoflife
 
-# Same as used in Ruby (update.rb)
-DEFAULT_TAG_TEMPLATE = (
+METHOD = 'distrowatch'
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 
@@ -42,32 +40,21 @@ def fetch_releases(distrowatch_id, regex, template):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "distrowatch":
-                            update_product(product_name, config)
+def update_product(product_name, configs):
+    releases = {}
 
+    for config in configs:
+        t = config.get("template", DEFAULT_TAG_TEMPLATE)
+        if "regex" in config:
+            regex = config["regex"]
+            releases = releases | fetch_releases(config[METHOD], regex, t)
 
-def update_product(product_name, config):
-    t = config.get("template", DEFAULT_TAG_TEMPLATE)
-    if "regex" in config:
-        print("::group::%s" % product_name)
-        r = fetch_releases(config["distrowatch"], config["regex"], t)
-        with open("releases/%s.json" % product_name, "w") as f:
-            f.write(json.dumps(r, indent=2))
-        print("::endgroup::")
+    with open("releases/%s.json" % product_name, "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")
diff --git a/src/github-releases.py b/src/github-releases.py
@@ -1,13 +1,11 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import subprocess
+from common import endoflife
 
+METHOD = "github_releases"
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
-AUTO_KEY = "github_releases"
 
 
 # This script is using the GitHub CLI with the GraphQL API in order to retrieve
@@ -68,34 +66,18 @@ def fetch_releases(repo_id, regex):
 
 
 def update_product(product_name, configs):
-    print("::group::%s" % product_name)
     releases = {}
 
     for config in configs:
         config = config if "regex" in config else config | {"regex": REGEX}
-        releases = releases | fetch_releases(config[AUTO_KEY], config["regex"])
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
 
     with open(f"releases/{product_name}.json", "w") as f:
         f.write(json.dumps(releases, indent=2))
-    print("::endgroup::")
-
-
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
 
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                configs = list(filter(lambda config: AUTO_KEY in config.keys(), data["auto"]))
-                if len(configs) > 0:
-                    update_product(product_name, configs)
 
-
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")
diff --git a/src/maven.py b/src/maven.py
@@ -1,13 +1,11 @@
-from glob import glob
-import os
 import sys
 import json
-import frontmatter
 import urllib.request
 import datetime
 import re
+from common import endoflife
 
-# Major.Minor + Optional Patch, no RC, nightly releases.
+METHOD = "maven"
 VERSION_REGEX = r'^\d+\.\d+(\.\d+)?$'
 
 
@@ -59,37 +57,21 @@ def fetch_releases(package_identifier):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            releases = {}
-            found_maven = False
-            print("::group::%s" % product_name)
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, _ in config.items():
-                        if key == "maven":
-                            found_maven = True
-                            releases = releases | fetch_releases(config["maven"])
-            if found_maven:
-                write_file(product_name, releases)
-            print("::endgroup::")
-
-
-def write_file(product_name, releases):
-    with open("releases/%s.json" % product_name, "w") as f:
+def update_product(product_name, configs):
+    releases = {}
+
+    for config in configs:
+        releases = releases | fetch_releases(config[METHOD])
+
+    with open(f"releases/{product_name}.json", "w") as f:
         f.write(json.dumps(dict(
             # sort by date then version (desc)
             sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
         ), indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")
diff --git a/src/npm.py b/src/npm.py
@@ -1,12 +1,11 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
+from common import endoflife
 
-DEFAULT_TAG_TEMPLATE = (
+METHOD = "npm"
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
@@ -35,34 +34,21 @@ def fetch_releases(npm_id, regex):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
+def update_product(product_name, configs):
+    releases = {}
 
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "npm":
-                            update_product(product_name, config)
+    for config in configs:
+        config = {"regex": REGEX} | config
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
 
+    with open(f"releases/{product_name}.json", "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
-def update_product(product_name, config):
-    if "npm" in config:
-        print(f"::group::{product_name}")
-        config = {"regex": REGEX} | config
-        r = fetch_releases(config["npm"], config["regex"])
-        print("::endgroup::")
 
-        with open(f"releases/{product_name}.json", "w") as f:
-            f.write(json.dumps(r, indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")
diff --git a/src/pypi.py b/src/pypi.py
@@ -1,14 +1,12 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
 from datetime import datetime
+from common import endoflife
 
-# Same as used in Ruby (update.rb)
-DEFAULT_TAG_TEMPLATE = (
+METHOD = "pypi"
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
@@ -37,32 +35,19 @@ def fetch_releases(pypi_id, regex):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "pypi":
-                            update_product(product_name, config)
-
+def update_product(product_name, configs):
+    releases = {}
 
-def update_product(product_name, config):
-    if "pypi" in config:
-        print("::group::%s" % product_name)
+    for config in configs:
         config = {"regex": REGEX} | config
-        r = fetch_releases(config["pypi"], config["regex"])
-        with open("releases/%s.json" % product_name, "w") as f:
-            f.write(json.dumps(r, indent=2))
-        print("::endgroup::")
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
+
+    with open(f"releases/{product_name}.json", "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")