From ecf87a38a3c86241e6bdbe7f648ca771ba64f4a3 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Fri, 2 Feb 2024 18:44:57 +0000 Subject: [PATCH 01/12] First pass --- ecosystem/cli/website.py | 66 ++++++++++++++++++++++++++++------------ tox.ini | 2 +- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index b08fa09acd..e8a4102d5e 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -22,48 +22,64 @@ class CliWebsite: def __init__(self, root_path: Optional[str] = None): """CliWebsite class.""" - self.current_dir = root_path or os.path.abspath(os.getcwd()) - resources_dir = Path(self.current_dir, "ecosystem/resources") - self.dao = DAO(path=resources_dir) - self.label_descriptions = { + pass + + def _load_from_file(self, resources_dir: Path): + """ + Loads: + * Projects list + * Website strings + * Label descriptions + * Jinja templates + """ + # Projects list + dao = DAO(path=resources_dir) + projects = sorted( + dao.get_all(), + key=lambda item: ( + -(item.stars or 0), + item.name, + ), + ) + + # Label descriptions + label_descriptions = { item["name"]: item["description"] for item in json.loads(Path(resources_dir, "labels.json").read_text()) } - self.web_data = toml.loads((resources_dir / "website.toml").read_text()) - def build_website(self): - """Generates the ecosystem web page reading the TOML files.""" - # pylint: disable=too-many-locals + # Website strings + web_data = toml.loads((resources_dir / "website.toml").read_text()) + + # Jinja templates environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) - projects = self.dao.storage.read() - projects_sorted = sorted( - projects.items(), - key=lambda item: ( - -item[1].stars if item[1].stars is not None else 0, - item[1].name, - ), - ) templates = { "website": environment.get_template("webpage.html.jinja"), "card": environment.get_template("card.html.jinja"), "tag": environment.get_template("tag.html.jinja"), "link": environment.get_template("link.html.jinja"), } - sections = {group["id"]: group for group in self.web_data["groups"]} + return projects, web_data, label_descriptions, templates + + def _build_html(self, projects, web_data, label_descriptions, templates) -> str: + """ + Take all data needed to build the website and produce a HTML string. + """ + sections = {group["id"]: group for group in web_data["groups"]} for section in sections.values(): section.setdefault("html", "") max_chars_description_visible = 400 min_chars_description_hidden = 100 count_read_more = 1 - for _, repo in projects_sorted: + for repo in projects: # Card tags tags = "" for index, label in enumerate(repo.labels): tags += templates["tag"].render( color="purple", text=label, - tooltip=self.label_descriptions[label], + tooltip=label_descriptions[label], # Sometimes tooltips are clipped by the browser window. # While not perfect, the following line solves 95% of cases alignment="bottom" if (index % 3) == 2 else "bottom-left", @@ -109,6 +125,16 @@ def build_website(self): sections[repo.group]["html"] += card return templates["website"].render( - header=self.web_data["header"], + header=web_data["header"], sections=sections.values(), ) + + def build_website(self, resources: str, output: str): + """ + Generates the ecosystem web page from data in `resources` dir, writing to `output` dir. + """ + resources_dir = Path(resources) + html = self._build_html(*self._load_from_file(resources_dir)) + with open(Path(output, "index.html"), "w") as f: + f.write(html) + diff --git a/tox.ini b/tox.ini index acc40d0bda..93f4eb6391 100644 --- a/tox.ini +++ b/tox.ini @@ -37,4 +37,4 @@ commands = black {posargs} ecosystem tests --check allowlist_externals = bash basepython = python3 commands = - bash -ec "python manager.py website build_website > website/index.html" + bash -ec "python manager.py website build_website --resources ecosystem/resources --output website" From 11ecb1cd9b492ccf179c550ac52e28328ff5a49c Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Fri, 2 Feb 2024 18:52:36 +0000 Subject: [PATCH 02/12] class -> function --- ecosystem/cli/__init__.py | 2 +- ecosystem/cli/website.py | 228 ++++++++++++++++++-------------------- manager.py | 4 +- tox.ini | 2 +- 4 files changed, 110 insertions(+), 126 deletions(-) diff --git a/ecosystem/cli/__init__.py b/ecosystem/cli/__init__.py index 11d7f6829d..b04f897656 100644 --- a/ecosystem/cli/__init__.py +++ b/ecosystem/cli/__init__.py @@ -1,4 +1,4 @@ """CLI.""" from .members import CliMembers -from .website import CliWebsite +from .website import build_website from .ci import CliCI diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index e8a4102d5e..d38c1d641e 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -9,132 +9,116 @@ from ecosystem.daos import DAO - -class CliWebsite: - """CliMembers class. - Entrypoint for all CLI website commands. - - Each public method of this class is CLI command - and arguments for method are options/flags for this command. - - Ex: `python manager.py website build_website` +def build_website(resources: str, output: str): """ + Generates the ecosystem web page from data in `resources` dir, writing to `output` dir. + """ + resources_dir = Path(resources) + html = _build_html(*_load_from_file(resources_dir)) + with open(Path(output, "index.html"), "w") as f: + f.write(html) - def __init__(self, root_path: Optional[str] = None): - """CliWebsite class.""" - pass - - def _load_from_file(self, resources_dir: Path): - """ - Loads: - * Projects list - * Website strings - * Label descriptions - * Jinja templates - """ - # Projects list - dao = DAO(path=resources_dir) - projects = sorted( - dao.get_all(), - key=lambda item: ( - -(item.stars or 0), - item.name, - ), - ) - - # Label descriptions - label_descriptions = { - item["name"]: item["description"] - for item in json.loads(Path(resources_dir, "labels.json").read_text()) - } - - # Website strings - web_data = toml.loads((resources_dir / "website.toml").read_text()) - - # Jinja templates - environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) - templates = { - "website": environment.get_template("webpage.html.jinja"), - "card": environment.get_template("card.html.jinja"), - "tag": environment.get_template("tag.html.jinja"), - "link": environment.get_template("link.html.jinja"), - } - return projects, web_data, label_descriptions, templates - - def _build_html(self, projects, web_data, label_descriptions, templates) -> str: - """ - Take all data needed to build the website and produce a HTML string. - """ - sections = {group["id"]: group for group in web_data["groups"]} - for section in sections.values(): - section.setdefault("html", "") - - max_chars_description_visible = 400 - min_chars_description_hidden = 100 - count_read_more = 1 - for repo in projects: - # Card tags - tags = "" - for index, label in enumerate(repo.labels): - tags += templates["tag"].render( - color="purple", - text=label, - tooltip=label_descriptions[label], - # Sometimes tooltips are clipped by the browser window. - # While not perfect, the following line solves 95% of cases - alignment="bottom" if (index % 3) == 2 else "bottom-left", - ) - - # Card links - links = "" - for url, link_text in ( - (repo.url, "repository"), - (repo.website, "website"), - (repo.reference_paper, "paper"), - (repo.documentation, "documentation"), - ): - if url: - links += templates["link"].render(url=url, place=link_text) - - # Card description - if ( - len(repo.description) - max_chars_description_visible - >= min_chars_description_hidden - ): - description = [ - repo.description[:max_chars_description_visible], - repo.description[max_chars_description_visible:], - ] - id_read_more = str(count_read_more) - count_read_more += 1 - else: - description = [repo.description, ""] - id_read_more = "None" - - # Create the card - card = templates["card"].render( - title=repo.name, - tags=tags, - description_visible=description[0], - description_hidden=description[1], - id_read_more=id_read_more, - links=links, +def _load_from_file(resources_dir: Path): + """ + Loads: + * Projects list + * Website strings + * Label descriptions + * Jinja templates + """ + # Projects list + dao = DAO(path=resources_dir) + projects = sorted( + dao.get_all(), + key=lambda item: ( + -(item.stars or 0), + item.name, + ), + ) + + # Label descriptions + label_descriptions = { + item["name"]: item["description"] + for item in json.loads(Path(resources_dir, "labels.json").read_text()) + } + + # Website strings + web_data = toml.loads((resources_dir / "website.toml").read_text()) + + # Jinja templates + environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) + templates = { + "website": environment.get_template("webpage.html.jinja"), + "card": environment.get_template("card.html.jinja"), + "tag": environment.get_template("tag.html.jinja"), + "link": environment.get_template("link.html.jinja"), + } + return projects, web_data, label_descriptions, templates + +def _build_html(projects, web_data, label_descriptions, templates) -> str: + """ + Take all data needed to build the website and produce a HTML string. + """ + sections = {group["id"]: group for group in web_data["groups"]} + for section in sections.values(): + section.setdefault("html", "") + + max_chars_description_visible = 400 + min_chars_description_hidden = 100 + count_read_more = 1 + for repo in projects: + # Card tags + tags = "" + for index, label in enumerate(repo.labels): + tags += templates["tag"].render( + color="purple", + text=label, + tooltip=label_descriptions[label], + # Sometimes tooltips are clipped by the browser window. + # While not perfect, the following line solves 95% of cases + alignment="bottom" if (index % 3) == 2 else "bottom-left", ) - # Adding the card to a section - sections[repo.group]["html"] += card - - return templates["website"].render( - header=web_data["header"], - sections=sections.values(), + # Card links + links = "" + for url, link_text in ( + (repo.url, "repository"), + (repo.website, "website"), + (repo.reference_paper, "paper"), + (repo.documentation, "documentation"), + ): + if url: + links += templates["link"].render(url=url, place=link_text) + + # Card description + if ( + len(repo.description) - max_chars_description_visible + >= min_chars_description_hidden + ): + description = [ + repo.description[:max_chars_description_visible], + repo.description[max_chars_description_visible:], + ] + id_read_more = str(count_read_more) + count_read_more += 1 + else: + description = [repo.description, ""] + id_read_more = "None" + + # Create the card + card = templates["card"].render( + title=repo.name, + tags=tags, + description_visible=description[0], + description_hidden=description[1], + id_read_more=id_read_more, + links=links, ) - def build_website(self, resources: str, output: str): - """ - Generates the ecosystem web page from data in `resources` dir, writing to `output` dir. - """ - resources_dir = Path(resources) - html = self._build_html(*self._load_from_file(resources_dir)) - with open(Path(output, "index.html"), "w") as f: - f.write(html) + # Adding the card to a section + sections[repo.group]["html"] += card + return templates["website"].render( + header=web_data["header"], + sections=sections.values(), + ) diff --git a/manager.py b/manager.py index af916ef15a..ebf5651a07 100644 --- a/manager.py +++ b/manager.py @@ -20,14 +20,14 @@ """ import fire -from ecosystem.cli import CliMembers, CliWebsite, CliCI +from ecosystem.cli import CliMembers, CliCI, build_website if __name__ == "__main__": fire.Fire( { "members": CliMembers, - "website": CliWebsite, + "build": build_website, "ci": CliCI, } ) diff --git a/tox.ini b/tox.ini index 93f4eb6391..76bf62c7c6 100644 --- a/tox.ini +++ b/tox.ini @@ -37,4 +37,4 @@ commands = black {posargs} ecosystem tests --check allowlist_externals = bash basepython = python3 commands = - bash -ec "python manager.py website build_website --resources ecosystem/resources --output website" + bash -ec "python manager.py build --resources ecosystem/resources --output website" From f5806e168043520cd282386a4d9489418acf0877 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Fri, 2 Feb 2024 19:08:20 +0000 Subject: [PATCH 03/12] Make pip installable --- ecosystem/__init__.py | 12 ++++++++++++ manager.py | 13 ++----------- setup.py | 3 +++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ecosystem/__init__.py b/ecosystem/__init__.py index a31b58d0ca..f203cd5b72 100644 --- a/ecosystem/__init__.py +++ b/ecosystem/__init__.py @@ -1 +1,13 @@ """Ecosystem main module.""" +import fire + +from ecosystem.cli import CliMembers, CliCI, build_website + +def main(): + fire.Fire( + { + "members": CliMembers, + "build": build_website, + "ci": CliCI, + } + ) diff --git a/manager.py b/manager.py index ebf5651a07..7aff540fa1 100644 --- a/manager.py +++ b/manager.py @@ -18,16 +18,7 @@ python manager.py website build_website" ``` """ -import fire - -from ecosystem.cli import CliMembers, CliCI, build_website - +from ecosystem import main if __name__ == "__main__": - fire.Fire( - { - "members": CliMembers, - "build": build_website, - "ci": CliCI, - } - ) + main() diff --git a/setup.py b/setup.py index ebb3a2ec18..e39884b716 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,9 @@ setuptools.setup( name="ecosystem", description="Ecosystem", + entry_points = { + 'console_scripts': ['ecosystem=ecosystem:main'], + }, long_description=long_description, packages=setuptools.find_packages(), install_requires=install_requires, From 6100b17ceabf3b61d52944e9a7e9115b4d38a636 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 12:39:26 +0000 Subject: [PATCH 04/12] lint --- ecosystem/cli/website.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index d38c1d641e..7903063f52 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -1,8 +1,6 @@ """CliWebsite class for controlling all CLI functions.""" from pathlib import Path -from typing import Optional import json -import os import toml from jinja2 import Environment, FileSystemLoader @@ -15,8 +13,8 @@ def build_website(resources: str, output: str): """ resources_dir = Path(resources) html = _build_html(*_load_from_file(resources_dir)) - with open(Path(output, "index.html"), "w") as f: - f.write(html) + with open(Path(output, "index.html"), "w") as file: + file.write(html) def _load_from_file(resources_dir: Path): """ From b52ac72d2f4c05caf91165e3541370a62d20c41e Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 13:37:42 +0000 Subject: [PATCH 05/12] Handle package data --- ecosystem/cli/website.py | 4 ++-- ecosystem/daos/dao.py | 2 +- setup.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 7903063f52..93327564b8 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -3,7 +3,7 @@ import json import toml -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, PackageLoader from ecosystem.daos import DAO @@ -44,7 +44,7 @@ def _load_from_file(resources_dir: Path): web_data = toml.loads((resources_dir / "website.toml").read_text()) # Jinja templates - environment = Environment(loader=FileSystemLoader("ecosystem/html_templates/")) + environment = Environment(loader=PackageLoader("ecosystem", "html_templates/")) templates = { "website": environment.get_template("webpage.html.jinja"), "card": environment.get_template("card.html.jinja"), diff --git a/ecosystem/daos/dao.py b/ecosystem/daos/dao.py index 2c98444593..0354a2e12a 100644 --- a/ecosystem/daos/dao.py +++ b/ecosystem/daos/dao.py @@ -43,7 +43,7 @@ def read(self) -> dict: { url (str): repo (Repository) } """ data = {} - for path in self.toml_dir.glob("*"): + for path in self.toml_dir.glob("*.toml"): repo = Repository.from_dict(toml.load(path)) data[repo.url] = repo return data diff --git a/setup.py b/setup.py index e39884b716..dc40622250 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ }, long_description=long_description, packages=setuptools.find_packages(), + package_data={"ecosystem": ["html_templates/*.jinja"]}, install_requires=install_requires, python_requires='>=3.6' ) From e2a0e1510b2cc18152f20a5175b7ebc0715ea7ec Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 13:38:09 +0000 Subject: [PATCH 06/12] lint --- ecosystem/__init__.py | 1 + ecosystem/cli/website.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/ecosystem/__init__.py b/ecosystem/__init__.py index f203cd5b72..b025d320b8 100644 --- a/ecosystem/__init__.py +++ b/ecosystem/__init__.py @@ -3,6 +3,7 @@ from ecosystem.cli import CliMembers, CliCI, build_website + def main(): fire.Fire( { diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 93327564b8..3eef717d4a 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -7,6 +7,7 @@ from ecosystem.daos import DAO + def build_website(resources: str, output: str): """ Generates the ecosystem web page from data in `resources` dir, writing to `output` dir. @@ -16,6 +17,7 @@ def build_website(resources: str, output: str): with open(Path(output, "index.html"), "w") as file: file.write(html) + def _load_from_file(resources_dir: Path): """ Loads: @@ -53,6 +55,7 @@ def _load_from_file(resources_dir: Path): } return projects, web_data, label_descriptions, templates + def _build_html(projects, web_data, label_descriptions, templates) -> str: """ Take all data needed to build the website and produce a HTML string. From 81e5a09a98f6e2721e92960a872b32af38f0ea10 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 13:38:26 +0000 Subject: [PATCH 07/12] Update ecosystem/cli/website.py Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- ecosystem/cli/website.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 3eef717d4a..20a647834b 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -14,8 +14,7 @@ def build_website(resources: str, output: str): """ resources_dir = Path(resources) html = _build_html(*_load_from_file(resources_dir)) - with open(Path(output, "index.html"), "w") as file: - file.write(html) + Path(output, "index.html").write_text(html) def _load_from_file(resources_dir: Path): From be9fd82d397e19149a5dea9eccc399a47c7647a3 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 13:56:58 +0000 Subject: [PATCH 08/12] Moar lint --- ecosystem/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecosystem/__init__.py b/ecosystem/__init__.py index b025d320b8..ae93ef22df 100644 --- a/ecosystem/__init__.py +++ b/ecosystem/__init__.py @@ -5,6 +5,7 @@ def main(): + # pylint: disable=missing-function-docstring fire.Fire( { "members": CliMembers, From a0bb22a44d85d8258ed578864ae38bc03ae1b874 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 14:24:40 +0000 Subject: [PATCH 09/12] Update ecosystem/cli/website.py Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- ecosystem/cli/website.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 20a647834b..51e2628a54 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -8,7 +8,7 @@ from ecosystem.daos import DAO -def build_website(resources: str, output: str): +def build_website(resources: str, output: str) -> None: """ Generates the ecosystem web page from data in `resources` dir, writing to `output` dir. """ From a33d22afc2e62041cd9f27f107e095d3d64b3f20 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 14:35:19 +0000 Subject: [PATCH 10/12] Add return type --- ecosystem/cli/website.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 20a647834b..a568732f5c 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -1,11 +1,15 @@ """CliWebsite class for controlling all CLI functions.""" +from __future__ import annotations + + from pathlib import Path import json import toml -from jinja2 import Environment, PackageLoader +from jinja2 import Environment, PackageLoader, Template from ecosystem.daos import DAO +from ecosystem.models.repository import Repository def build_website(resources: str, output: str): @@ -17,13 +21,18 @@ def build_website(resources: str, output: str): Path(output, "index.html").write_text(html) -def _load_from_file(resources_dir: Path): +def _load_from_file( + resources_dir: Path, +) -> tuple[ + list[Repository], dict[str, str | dict], dict[str, str], dict[str, Template] +]: """ - Loads: - * Projects list - * Website strings - * Label descriptions - * Jinja templates + Loads website data from file. + Returns: + * Projects: List of Repository objects from `members` folder. + * Web data: Strings (title / descriptions etc.) from `website.toml`. + * Label descriptions: from `labels.json`. + * Jinja templates: from `html_templates` folder. """ # Projects list dao = DAO(path=resources_dir) From a5a07e7ede327354a02bc6cd74fdda54c598094f Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 14:39:35 +0000 Subject: [PATCH 11/12] Fix lying type hint --- ecosystem/cli/website.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 52f9cbb20d..8b90d47086 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -3,6 +3,7 @@ from pathlib import Path +from typing import Any import json import toml @@ -24,7 +25,7 @@ def build_website(resources: str, output: str) -> None: def _load_from_file( resources_dir: Path, ) -> tuple[ - list[Repository], dict[str, str | dict], dict[str, str], dict[str, Template] + list[Repository], dict[str, Any], dict[str, str], dict[str, Template] ]: """ Loads website data from file. From 0a2062f810586dcb468c1971123e50ce85b05c97 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 5 Feb 2024 14:41:08 +0000 Subject: [PATCH 12/12] lint --- ecosystem/cli/website.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ecosystem/cli/website.py b/ecosystem/cli/website.py index 8b90d47086..e129d23ccc 100644 --- a/ecosystem/cli/website.py +++ b/ecosystem/cli/website.py @@ -24,9 +24,7 @@ def build_website(resources: str, output: str) -> None: def _load_from_file( resources_dir: Path, -) -> tuple[ - list[Repository], dict[str, Any], dict[str, str], dict[str, Template] -]: +) -> tuple[list[Repository], dict[str, Any], dict[str, str], dict[str, Template]]: """ Loads website data from file. Returns: