diff --git a/tools/lint_software_list.py b/tools/lint_software_list.py index f94582062..db31aca5f 100755 --- a/tools/lint_software_list.py +++ b/tools/lint_software_list.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -''' -Lint {clients,servers,libraries}.json -''' +""" +Lint software.json +""" from typing import Any import json @@ -9,35 +9,32 @@ import sys VALID_ENTRY_KEYS = { - 'platforms', - 'name', - 'doap', - 'url', - 'categories', + "platforms", + "name", + "doap", + "url", + "categories", } -def emit_violation(entry_name: str, - message: str, - warning: bool = False - ) -> None: - '''Prints warnings and errors''' - prefix = 'WARN' if warning else 'ERROR' - print(f'{prefix}: entry {entry_name!r}: {message}', file=sys.stderr) +def emit_violation(entry_name: str, message: str, warning: bool = False) -> None: + """Prints warnings and errors""" + prefix = "WARN" if warning else "ERROR" + print(f"{prefix}: entry {entry_name!r}: {message}", file=sys.stderr) -def check_entries(entries: dict[str, Any]) -> int: - '''Checks entries for violations and returns their count''' +def check_entries(entries: list[dict[str, Any]]) -> int: + """Checks entries for violations and returns their count""" violations = 0 previous_name = None previous_key = None for entry in entries: - key = entry['name'].casefold() + key = entry["name"].casefold() if previous_key is not None and previous_key > key: emit_violation( - entry['name'], - f'should be placed in front of {previous_name!r} (all entries must be ' - f'ordered alphabetically by case-folded name)' + entry["name"], + f"should be placed in front of {previous_name!r} (all entries must be " + f"ordered alphabetically by case-folded name)", ) violations += 1 @@ -47,46 +44,46 @@ def check_entries(entries: dict[str, Any]) -> int: if unknown: emit_violation( - entry['name'], - f'has unknown keys: {", ".join(map(repr, unknown))}' + entry["name"], f'has unknown keys: {", ".join(map(repr, unknown))}' ) violations += 1 if missing: emit_violation( - entry['name'], - f'misses the following required properties: ' + entry["name"], + f"misses the following required properties: " f'{", ".join(map(repr, missing))} ' - f'(see other entries for a reference)' + f"(see other entries for a reference)", ) violations += 1 - supported_platforms = entry.get('platforms', []) + supported_platforms = entry.get("platforms", []) - sorted_platforms = sorted(supported_platforms, - key=lambda x: x.casefold()) + sorted_platforms = sorted(supported_platforms, key=lambda x: x.casefold()) if sorted_platforms != supported_platforms: emit_violation( - entry['name'], - f'platform order must be: ' + entry["name"], + f"platform order must be: " f'{", ".join(map(repr, sorted_platforms))} ' - f'(platforms must be ordered alphabetically)' + f"(platforms must be ordered alphabetically)", ) violations += 1 - previous_key, previous_name = key, entry['name'] + previous_key, previous_name = key, entry["name"] return violations -if __name__ == '__main__': +if __name__ == "__main__": base_path = os.path.dirname(os.path.abspath(sys.argv[0])) - input_file = os.path.join(base_path, '../data/software.json') - with open(input_file, 'rb') as data_file: + input_file = os.path.join(base_path, "../data/software.json") + with open(input_file, "rb") as data_file: data = json.load(data_file) violations_count = check_entries(data) if violations_count: - print(f'Found {violations_count} severe violations. Please fix them.', - file=sys.stderr) + print( + f"Found {violations_count} severe violations. Please fix them.", + file=sys.stderr, + ) sys.exit(1) diff --git a/tools/newsletter_email.py b/tools/newsletter_email.py index 128557d40..74fcae407 100644 --- a/tools/newsletter_email.py +++ b/tools/newsletter_email.py @@ -1,56 +1,61 @@ #!/usr/bin/env python3 -'''In order to have newsletter e-mails formatted properly, +"""In order to have newsletter e-mails formatted properly, we need to add inline style attributes to some elements. This tool takes a post URL for input, processes its HTML, -and stores it afterwards ready for copy and paste.''' +and stores it afterwards ready for copy and paste.""" from http import HTTPStatus import requests from bs4 import BeautifulSoup +from bs4 import Tag def process(input_url: str) -> None: - '''Processes page content for sending via e-mail.''' + """Processes page content for sending via e-mail.""" - print('Processing...') + print("Processing...") with requests.get(input_url, timeout=5) as response: if response.status_code != HTTPStatus.OK: - print('Could not fetch URL:', input_url) + print("Could not fetch URL:", input_url) return - soup = BeautifulSoup(response.text, 'html.parser') + soup = BeautifulSoup(response.text, "html.parser") - article = soup.find('article', {'role': 'main'}) + article = soup.find("article", {"role": "main"}) if article is None: - print('Could not find post’s article element.') + print("Could not find post’s article element.") return + assert isinstance(article, Tag) # Remove social share box, since it uses FontAwesome icons # (not available in emails) - social_share = article.find('section', {'id': 'social-share'}) - if social_share is not None: + social_share = article.find("section", {"id": "social-share"}) + if isinstance(social_share, Tag): social_share.decompose() # Add body padding - article['style'] = 'padding: 2em;' + article["style"] = "padding: 2em;" # Change color and text decoration of heading - header_box = article.find('div', {'class': 'header-internal'}) - link = header_box.find('a') - link['style'] = 'text-decoration: none; color: #333;' + header_box = article.find("div", {"class": "header-internal"}) + assert isinstance(header_box, Tag) + link = header_box.find("a") + assert isinstance(link, Tag) + link["style"] = "text-decoration: none; color: #333;" # Change post meta color - meta_span = header_box.find('span', {'class': 'text-body-secondary'}) - meta_span['style'] = 'color: gray;' + meta_span = header_box.find("span", {"class": "text-body-secondary"}) + assert isinstance(meta_span, Tag) + meta_span["style"] = "color: gray;" # Improve rendering of figures - figures = article.find_all('figure') + figures = article.find_all("figure") for figure in figures: - img = figure.find('img') - img['style'] = 'max-width: 100%;' + img = figure.find("img") + img["style"] = "max-width: 100%;" - with open('newsletter-mail.html', 'w', encoding='utf-8') as html_file: + with open("newsletter-mail.html", "w", encoding="utf-8") as html_file: html_file.write(str(article)) print( 'All done! Please copy and paste contents from "newsletter-mail.html" ' @@ -58,13 +63,13 @@ def process(input_url: str) -> None: ) -if __name__ == '__main__': - print(50 * '=') +if __name__ == "__main__": + print(50 * "=") print( - 'This tool processes newsletter posts for emails.\n' - 'It takes a post URL, processes its content, ' - 'and saves HTML ready for copy and paste.' + "This tool processes newsletter posts for emails.\n" + "It takes a post URL, processes its content, " + "and saves HTML ready for copy and paste." ) - print(50 * '=') - url = input('Please paste the URL you want to process: ') + print(50 * "=") + url = input("Please paste the URL you want to process: ") process(url) diff --git a/tools/prepare_compliance.py b/tools/prepare_compliance.py index ec3321f9b..1b0047c37 100644 --- a/tools/prepare_compliance.py +++ b/tools/prepare_compliance.py @@ -1,7 +1,7 @@ -''' +""" Adds compliance ratings to software_list_doap.json via compliancer (https://code.zash.se/compliancer/) -''' +""" import json import os @@ -12,93 +12,99 @@ from colorama import Fore from colorama import Style from packaging.version import Version as V - from util import download_file -DOWNLOAD_PATH = Path('downloads') -DATA_PATH = Path('data') -COMPLIANCE_SUITE_URL = 'https://xmpp.org/extensions/xep-0459.xml' -COMPLIANCER_BUILD_URL = 'https://prosody.im/files/compliance' +DOWNLOAD_PATH = Path("downloads") +DATA_PATH = Path("data") +COMPLIANCE_SUITE_URL = "https://xmpp.org/extensions/xep-0459.xml" +COMPLIANCER_BUILD_URL = "https://prosody.im/files/compliance" def generate_compliance_json() -> None: - ''' + """ Runs the 'compliancer' tool to generate a 'comppliance_suite.json' file - ''' + """ try: - result = subprocess.check_output([ - 'lua', - f'{DOWNLOAD_PATH}/compliancer', - '-v', - f'{DOWNLOAD_PATH}/compliance-suite.xml']) + result = subprocess.check_output( + [ + "lua", + f"{DOWNLOAD_PATH}/compliancer", + "-v", + f"{DOWNLOAD_PATH}/compliance-suite.xml", + ] + ) json_result = json.loads(result) - with open(DATA_PATH / 'compliance_suite.json', - 'w', - encoding='utf-8') as compliance_suite_file: + with open( + DATA_PATH / "compliance_suite.json", "w", encoding="utf-8" + ) as compliance_suite_file: json.dump(json_result, compliance_suite_file, indent=4) except subprocess.CalledProcessError as err: print(err) def check_packages_compliance() -> None: - ''' + """ Runs the 'compliancer' tool against every package's DOAP file and adds the result to '{clients,libraries,servers}_list.doap' - ''' + """ + def add_compliance_data() -> None: - with open(DATA_PATH / 'software_list_doap.json', - 'rb') as json_file: + with open(DATA_PATH / "software_list_doap.json", "rb") as json_file: package_list = json.load(json_file) for name, props in package_list.items(): compliance_data = compliance_dict.pop(name, None) if compliance_data is None: - props['badges'] = {} + props["badges"] = {} continue - props['badges'] = compliance_data['badges'] + props["badges"] = compliance_data["badges"] - with open(DATA_PATH / 'software_list_doap.json', - 'w', - encoding='utf-8') as clients_data_file: + with open( + DATA_PATH / "software_list_doap.json", "w", encoding="utf-8" + ) as clients_data_file: json.dump(package_list, clients_data_file, indent=4) - compliance_dict: dict[ - str, dict[str, str | dict[str, list[str]]]] = {} + compliance_dict: dict[str, dict[str, str | dict[str, list[str]]]] = {} - for subdir, _dirs, files in os.walk(f'{DOWNLOAD_PATH}/doap_files'): + for subdir, _dirs, files in os.walk(f"{DOWNLOAD_PATH}/doap_files"): for file in files: try: - result = subprocess.check_output([ - 'lua', - f'{DOWNLOAD_PATH}/compliancer', - '-v', - f'{DOWNLOAD_PATH}/compliance-suite.xml', - os.path.join(subdir, file)]) - json_result = json.loads(result.decode('unicode_escape')) - compliance_dict[json_result['name']] = json_result + result = subprocess.check_output( + [ + "lua", + f"{DOWNLOAD_PATH}/compliancer", + "-v", + f"{DOWNLOAD_PATH}/compliance-suite.xml", + os.path.join(subdir, file), + ] + ) + json_result = json.loads(result.decode("unicode_escape")) + compliance_dict[json_result["name"]] = json_result except subprocess.CalledProcessError as err: print(err) add_compliance_data() for _name, props in compliance_dict.items(): - if props['badges']: - print(f'{Fore.YELLOW}Compliance data available, but no match for' - f'{Style.RESET_ALL}:', - props['name']) + if props["badges"]: + print( + f"{Fore.YELLOW}Compliance data available, but no match for" + f"{Style.RESET_ALL}:", + props["name"], + ) -if __name__ == '__main__': +if __name__ == "__main__": # Make sure we're using Lua >= 5.2 - lua_version_string = subprocess.check_output( - ['lua', '-v']).decode('unicode_escape')[4:9] - if V(lua_version_string) < V('5.2.0'): - print('Lua >= 5.2.0 required') + lua_version_string = subprocess.check_output(["lua", "-v"]).decode( + "unicode_escape" + )[4:9] + if V(lua_version_string) < V("5.2.0"): + print("Lua >= 5.2.0 required") sys.exit(1) - download_file( - COMPLIANCE_SUITE_URL, Path('compliance-suite.xml')) - download_file(COMPLIANCER_BUILD_URL, Path('compliancer')) + download_file(COMPLIANCE_SUITE_URL, Path("compliance-suite.xml")) + download_file(COMPLIANCER_BUILD_URL, Path("compliancer")) generate_compliance_json() check_packages_compliance() diff --git a/tools/prepare_rfc_list.py b/tools/prepare_rfc_list.py index 44943b244..e70e95dfe 100755 --- a/tools/prepare_rfc_list.py +++ b/tools/prepare_rfc_list.py @@ -1,7 +1,7 @@ -''' +""" This file is used to download RFC references and convert them to a single JSON file -''' +""" from typing import Any import json @@ -41,115 +41,99 @@ 8084, 8266, 8284, - 8600 + 8600, ] -BASIC_RFC_NUMBERS = [ - 6120, - 6121, - 7395, - 7590, - 7622 -] +BASIC_RFC_NUMBERS = [6120, 6121, 7395, 7590, 7622] -SELFHOSTED_RFCS = [ - 3920, - 3921, - 3922, - 3923, - 4622, - 4854, - 5122, - 6120, - 6121, - 6122 -] +SELFHOSTED_RFCS = [3920, 3921, 3922, 3923, 4622, 4854, 5122, 6120, 6121, 6122] -BIB_XML_PATH = 'https://xml2rfc.tools.ietf.org/public/rfc/bibxml' +BIB_XML_PATH = "https://xml2rfc.tools.ietf.org/public/rfc/bibxml" def get_rfc_data(number: int) -> dict[str, Any]: - ''' + """ Downloads and parses RFC references and builds an RFC list with additional parameters (e.g. selfhosted). Stores data in rfc_list.json - ''' - print(f'Get RFC data for RFC {number}') + """ + print(f"Get RFC data for RFC {number}") try: - request = requests.get( - f'{BIB_XML_PATH}/reference.RFC.{number}.xml', timeout=5) + request = requests.get(f"{BIB_XML_PATH}/reference.RFC.{number}.xml", timeout=5) except requests.exceptions.RequestException as err: - sys.exit(f'Error while downloading reference for ' - f'RFC {number} ({err})') + sys.exit(f"Error while downloading reference for " f"RFC {number} ({err})") if not 200 >= request.status_code < 400: - sys.exit(f'Error while downloading reference for ' - f'RFC {number} ({request.status_code})') + sys.exit( + f"Error while downloading reference for " + f"RFC {number} ({request.status_code})" + ) try: root = fromstring(request.content) except ParseError: - sys.exit(f'Error while parsing RFC reference for RFC {number}') + sys.exit(f"Error while parsing RFC reference for RFC {number}") authors: str | None = None title: str | None = None date: str | None = None abstract: str | None = None for item in root.iter(): - if item.tag == 'title': + if item.tag == "title": title = item.text - if item.tag == 'date': - date = item.attrib.get('year') - if item.tag == 'author': + if item.tag == "date": + date = item.attrib.get("year") + if item.tag == "author": if authors is None: - authors = item.attrib.get('fullname') + authors = item.attrib.get("fullname") else: authors += f', {item.attrib.get("fullname")}' - if item.tag == 'abstract': - abstract = item.find('t').text + if item.tag == "abstract": + abstract = item.find("t").text obsoletes: str | None = None obsoleted_by: str | None = None if number == 3920: - obsoleted_by = '6120' + obsoleted_by = "6120" if number == 3921: - obsoleted_by = '6121' + obsoleted_by = "6121" if number == 4622: - obsoleted_by = '5122' + obsoleted_by = "5122" if number == 5122: - obsoletes = '4622' + obsoletes = "4622" if number == 6120: - obsoletes = '3920' + obsoletes = "3920" if number == 6121: - obsoletes = '3921' + obsoletes = "3921" if number == 7248: - obsoleted_by = '8084' + obsoleted_by = "8084" if number == 7700: - obsoleted_by = '8266' + obsoleted_by = "8266" if number == 8084: - obsoletes = '7248' + obsoletes = "7248" if number == 8266: - obsoletes = '7700' + obsoletes = "7700" basic = bool(number in BASIC_RFC_NUMBERS) selfhosted = bool(number in SELFHOSTED_RFCS) return { - 'number': number, - 'title': title, - 'date': date, - 'authors': authors, - 'abstract': abstract, - 'obsoletes': obsoletes, - 'obsoleted_by': obsoleted_by, - 'basic': basic, - 'selfhosted': selfhosted, + "number": number, + "title": title, + "date": date, + "authors": authors, + "abstract": abstract, + "obsoletes": obsoletes, + "obsoleted_by": obsoleted_by, + "basic": basic, + "selfhosted": selfhosted, } + def build_rfc_list() -> None: - ''' + """ Generates rfc_list.json from downloaded data - ''' + """ base_path = os.path.dirname(os.path.abspath(sys.argv[0])) rfcs: list[dict[str, Any]] = [] @@ -157,15 +141,13 @@ def build_rfc_list() -> None: for result in results: rfcs.append(result) - rfcs = sorted(rfcs, key=lambda d: d['number']) + rfcs = sorted(rfcs, key=lambda d: d["number"]) - with open(f'{base_path}/../data/rfc_list.json', - 'w', - encoding='utf-8') as json_file: + with open(f"{base_path}/../data/rfc_list.json", "w", encoding="utf-8") as json_file: json.dump(rfcs, json_file, indent=4) - print(f'RFC list prepared successfully ({len(RFC_NUMBERS)} RFCs)') + print(f"RFC list prepared successfully ({len(RFC_NUMBERS)} RFCs)") -if __name__ == '__main__': +if __name__ == "__main__": build_rfc_list() diff --git a/tools/prepare_software_list.py b/tools/prepare_software_list.py index 0cef4ba0e..f7bbfd4fc 100644 --- a/tools/prepare_software_list.py +++ b/tools/prepare_software_list.py @@ -1,7 +1,7 @@ -''' +""" Download / prepare / process XMPP DOAP files for the software list Requires: Pillow, python-slugify -''' +""" from typing import Any import json @@ -20,40 +20,39 @@ from PIL import UnidentifiedImageError from PIL.Image import Resampling from slugify import slugify - from util import download_file from util import initialize_directory -SOFTWARE_PATH = Path('content/software') -DATA_PATH = Path('data') -DOWNLOAD_PATH = Path('downloads') -STATIC_PATH = Path('static') -STATIC_DOAP_PATH = STATIC_PATH / 'doap' -LOGOS_PATH = STATIC_PATH / 'images' / 'packages' - -DOAP_NS = 'http://usefulinc.com/ns/doap#' -XMPP_NS = 'https://linkmauve.fr/ns/xmpp-doap#' -SCHEMA_NS = 'https://schema.org/' -RDF_RESOURCE = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource' -DOAP_NAME = f'.//{{{DOAP_NS}}}name' -DOAP_SHORTDESC = f'.//{{{DOAP_NS}}}shortdesc' -DOAP_HOMEPAGE = f'.//{{{DOAP_NS}}}homepage' -DOAP_OS = f'.//{{{DOAP_NS}}}os' -DOAP_PROGRAMMING_LANGUAGE = f'.//{{{DOAP_NS}}}programming-language' -DOAP_LOGO = f'.//{{{SCHEMA_NS}}}logo' -DOAP_IMPLEMENTS = f'.//{{{DOAP_NS}}}implements' -DOAP_SUPPORTED_XEP = f'.//{{{XMPP_NS}}}SupportedXep' -DOAP_XEP_NUMBER = f'.//{{{XMPP_NS}}}xep' -DOAP_XEP_VERSION = f'.//{{{XMPP_NS}}}version' -DOAP_XEP_STATUS = f'.//{{{XMPP_NS}}}status' - -RFC_REGEX = r'rfc\d{1,4}' -XEP_REGEX = r'xep-\d{1,4}' - -XML_DECLARATION = '' -XMPP_XSL = '' - -MD_FRONTMATTER = '''--- +SOFTWARE_PATH = Path("content/software") +DATA_PATH = Path("data") +DOWNLOAD_PATH = Path("downloads") +STATIC_PATH = Path("static") +STATIC_DOAP_PATH = STATIC_PATH / "doap" +LOGOS_PATH = STATIC_PATH / "images" / "packages" + +DOAP_NS = "http://usefulinc.com/ns/doap#" +XMPP_NS = "https://linkmauve.fr/ns/xmpp-doap#" +SCHEMA_NS = "https://schema.org/" +RDF_RESOURCE = "{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource" +DOAP_NAME = f".//{{{DOAP_NS}}}name" +DOAP_SHORTDESC = f".//{{{DOAP_NS}}}shortdesc" +DOAP_HOMEPAGE = f".//{{{DOAP_NS}}}homepage" +DOAP_OS = f".//{{{DOAP_NS}}}os" +DOAP_PROGRAMMING_LANGUAGE = f".//{{{DOAP_NS}}}programming-language" +DOAP_LOGO = f".//{{{SCHEMA_NS}}}logo" +DOAP_IMPLEMENTS = f".//{{{DOAP_NS}}}implements" +DOAP_SUPPORTED_XEP = f".//{{{XMPP_NS}}}SupportedXep" +DOAP_XEP_NUMBER = f".//{{{XMPP_NS}}}xep" +DOAP_XEP_VERSION = f".//{{{XMPP_NS}}}version" +DOAP_XEP_STATUS = f".//{{{XMPP_NS}}}status" + +RFC_REGEX = r"rfc\d{1,4}" +XEP_REGEX = r"xep-\d{1,4}" + +XML_DECLARATION = '' +XMPP_XSL = '' + +MD_FRONTMATTER = """--- title: "%(title)s" date: %(date)s layout: packages @@ -62,68 +61,66 @@ --- {{< package-details name_slug="%(name_slug)s" package_type="%(type)s" >}} -''' +""" SOFTWARE_CATEGORIES: list[str] = [ - 'client', - 'component', - 'library', - 'server', - 'tool', + "client", + "component", + "library", + "server", + "tool", ] PLATFORMS: list[str] = [ - 'Android', - 'iOS', - 'Browser', - 'Windows', - 'macOS', - 'Linux', + "Android", + "iOS", + "Browser", + "Windows", + "macOS", + "Linux", ] -DoapInfoT = dict[ - str, str | list[str] | list[dict[str, str]] | None] | None +DoapInfoT = dict[str, str | list[str] | list[dict[str, str]] | None] | None def parse_doap_infos(doap_file: str) -> DoapInfoT: - ''' + """ Parse DOAP file and return infos - ''' + """ try: - doap = parse( - DOWNLOAD_PATH / f'doap_files/{doap_file}.doap') + doap = parse(DOWNLOAD_PATH / f"doap_files/{doap_file}.doap") except (FileNotFoundError, ParseError) as err: - print('Error while trying to parse DOAP file:', doap_file, err) + print("Error while trying to parse DOAP file:", doap_file, err) return None info: dict[str, str | list[str] | list[dict[str, str]] | None] = {} - info['name'] = None + info["name"] = None doap_name = doap.find(DOAP_NAME) if doap_name is not None: - info['name'] = doap_name.text + info["name"] = doap_name.text - info['homepage'] = None + info["homepage"] = None doap_homepage = doap.find(DOAP_HOMEPAGE) if doap_homepage is not None: - info['homepage'] = doap_homepage.attrib.get(RDF_RESOURCE) + info["homepage"] = doap_homepage.attrib.get(RDF_RESOURCE) - info['shortdesc'] = None + info["shortdesc"] = None doap_shortdesc = doap.find(DOAP_SHORTDESC) if doap_shortdesc is not None: - info['shortdesc'] = doap_shortdesc.text + info["shortdesc"] = doap_shortdesc.text - info['platforms'] = [] + info["platforms"] = [] for entry in doap.findall(DOAP_OS): - info['platforms'].append(entry.text) + info["platforms"].append(entry.text) - info['programming_lang'] = [] + info["programming_lang"] = [] for entry in doap.findall(DOAP_PROGRAMMING_LANGUAGE): - info['programming_lang'].append(entry.text) + info["programming_lang"].append(entry.text) - info['logo'] = None + info["logo"] = None doap_logo = doap.find(DOAP_LOGO) if doap_logo is not None: - info['logo'] = doap_logo.attrib.get(RDF_RESOURCE) + info["logo"] = doap_logo.attrib.get(RDF_RESOURCE) rfcs: list[str] = [] xeps: list[dict[str, str]] = [] @@ -139,7 +136,7 @@ def parse_doap_infos(doap_file: str) -> DoapInfoT: number = supported_xep.find(DOAP_XEP_NUMBER) if number is not None: number = number.attrib.get(RDF_RESOURCE) - match = re.search(XEP_REGEX, number or '') + match = re.search(XEP_REGEX, number or "") if match: number = match.group()[4:] @@ -151,31 +148,33 @@ def parse_doap_infos(doap_file: str) -> DoapInfoT: if status is not None: status = status.text - xeps.append({ - 'number': number, - 'version': version, - 'status': status, - }) + xeps.append( + { + "number": number, + "version": version, + "status": status, + } + ) - info['rfcs'] = rfcs - info['xeps'] = xeps + info["rfcs"] = rfcs + info["xeps"] = xeps return info def check_image_file(file_path: Path, extension: str) -> bool: - ''' + """ Check if file size is greater than 300 KiB and if so, resize image Returns success - ''' - if extension == 'svg': + """ + if extension == "svg": # No need to resize SVG files return True try: file_size = os.path.getsize(file_path) except OSError as error: - print('An error occurred while trying to open logo:', error) + print("An error occurred while trying to open logo:", error) return False if file_size <= 300000: @@ -187,65 +186,63 @@ def check_image_file(file_path: Path, extension: str) -> bool: width, height = img.size new_width = 400 new_height = int(new_width * height / width) - img = img.resize( - (new_width, new_height), Resampling.LANCZOS) + img = img.resize((new_width, new_height), Resampling.LANCZOS) img.save(file_path) - print(f' Logo at {file_path} ' - f'(file size: {file_size / (1<<10):,.0f} KB) ' - f'too big, had to be resized') + print( + f" Logo at {file_path} " + f"(file size: {file_size / (1<<10):,.0f} KB) " + f"too big, had to be resized" + ) except (ValueError, OSError, UnidentifiedImageError) as error: - print('An error occurred while trying to resize logo:', error) + print("An error occurred while trying to resize logo:", error) return False return True def process_logo(package_name: str, uri: str) -> str | None: - ''' + """ Download package logo and return logo URI - ''' + """ image_url = urlparse(uri) _, extension = os.path.splitext(image_url.path) - file_name = f'{package_name}{extension}' - success = download_file( - uri, - Path(file_name)) + file_name = f"{package_name}{extension}" + success = download_file(uri, Path(file_name)) if not success: return None - success = check_image_file( - DOWNLOAD_PATH / file_name, extension[1:].lower()) + success = check_image_file(DOWNLOAD_PATH / file_name, extension[1:].lower()) if not success: return None - logo_uri = f'/images/packages/{package_name}{extension}' - shutil.copyfile( - DOWNLOAD_PATH / file_name, - Path(LOGOS_PATH / file_name)) + logo_uri = f"/images/packages/{package_name}{extension}" + shutil.copyfile(DOWNLOAD_PATH / file_name, Path(LOGOS_PATH / file_name)) return logo_uri def prepare_package_data() -> None: - ''' + """ Download and prepare package data (software.json) for rendering with Hugo - ''' + """ for category in SOFTWARE_CATEGORIES: - if category == 'library': - category = 'libraries' + if category == "library": + category = "libraries" else: - category = f'{category}s' + category = f"{category}s" - shutil.copy(SOFTWARE_PATH / '_index.md', - DOWNLOAD_PATH / 'software_index.md') - shutil.copy(SOFTWARE_PATH / 'software-comparison.md', - DOWNLOAD_PATH / 'software-comparison.md') + shutil.copy(SOFTWARE_PATH / "_index.md", DOWNLOAD_PATH / "software_index.md") + shutil.copy( + SOFTWARE_PATH / "software-comparison.md", + DOWNLOAD_PATH / "software-comparison.md", + ) initialize_directory(SOFTWARE_PATH) - shutil.copy(DOWNLOAD_PATH / 'software_index.md', - SOFTWARE_PATH / '_index.md') - shutil.copy(DOWNLOAD_PATH / 'software-comparison.md', - SOFTWARE_PATH / 'software-comparison.md') + shutil.copy(DOWNLOAD_PATH / "software_index.md", SOFTWARE_PATH / "_index.md") + shutil.copy( + DOWNLOAD_PATH / "software-comparison.md", + SOFTWARE_PATH / "software-comparison.md", + ) - with open(DATA_PATH / 'software.json', 'rb') as json_file: + with open(DATA_PATH / "software.json", "rb") as json_file: xsf_package_list = json.load(json_file) package_infos: dict[str, Any] = {} @@ -253,155 +250,149 @@ def prepare_package_data() -> None: number_of_doap_packages = 0 for package in xsf_package_list: - if package['doap'] is None: - print(f'{Fore.YELLOW}DOAP n/a' - f'{Style.RESET_ALL} ', - package['name']) + if package["doap"] is None: + print( + f"{Fore.YELLOW}DOAP n/a" f"{Style.RESET_ALL} ", package["name"] + ) continue # DOAP is available number_of_doap_packages += 1 - package_name_slug = slugify( - package['name'], - replacements=[['+', 'plus']]) + package_name_slug = slugify(package["name"], replacements=[["+", "plus"]]) - doap_url = package['doap'] - if doap_url.startswith('/hosted-doap'): + doap_url = package["doap"] + if doap_url.startswith("/hosted-doap"): # DOAP file is hosted at xmpp.org - print(f'{Fore.LIGHTCYAN_EX}DOAP by xmpp.org' - f'{Style.RESET_ALL} ', - package['name']) + print( + f"{Fore.LIGHTCYAN_EX}DOAP by xmpp.org" f"{Style.RESET_ALL} ", + package["name"], + ) shutil.copyfile( - f'{STATIC_PATH}{doap_url}', - Path(f'{DOWNLOAD_PATH}/doap_files/{package_name_slug}.doap')) + f"{STATIC_PATH}{doap_url}", + Path(f"{DOWNLOAD_PATH}/doap_files/{package_name_slug}.doap"), + ) else: - print(f'{Fore.LIGHTBLUE_EX}DOAP by vendor' - f'{Style.RESET_ALL} ', - package['name']) - download_file( - package['doap'], - Path(f'doap_files/{package_name_slug}.doap')) + print( + f"{Fore.LIGHTBLUE_EX}DOAP by vendor" f"{Style.RESET_ALL} ", + package["name"], + ) + download_file(package["doap"], Path(f"doap_files/{package_name_slug}.doap")) parsed_package_infos = parse_doap_infos(package_name_slug) if parsed_package_infos is None: continue logo_uri = None - logo = parsed_package_infos['logo'] + logo = parsed_package_infos["logo"] if logo is not None and isinstance(logo, str): - logo_uri = process_logo( - package_name_slug, logo) - - package_infos[package['name']] = { - 'categories': package['categories'], - 'name_slug': package_name_slug, - 'homepage': parsed_package_infos['homepage'], - 'logo': logo_uri, - 'shortdesc': parsed_package_infos['shortdesc'], - 'platforms': parsed_package_infos['platforms'], - 'programming_lang': parsed_package_infos['programming_lang'], - 'rfcs': parsed_package_infos['rfcs'], - 'xeps': parsed_package_infos['xeps'], + logo_uri = process_logo(package_name_slug, logo) + + package_infos[package["name"]] = { + "categories": package["categories"], + "name_slug": package_name_slug, + "homepage": parsed_package_infos["homepage"], + "logo": logo_uri, + "shortdesc": parsed_package_infos["shortdesc"], + "platforms": parsed_package_infos["platforms"], + "programming_lang": parsed_package_infos["programming_lang"], + "rfcs": parsed_package_infos["rfcs"], + "xeps": parsed_package_infos["xeps"], } - for category in package['categories']: - if category == 'library': - category = 'libraries' + for category in package["categories"]: + if category == "library": + category = "libraries" else: - category = f'{category}s' - create_package_page(category, package_name_slug, package['name']) - - print(f'Number of packages:\n' - f'total: {len(xsf_package_list)} ' - f'(with DOAP: {number_of_doap_packages}), ' - f'\n{42 * "="}') - with open(DATA_PATH / 'software_list_doap.json', - 'w', - encoding='utf-8') as package_data_file: + category = f"{category}s" + create_package_page(category, package_name_slug, package["name"]) + + print( + f"Number of packages:\n" + f"total: {len(xsf_package_list)} " + f"(with DOAP: {number_of_doap_packages}), " + f'\n{42 * "="}' + ) + with open( + DATA_PATH / "software_list_doap.json", "w", encoding="utf-8" + ) as package_data_file: json.dump(package_infos, package_data_file, indent=4) def add_doap_data_to_xeplist() -> None: - ''' + """ Adds data from DOAP files (implementations) to xeplist.json - ''' - with open(DATA_PATH / 'software_list_doap.json', - encoding='utf-8') as software_list: + """ + with open(DATA_PATH / "software_list_doap.json", encoding="utf-8") as software_list: software_data = json.load(software_list) - with open(DATA_PATH / 'xeplist.json', encoding='utf-8') as xep_list: + with open(DATA_PATH / "xeplist.json", encoding="utf-8") as xep_list: xep_data = json.load(xep_list) for xep in xep_data: - xep['implementations'] = [] + xep["implementations"] = [] for name, package_data in software_data.items(): - if not package_data['xeps']: + if not package_data["xeps"]: continue - for supported_xep in package_data['xeps']: - if supported_xep['number'] == f'{xep["number"]:04d}': - xep['implementations'].append({ - 'package_name': name, - 'package_name_slug': package_data['name_slug'], - 'package_categories': package_data['categories'], - 'implemented_version': supported_xep['version'], - 'implementation_status': supported_xep['status'] - }) + for supported_xep in package_data["xeps"]: + if supported_xep["number"] == f'{xep["number"]:04d}': + xep["implementations"].append( + { + "package_name": name, + "package_name_slug": package_data["name_slug"], + "package_categories": package_data["categories"], + "implemented_version": supported_xep["version"], + "implementation_status": supported_xep["status"], + } + ) break - with open(DATA_PATH / 'xeplist.json', - 'w', - encoding='utf-8') as xep_list: + with open(DATA_PATH / "xeplist.json", "w", encoding="utf-8") as xep_list: json.dump(xep_data, xep_list, indent=4) + def create_package_page(package_type: str, name_slug: str, name: str) -> None: - ''' + """ Create an .md page for package, containing a shortcode for displaying package details - ''' + """ today = date.today() - date_formatted = today.strftime('%Y-%m-%d') - with open(SOFTWARE_PATH / f'{name_slug}.md', - 'w', - encoding='utf8') as md_file: + date_formatted = today.strftime("%Y-%m-%d") + with open(SOFTWARE_PATH / f"{name_slug}.md", "w", encoding="utf8") as md_file: md_file.write( - MD_FRONTMATTER % { - 'title': f'XMPP {package_type.capitalize()}: {name}', - 'date': date_formatted, - 'type': package_type, - 'name_slug': name_slug, + MD_FRONTMATTER + % { + "title": f"XMPP {package_type.capitalize()}: {name}", + "date": date_formatted, + "type": package_type, + "name_slug": name_slug, } ) def prepare_doap_files() -> None: - ''' + """ Copy DOAP files to /static/doap/ and replace the xml-stylesheet with our stylesheet (or add it, if there is none) - ''' - for entry in os.scandir(DOWNLOAD_PATH / 'doap_files'): - shutil.copy(DOWNLOAD_PATH / 'doap_files' / entry.name, - STATIC_DOAP_PATH / entry.name) + """ + for entry in os.scandir(DOWNLOAD_PATH / "doap_files"): + shutil.copy( + DOWNLOAD_PATH / "doap_files" / entry.name, STATIC_DOAP_PATH / entry.name + ) - for entry in os.scandir(STATIC_PATH / 'hosted-doap'): - shutil.copy(STATIC_PATH / 'hosted-doap' / entry.name, - STATIC_DOAP_PATH / entry.name) + for entry in os.scandir(STATIC_PATH / "hosted-doap"): + shutil.copy( + STATIC_PATH / "hosted-doap" / entry.name, STATIC_DOAP_PATH / entry.name + ) - xml_declaration_pattern = r'<\?xml version.+?\?>' - stylesheet_pattern = r'<\?xml-stylesheet.+?\?>' + xml_declaration_pattern = r"<\?xml version.+?\?>" + stylesheet_pattern = r"<\?xml-stylesheet.+?\?>" for entry in os.scandir(STATIC_DOAP_PATH): - if not entry.name.endswith('.doap'): + if not entry.name.endswith(".doap"): continue - with open(STATIC_DOAP_PATH / entry.name, - 'r+', - encoding='utf-8') as doap_file: + with open(STATIC_DOAP_PATH / entry.name, "r+", encoding="utf-8") as doap_file: content = doap_file.read() - result = re.sub( - stylesheet_pattern, - XMPP_XSL, - content, - 0, - re.MULTILINE) + result = re.sub(stylesheet_pattern, XMPP_XSL, content, 0, re.MULTILINE) if result != content: # Replaced custom stylesheet with our stylesheet doap_file.truncate(0) @@ -412,26 +403,27 @@ def prepare_doap_files() -> None: # No custom stylesheet found result = re.sub( xml_declaration_pattern, - f'{XML_DECLARATION}\n{XMPP_XSL}', + f"{XML_DECLARATION}\n{XMPP_XSL}", content, 0, - re.MULTILINE) + re.MULTILINE, + ) if result != content: # Added our stylesheet doap_file.truncate(0) doap_file.seek(0) doap_file.write(result) else: - print('WARNING: Could not alter XML header of', entry.name) + print("WARNING: Could not alter XML header of", entry.name) # Remove content entirely, since we can't # control what would be rendered doap_file.truncate(0) -if __name__ == '__main__': +if __name__ == "__main__": initialize_directory(DOWNLOAD_PATH) initialize_directory(LOGOS_PATH) - Path(DOWNLOAD_PATH / 'doap_files').mkdir(parents=True) + Path(DOWNLOAD_PATH / "doap_files").mkdir(parents=True) prepare_package_data() add_doap_data_to_xeplist() diff --git a/tools/prepare_xep_list.py b/tools/prepare_xep_list.py index 9cb6013d1..e740c8053 100755 --- a/tools/prepare_xep_list.py +++ b/tools/prepare_xep_list.py @@ -1,6 +1,6 @@ -''' +""" This file is used to download the XEP list and convert it to JSON -''' +""" from typing import Any import json @@ -11,54 +11,53 @@ from defusedxml.ElementTree import fromstring from defusedxml.ElementTree import ParseError -XEP_LIST_URL = 'https://xmpp.org/extensions/xeplist.xml' +XEP_LIST_URL = "https://xmpp.org/extensions/xeplist.xml" def build_xep_list() -> None: - ''' + """ Download and parse xeplist.xml and build xeplist.json - ''' + """ try: xeplist_request = requests.get(XEP_LIST_URL, timeout=5) except requests.exceptions.RequestException as err: - sys.exit(f'Error while requesting xeplist.xml ({err})') + sys.exit(f"Error while requesting xeplist.xml ({err})") if not 200 >= xeplist_request.status_code < 400: - sys.exit(f'Error while downloading xeplist.xml ' - f'({xeplist_request.status_code})') + sys.exit( + f"Error while downloading xeplist.xml " f"({xeplist_request.status_code})" + ) try: root = fromstring(xeplist_request.content) except ParseError: - sys.exit('Error while parsing xeplist.xml') + sys.exit("Error while parsing xeplist.xml") def fix_status(status: str) -> str: - if status == 'Draft': - return 'Stable' + if status == "Draft": + return "Stable" return status xeps: list[dict[str, Any]] = [] - for xep in root.findall('xep'): - if xep.get('accepted') == 'true': + for xep in root.findall("xep"): + if xep.get("accepted") == "true": xeps.append( { - 'title': xep.find('title').text, - 'status': fix_status(xep.find('status').text), - 'number': int(xep.find('number').text), - 'last_updated': xep.find('last-revision').find('date').text, - 'type': xep.find('type').text, + "title": xep.find("title").text, + "status": fix_status(xep.find("status").text), + "number": int(xep.find("number").text), + "last_updated": xep.find("last-revision").find("date").text, + "type": xep.find("type").text, } ) - xeps_sorted = sorted(xeps, key=lambda xep: xep['number']) + xeps_sorted = sorted(xeps, key=lambda xep: xep["number"]) base_path = os.path.dirname(os.path.abspath(sys.argv[0])) - with open(f'{base_path}/../data/xeplist.json', - 'w', - encoding='utf-8') as json_file: + with open(f"{base_path}/../data/xeplist.json", "w", encoding="utf-8") as json_file: json.dump(xeps_sorted, json_file, indent=4) - print('XEP List prepared successfully') + print("XEP List prepared successfully") -if __name__ == '__main__': +if __name__ == "__main__": build_xep_list() diff --git a/tools/update_entry.py b/tools/update_entry.py index 89918fe87..f4531ac8d 100755 --- a/tools/update_entry.py +++ b/tools/update_entry.py @@ -1,106 +1,101 @@ #!/usr/bin/env python3 -''' +""" Tool for maintaining software list entries -''' +""" from typing import Any import argparse -import difflib import copy +import difflib import json import sys def json_as_lines(data: Any) -> list[str]: - '''Returns a json file as newline terminated strings''' + """Returns a json file as newline terminated strings""" return [ - line+'\n' - for line in json.dumps( - data, - indent=4, - sort_keys=True - ).split('\n') + line + "\n" for line in json.dumps(data, indent=4, sort_keys=True).split("\n") ] def main(): - ''' + """ Main logic of update_entry tool - ''' + """ parser = argparse.ArgumentParser( - description='Modify a software entry in the software list.' + description="Modify a software entry in the software list." ) parser.add_argument( - 'name', - nargs='?', + "name", + nargs="?", default=None, - metavar='NAME', - help='Current name of the project', + metavar="NAME", + help="Current name of the project", ) parser.add_argument( - '--rename', - dest='new_name', - metavar='NAME', + "--rename", + dest="new_name", + metavar="NAME", default=None, - help='Rename the project', + help="Rename the project", ) parser.add_argument( - '--set-url', - dest='new_url', - metavar='URL', + "--set-url", + dest="new_url", + metavar="URL", default=None, - help='Change the URL of the project', + help="Change the URL of the project", ) parser.add_argument( - '--set-doap', - dest='new_doap', - metavar='URL', + "--set-doap", + dest="new_doap", + metavar="URL", default=None, - help='Change the URL of the project DOAP file', + help="Change the URL of the project DOAP file", ) parser.add_argument( - '--set-platforms', - dest='new_platforms', - metavar='PLATFORM', + "--set-platforms", + dest="new_platforms", + metavar="PLATFORM", default=None, - nargs='+', - help='Change the contents of the last column', + nargs="+", + help="Change the contents of the last column", ) parser.add_argument( - '--no-ask', - dest='ask', + "--no-ask", + dest="ask", default=True, - action='store_false', - help='Do not ask for confirmation before applying changes. ' + action="store_false", + help="Do not ask for confirmation before applying changes. ", ) args = parser.parse_args() - with open('../data/software.json', encoding='utf-8') as software_file: + with open("../data/software.json", encoding="utf-8") as software_file: filename = software_file.name data = json.load(software_file) - name_map = {item['name']: item for item in data} + name_map = {item["name"]: item for item in data} try: item = name_map[args.name] except KeyError: if args.name is not None: - print(f'Error: no such project: {args.name!r}', - file=sys.stderr) - print('Hint: the following projects exist:') + print(f"Error: no such project: {args.name!r}", file=sys.stderr) + print("Hint: the following projects exist:") print( - ' ', '\n '.join(sorted(name_map.keys())), - sep='', + " ", + "\n ".join(sorted(name_map.keys())), + sep="", file=sys.stderr, ) - print('Hint: capitalisation matters!', file=sys.stderr) + print("Hint: capitalisation matters!", file=sys.stderr) sys.exit(1) orig = copy.deepcopy(item) @@ -111,65 +106,63 @@ def main(): except KeyError: pass else: - print(f'Error: new name {args.new_name!r} ' - f'already in use by a project', - file=sys.stderr, + print( + f"Error: new name {args.new_name!r} " f"already in use by a project", + file=sys.stderr, ) - print('Hint: the existing project looks like this:', - file=sys.stderr) + print("Hint: the existing project looks like this:", file=sys.stderr) json.dump(existing, sys.stderr, indent=4, sort_keys=True) print(file=sys.stderr) sys.exit(2) - item['name'] = args.new_name + item["name"] = args.new_name if args.new_url is not None: - item['url'] = args.new_url or None + item["url"] = args.new_url or None if args.new_doap is not None: - item['doap'] = args.new_doap or None + item["doap"] = args.new_doap or None if args.new_platforms is not None: - item['platforms'] = list(set( - platform.strip() - for platform in args.new_platforms - )) + item["platforms"] = list( + set(platform.strip() for platform in args.new_platforms) + ) - item['platforms'].sort(key=lambda x: x.casefold()) + item["platforms"].sort(key=lambda x: x.casefold()) if args.ask: old_entry = json_as_lines(orig) new_entry = json_as_lines(item) - print('difference between old and new:') + print("difference between old and new:") sys.stdout.writelines( difflib.unified_diff( old_entry, new_entry, - fromfile='before', - tofile='after', + fromfile="before", + tofile="after", n=1000, ) ) - prompt = 'is this okay? [y/n]' + prompt = "is this okay? [y/n]" try: chosen = input(prompt) - while chosen not in 'yn': - print('please choose y or n!') + while chosen not in "yn": + print("please choose y or n!") chosen = input(prompt) except (EOFError, KeyboardInterrupt): - chosen = 'n' + chosen = "n" - if chosen != 'y': - print('aborting per user request') + if chosen != "y": + print("aborting per user request") sys.exit(3) - data.sort(key=lambda x: x['name'].casefold()) + data.sort(key=lambda x: x["name"].casefold()) - with open(filename, 'w', encoding='utf-8') as software_file: + with open(filename, "w", encoding="utf-8") as software_file: json.dump(data, software_file, indent=4, sort_keys=True) - software_file.write('\n') + software_file.write("\n") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/util.py b/tools/util.py index 3c5cc8448..a34626c61 100644 --- a/tools/util.py +++ b/tools/util.py @@ -1,6 +1,6 @@ -''' +""" Utilities for files/folders/downloads management -''' +""" import os import shutil @@ -8,13 +8,13 @@ import requests -DOWNLOAD_PATH = Path('downloads') +DOWNLOAD_PATH = Path("downloads") def initialize_directory(path: Path) -> None: - ''' + """ Remove path (if it exists) and containing files, then recreate path - ''' + """ if path.exists() and path.is_dir(): shutil.rmtree(path) os.mkdir(path) @@ -23,21 +23,21 @@ def initialize_directory(path: Path) -> None: def download_file(url: str, path: Path) -> bool: - ''' + """ Downloads file from url and stores it in /downloads/path returns success - ''' + """ try: file_request = requests.get(url, stream=True, timeout=5) except requests.exceptions.RequestException as err: - print('Error while requesting file', url, err) + print("Error while requesting file", url, err) return False if not 200 >= file_request.status_code < 400: - print('Error while trying to download from', url) + print("Error while trying to download from", url) return False - with open(DOWNLOAD_PATH / path, 'wb') as data_file: + with open(DOWNLOAD_PATH / path, "wb") as data_file: max_size = 1024 * 1024 * 10 # 10 MiB size = 0 for chunk in file_request.iter_content(chunk_size=8192): @@ -45,6 +45,6 @@ def download_file(url: str, path: Path) -> bool: size += len(chunk) if size > max_size: file_request.close() - print('File size exceeds 10 MiB:', url, path) + print("File size exceeds 10 MiB:", url, path) return False return True