diff --git a/netkan/netkan/cli/__init__.py b/netkan/netkan/cli/__init__.py index d973625..12b37c4 100644 --- a/netkan/netkan/cli/__init__.py +++ b/netkan/netkan/cli/__init__.py @@ -17,6 +17,7 @@ download_counter, ticket_closer, mirror_purge_epochs, + analyze_mod, ) @@ -39,3 +40,4 @@ def netkan() -> None: netkan.add_command(spacedock_adder) netkan.add_command(mirrorer) netkan.add_command(mirror_purge_epochs) +netkan.add_command(analyze_mod) diff --git a/netkan/netkan/cli/common.py b/netkan/netkan/cli/common.py index 1de3bac..741c62d 100644 --- a/netkan/netkan/cli/common.py +++ b/netkan/netkan/cli/common.py @@ -56,7 +56,7 @@ def ctx_callback(ctx: click.Context, param: click.Parameter, click.option('--ia-collections', envvar='IA_COLLECTIONS', expose_value=False, help='game=Collection, for mirroring mods in on Internet Archive', multiple=True, callback=ctx_callback), - click.option('--game-id', envvar='GAME_ID', help='Game ID for this task', + click.option('--game-id', default='KSP', envvar='GAME_ID', help='Game ID for this task', expose_value=False, callback=ctx_callback) ] @@ -219,7 +219,8 @@ def ssh_key(self) -> Optional[str]: @ssh_key.setter def ssh_key(self, value: str) -> None: - init_ssh(value, Path(Path.home(), '.ssh')) + if value: + init_ssh(value, Path(Path.home(), '.ssh')) self._ssh_key = value def game(self, game: str) -> Game: diff --git a/netkan/netkan/cli/services.py b/netkan/netkan/cli/services.py index 22bfcab..3749965 100644 --- a/netkan/netkan/cli/services.py +++ b/netkan/netkan/cli/services.py @@ -10,14 +10,18 @@ from ..mirrorer import Mirrorer -@click.command() +@click.command(short_help='The Indexer service') @common_options @pass_state def indexer(common: SharedArgs) -> None: + """ + Retrieves inflated metadata from the Inflator's output queue + and updates the metadata repo as needed + """ IndexerQueueHandler(common).run() -@click.command() +@click.command(short_help='The Scheduler service') @click.option( '--max-queued', default=20, envvar='MAX_QUEUED', help='SQS Queue to send netkan metadata for scheduling', @@ -44,6 +48,10 @@ def scheduler( min_cpu: int, min_io: int ) -> None: + """ + Reads netkans from a NetKAN repo and submits them to the + Inflator's input queue + """ for game_id in common.game_ids: game = common.game(game_id) sched = NetkanScheduler( @@ -56,10 +64,14 @@ def scheduler( logging.info("NetKANs submitted to %s", game.inflation_queue) -@click.command() +@click.command(short_help='The Mirrorer service') @common_options @pass_state def mirrorer(common: SharedArgs) -> None: + """ + Uploads redistributable mods to archive.org as they + are added to the meta repo + """ # We need at least 50 mods for a collection for ksp2, keeping # to just ksp for now Mirrorer( @@ -68,8 +80,12 @@ def mirrorer(common: SharedArgs) -> None: ).process_queue(common.queue, common.timeout) -@click.command() +@click.command(short_help='The SpaceDockAdder service') @common_options @pass_state def spacedock_adder(common: SharedArgs) -> None: + """ + Submits pull requests to a NetKAN repo when users + click the Add to CKAN checkbox on SpaceDock + """ SpaceDockAdderQueueHandler(common).run() diff --git a/netkan/netkan/cli/utilities.py b/netkan/netkan/cli/utilities.py index 9f7dbde..8a7b31b 100644 --- a/netkan/netkan/cli/utilities.py +++ b/netkan/netkan/cli/utilities.py @@ -2,12 +2,14 @@ import json import logging import time +import io from pathlib import Path from typing import Tuple import boto3 import click +from ruamel.yaml import YAML from .common import common_options, pass_state, SharedArgs @@ -16,9 +18,10 @@ from ..ticket_closer import TicketCloser from ..auto_freezer import AutoFreezer from ..mirrorer import Mirrorer +from ..mod_analyzer import ModAnalyzer -@click.command() +@click.command(short_help='Submit or update a PR freezing idle mods') @click.option( '--days-limit', default=1000, help='Number of days to wait before freezing a mod as idle', @@ -30,6 +33,10 @@ @common_options @pass_state def auto_freezer(common: SharedArgs, days_limit: int, days_till_ignore: int) -> None: + """ + Scan the given NetKAN repo for mods that haven't updated + in a given number of days and submit or update a pull request to freeze them + """ for game_id in common.game_ids: afr = AutoFreezer( common.game(game_id).netkan_repo, @@ -40,10 +47,14 @@ def auto_freezer(common: SharedArgs, days_limit: int, days_till_ignore: int) -> afr.mark_frozen_mods() -@click.command() +@click.command(short_help='Update download counts in a given repo') @common_options @pass_state def download_counter(common: SharedArgs) -> None: + """ + Count downloads for all the mods in the given repo + and update the download_counts.json file + """ for game_id in common.game_ids: logging.info('Starting Download Count Calculation (%s)...', game_id) DownloadCounter( @@ -53,7 +64,28 @@ def download_counter(common: SharedArgs) -> None: logging.info('Download Counter completed! (%s)', game_id) -@click.command() +@click.command(short_help='Autogenerate a mod\'s .netkan properties') +@click.argument('ident', required=True) +@click.argument('download_url', required=True) +@common_options +@pass_state +def analyze_mod(common: SharedArgs, ident: str, download_url: str) -> None: + """ + Download a mod with identifier IDENT from DOWNLOAD_URL + and guess its netkan properties + """ + sio = io.StringIO() + yaml = YAML() + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.dump(ModAnalyzer(ident, download_url, common.game(common.game_id or 'KSP')) + .get_netkan_properties(), + sio) + click.echo('spec_version: v1.18') + click.echo(f'identifier: {ident}') + click.echo(sio.getvalue()) + + +@click.command(short_help='Update the JSON status file on s3') @click.option( '--status-bucket', envvar='STATUS_BUCKET', required=True, help='Bucket to Dump status.json', @@ -68,6 +100,10 @@ def download_counter(common: SharedArgs) -> None: help='Dump status to S3 every `interval` seconds', ) def export_status_s3(status_bucket: str, status_keys: Tuple[str, ...], interval: int) -> None: + """ + Retrieves the mod timestamps and warnings/errors from the status database + and saves them where the status page can see them in JSON format + """ frequency = f'every {interval} seconds' if interval else 'once' while True: for status in status_keys: @@ -85,14 +121,22 @@ def export_status_s3(status_bucket: str, status_keys: Tuple[str, ...], interval: logging.info('Done.') -@click.command() +@click.command(short_help='Print the mod status JSON') def dump_status() -> None: + """ + Retrieves the mod timestamps and warnings/errors from the status database + and prints them in JSON format + """ click.echo(json.dumps(ModStatus.export_all_mods())) -@click.command() +@click.command(short_help='Normalize status database entries') @click.argument('filename') def restore_status(filename: str) -> None: + """ + Normalize the status info for all mods in database and + commit them in groups of 5 per second + """ click.echo( 'To keep within free tier rate limits, this could take some time' ) @@ -100,21 +144,30 @@ def restore_status(filename: str) -> None: click.echo('Done!') -@click.command() +@click.command(short_help='Set status timestamps based on git repo') @common_options @pass_state def recover_status_timestamps(common: SharedArgs) -> None: + """ + If a mod's status entry is missing a last indexed timestamp, + set it to the timstamp from the most recent commit in the meta repo + """ ModStatus.recover_timestamps(common.game(common.game_id).ckanmeta_repo) -@click.command() +@click.command(short_help='Update and restart one of the bot\'s containers') @click.option( - '--cluster', help='ECS Cluster running the service', + '--cluster', required=True, + help='ECS Cluster running the service', ) @click.option( - '--service-name', help='Name of ECS Service to restart', + '--service-name', required=True, + help='Name of ECS Service to restart', ) def redeploy_service(cluster: str, service_name: str) -> None: + """ + Update and restart the given service on the given container + """ click.secho( f'Forcing redeployment of {cluster}:{service_name}', fg='green' @@ -145,7 +198,7 @@ def redeploy_service(cluster: str, service_name: str) -> None: click.secho('Service Redeployed', fg='green') -@click.command() +@click.command(short_help='Close inactive issues on GitHub') @click.option( '--days-limit', default=7, help='Number of days to wait for OP to reply', @@ -153,10 +206,15 @@ def redeploy_service(cluster: str, service_name: str) -> None: @common_options @pass_state def ticket_closer(common: SharedArgs, days_limit: int) -> None: + """ + Close issues with the Support tag where the most recent + reply isn't from the original author and that have been + inactive for the given number of days + """ TicketCloser(common.token, common.user).close_tickets(days_limit) -@click.command() +@click.command(short_help='Purge old downloads from the bot\'s download cache') @click.option( '--days', help='Purge items older than X from cache', ) @@ -166,6 +224,10 @@ def ticket_closer(common: SharedArgs, days_limit: int) -> None: help='Absolute path to the mod download cache' ) def clean_cache(days: int, cache: str) -> None: + """ + Purge downloads from the bot's download cach that are + older than the given number of days + """ older_than = ( datetime.datetime.now() - datetime.timedelta(days=int(days)) ).timestamp() @@ -176,15 +238,19 @@ def clean_cache(days: int, cache: str) -> None: item.unlink() -@click.command() +@click.command(short_help='Remove epoch strings from archive.org entries') @click.option( - '--dry-run', - help='', - default=False, + '--dry-run', default=False, + help='True to report what would be done instead of doing it' ) @common_options @pass_state def mirror_purge_epochs(common: SharedArgs, dry_run: bool) -> None: + """ + Loop over mods mirrored to archive.org + and remove their version epoch prefixes. + This has never actually been used. + """ Mirrorer( common.game(common.game_id).ckanmeta_repo, common.ia_access, common.ia_secret, common.game(common.game_id).ia_collection diff --git a/netkan/netkan/notifications.py b/netkan/netkan/notifications.py index d8221d8..a0c0af2 100644 --- a/netkan/netkan/notifications.py +++ b/netkan/netkan/notifications.py @@ -41,7 +41,7 @@ def setup_log_handler(debug: bool = False) -> bool: logging.basicConfig( format='[%(asctime)s] [%(levelname)-8s] %(message)s', level=level ) - logging.info('Logging started for \'%s\' at log level %s', sys.argv[1], level) + logging.debug('Logging started for \'%s\' at log level %s', sys.argv[1], level) # Set up Discord logger so we can see errors discord_webhook_id = os.environ.get('DISCORD_WEBHOOK_ID') diff --git a/netkan/netkan/utils.py b/netkan/netkan/utils.py index 75f1240..0c8b3fe 100644 --- a/netkan/netkan/utils.py +++ b/netkan/netkan/utils.py @@ -19,7 +19,7 @@ def init_repo(metadata: str, path: str, deep_clone: bool) -> Repo: def init_ssh(key: str, key_path: Path) -> None: if not key: - logging.warning('Private Key required for SSH Git') + logging.warning('Private key required for SSH Git') return logging.info('Private Key found, writing to disk') key_path.mkdir(exist_ok=True)