From 97221609f4dc36d7fc03abe910ac9864f11dab42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 28 Oct 2024 14:57:20 +0100 Subject: [PATCH] Remove audit, cleanup --- README.md | 19 +- c2cciutils/__init__.py | 212 -------------- c2cciutils/audit.py | 175 ----------- c2cciutils/configuration.py | 494 ++------------------------------ c2cciutils/pr_checks.py | 286 ------------------ c2cciutils/publish.py | 6 +- c2cciutils/schema.json | 217 -------------- c2cciutils/scripts/audit.py | 41 --- c2cciutils/scripts/main.py | 5 - c2cciutils/scripts/pr_checks.py | 78 ----- config.md | 56 ---- poetry.lock | 56 +--- pyproject.toml | 18 +- 13 files changed, 39 insertions(+), 1624 deletions(-) delete mode 100644 c2cciutils/audit.py delete mode 100644 c2cciutils/pr_checks.py delete mode 100644 c2cciutils/scripts/audit.py delete mode 100644 c2cciutils/scripts/pr_checks.py diff --git a/README.md b/README.md index 357777b5f..9b5f47c92 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,6 @@ see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing When we create a tag by default with the `changelog` workflow a release is created on GitHub, a changelog is generated and added to the release. -## Security - -The security is managed by the `c2cciutils-audit` command with Snyk, it will audit the dependencies of the project on every -stabilization branches, if possible a pull request is created automatically to update the dependencies. - -When we publish a Docker image the generated image is monitored by Snyk, this means that Snyk will search -for all the dependencies and send the list to the Snyk web site to be monitored. -We also do a test of the image and log the result (This will never cause the build to fail). - ## Checks C2C CI utils will no more provide a tool to do a check of the project, this is replaced by `pre-commit`, @@ -53,21 +44,16 @@ workflow will delete the workflows older than 500 days. C2cciutils make easier to have those workflows in a project: -- `audit.yaml`: Audit the stabilization branches of the application against vulnerabilities in the python and node dependency - `auto-review.yaml`: Auto review the Renovate pull requests - `backport.yaml`: Trigger the backports (work with labels) - `clean.yaml`: Clean the Docker images related on a deleted feature branch - `main.yaml`: Main workflow especially with the c2cciutils-checks command -- `changelog.yaml`: Generate the changelog and create the release on GitHub -- `delete-old-workflows-run.yaml`: Delete the old workflows -- `pr-checks.yaml`: Run the checks on the pull requests All the provided commands used in the workflow: - `c2cciutils`: some generic tools. - `c2cciutils-version`: Create a new version of the project. -- `c2cciutils-checks`: Run the checks on the code (those checks don't need any project dependencies). -- `c2cciutils-audit`: Do the audit, the main difference with checks is that it can change between runs on the same code. +- `c2cciutils-env`: Print some environment information. - `c2cciutils-publish`: Publish the project. - `c2cciutils-clean`: Delete Docker images on Docker Hub after corresponding branch have been deleted. @@ -128,7 +114,6 @@ You can override the configuration with the file `ci/config.yaml`. At the base of the configuration you have: - `version`: Contains some regular expressions to find the versions branches and tags, and to convert them into application versions. -- `audit`: The audit configuration, see `c2cciutils/audit.py` for more information. - `publish`: The publishing configuration, see `c2cciutils/publish.py` for more information. Many actions can be disabled by setting the corresponding configuration part to `False`. @@ -239,7 +224,7 @@ To make it working in the `Dockerfile` you should have in the `poetry` stage: ```Dockerfile ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev -RUN poetry export --extras=checks --extras=publish --extras=audit --output=requirements.txt \ +RUN poetry export --extras=checks --extras=publish --output=requirements.txt \ && poetry export --with=dev --output=requirements-dev.txt ``` diff --git a/c2cciutils/__init__.py b/c2cciutils/__init__.py index f94c1e97c..bf2565953 100644 --- a/c2cciutils/__init__.py +++ b/c2cciutils/__init__.py @@ -11,7 +11,6 @@ from re import Match, Pattern from typing import Any, Optional, TypedDict, cast -import magic import requests import ruamel.yaml @@ -126,8 +125,6 @@ def get_config() -> c2cciutils.configuration.Configuration: default_config = { "publish": publish_config, - "pr-checks": c2cciutils.configuration.PULL_REQUEST_CHECKS_DEFAULT, - "audit": c2cciutils.configuration.AUDIT_DEFAULT, } merge(default_config, config) @@ -411,160 +408,6 @@ def graphql(query_file: str, variables: dict[str, Any], default: Any = None) -> return cast(dict[str, Any], json_response["data"]) -def get_git_files_mime( - mime_type: Optional[list[str]] = None, - extensions: Optional[list[str]] = None, - ignore_patterns_re: Optional[list[str]] = None, -) -> list[str]: - """ - Get list of paths from git with all the files that have the specified mime type. - - Arguments: - mime_type: The considered MIME type - extensions: The considered extensions - ignore_patterns_re: A list of regular expressions of files that we should ignore - """ - if mime_type is None: - mime_type = ["text/x-python", "text/x-script.python"] - if extensions is None: - extensions = [".py"] - ignore_patterns_compiled = [re.compile(p) for p in ignore_patterns_re or []] - result = [] - - for filename in subprocess.check_output(["git", "ls-files"]).decode().strip().split("\n"): - if os.path.isfile(filename) and ( - os.path.splitext(filename)[1] in extensions or magic.from_file(filename, mime=True) in mime_type - ): - accept = True - for pattern in ignore_patterns_compiled: - if pattern.search(filename): - accept = False - break - if accept: - result.append(filename) - return result - - -def get_branch(branch: Optional[str], master_branch: str = "master") -> str: - """ - Get the branch name. - - Arguments: - branch: The forced to use branch name - master_branch: The master branch name, can be used as default value - - Return the branch name - """ - if branch is not None: - return branch - try: - branch = ( - subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], check=True, stdout=subprocess.PIPE) - .stdout.decode() - .strip() - ) - except subprocess.CalledProcessError as exception: - print(f"Error getting branch: {exception}") - branch = "HEAD" - - if branch == "HEAD": - branch = os.environ.get("GITHUB_HEAD_REF", master_branch) - assert branch is not None - return branch - - -def get_based_on_master( - repo: list[str], - override_current_branch: Optional[str], - master_branch: str, - config: c2cciutils.configuration.Configuration, -) -> bool: - """ - Check that we are not on a release branch (to avoid errors in versions check). - - This function will check the last 20 commits in current branch, - and for each other branch (max 50) check if any commit in last 10 commits is the current one. - - Arguments: - repo: The repository [, ] - override_current_branch: The branch to use instead of the current one - master_branch: The master branch name - config: The full configuration - """ - if os.environ.get("GITHUB_REF", "").startswith("refs/tags/"): - # The tags are never consider as based on master - return False - current_branch = get_branch(override_current_branch, master_branch) - if current_branch == master_branch: - return True - branches_re = compile_re(config["version"].get("branch_to_version_re", [])) - if does_match(current_branch, branches_re): - return False - if os.environ.get("GITHUB_BASE_REF"): - return os.environ.get("GITHUB_BASE_REF") == master_branch - commits_repository_json = graphql( - "commits.graphql", {"name": repo[1], "owner": repo[0], "branch": current_branch} - ).get("repository", {}) - commits_json = ( - commits_repository_json.get("ref", {}).get("target", {}).get("history", {}).get("nodes", []) - if commits_repository_json.get("ref") - else [] - ) - branches_json = [ - branch - for branch in ( - graphql("branches.graphql", {"name": repo[1], "owner": repo[0]})["repository"]["refs"]["nodes"] - ) - if branch["name"] != current_branch and does_match(branch["name"], branches_re) - ] - based_branch = master_branch - found = False - for commit in commits_json: - for branch in branches_json: - commits = [ - branch_commit - for branch_commit in branch["target"]["history"]["nodes"] - if commit["oid"] == branch_commit["oid"] - ] - if commits: - based_branch = branch["name"] - found = True - break - if found: - break - return based_branch == master_branch - - -def get_codespell_command(config: c2cciutils.configuration.Configuration, fix: bool = False) -> list[str]: - """ - Get the codespell command. - - Arguments: - config: The full configuration - fix: If we should fix the errors - """ - codespell_config = config.get("codespell", {}) - codespell_config = codespell_config if isinstance(codespell_config, dict) else {} - command = ["codespell"] - if fix: - command.append("--write-changes") - for spell_ignore_file in ( - ".github/spell-ignore-words.txt", - "spell-ignore-words.txt", - ".spell-ignore-words.txt", - ): - if os.path.exists(spell_ignore_file): - command.append(f"--ignore-words={spell_ignore_file}") - break - dictionaries = codespell_config.get( - "internal_dictionaries", c2cciutils.configuration.CODESPELL_DICTIONARIES_DEFAULT - ) - if dictionaries: - command.append("--builtin=" + ",".join(dictionaries)) - command += codespell_config.get("arguments", c2cciutils.configuration.CODESPELL_ARGUMENTS_DEFAULT) - return command - - def snyk_exec() -> tuple[str, dict[str, str]]: """Get the Snyk cli executable path.""" if not os.path.exists(os.path.join(os.path.dirname(__file__), "node_modules")): @@ -580,58 +423,3 @@ def snyk_exec() -> tuple[str, dict[str, str]]: subprocess.run(["snyk", "config", "set", f"org={env['SNYK_ORG']}"], check=True, env=env) return os.path.join(os.path.dirname(os.path.abspath(__file__)), "node_modules/snyk/bin/snyk"), env - - -def create_pull_request_if_needed( - current_branch: str, - new_branch: str, - commit_message: str, - pull_request_extra_arguments: Optional[list[str]] = None, -) -> bool: - """ - Create a pull request if there are changes. - """ - if pull_request_extra_arguments is None: - pull_request_extra_arguments = ["--fill"] - - diff_proc = subprocess.run(["git", "diff", "--quiet"]) # pylint: disable=subprocess-run-check - if diff_proc.returncode != 0: - print("::group::Diff") - sys.stdout.flush() - sys.stderr.flush() - subprocess.run(["git", "diff"], check=True) - print("::endgroup::") - - git_hash = subprocess.run( - ["git", "rev-parse", "HEAD"], check=True, stdout=subprocess.PIPE, encoding="utf-8" - ).stdout.strip() - subprocess.run(["git", "checkout", "-b", new_branch], check=True) - subprocess.run(["git", "add", "--all"], check=True) - subprocess.run(["git", "commit", f"--message={commit_message}"], check=True) - if os.environ.get("TEST") != "TRUE": - subprocess.run( - ["git", "push", "--force", "origin", new_branch], - check=True, - ) - env = os.environ.copy() - if "GH_TOKEN" not in env: - if "GITHUB_TOKEN" in env: - env["GH_TOKEN"] = env["GITHUB_TOKEN"] - else: - env["GH_TOKEN"] = str(c2cciutils.gopass("gs/ci/github/token/gopass")) - subprocess.run( - [ - "gh", - "pr", - "create", - f"--base={current_branch}", - *pull_request_extra_arguments, - ], - check=True, - env=env, - ) - else: - subprocess.run(["git", "reset", "--hard"], check=True) - subprocess.run(["git", "checkout", git_hash], check=True) - - return diff_proc.returncode != 0 diff --git a/c2cciutils/audit.py b/c2cciutils/audit.py deleted file mode 100644 index bbd0b0177..000000000 --- a/c2cciutils/audit.py +++ /dev/null @@ -1,175 +0,0 @@ -""" -The auditing functions. -""" - -import os.path -import subprocess # nosec -import sys -from argparse import Namespace - -import c2cciutils -import c2cciutils.configuration - - -def print_versions( - config: c2cciutils.configuration.PrintVersions, - full_config: c2cciutils.configuration.Configuration, - args: Namespace, -) -> bool: - """ - Print the versions. - - Arguments: - config: The audit section config - full_config: All the CI config - args: The parsed command arguments - """ - del full_config, args - - print("::group::Versions") - c2cciutils.print_versions(config) - print("::endgroup::") - - return True - - -def snyk( - config: c2cciutils.configuration.AuditSnykConfig, - full_config: c2cciutils.configuration.Configuration, - args: Namespace, -) -> bool: - """ - Audit the code with Snyk. - """ - del full_config - - one_done = False - install_success = True - test_success = True - - for file in ( - subprocess.run( - ["git", "ls-files", "requirements.txt", "*/requirements.txt"], stdout=subprocess.PIPE, check=True - ) - .stdout.decode() - .strip() - .split("\n") - ): - if not file: - continue - if file in config.get( - "files_no_install", c2cciutils.configuration.AUDIT_SNYK_FILES_NO_INSTALL_DEFAULT - ): - continue - print(f"::notice::Install from: {file}") - if not one_done: - print("::group::Install dependencies") - one_done = True - sys.stdout.flush() - sys.stderr.flush() - proc = subprocess.run( # pylint: disable=subprocess-run-check - [ - "pip", - "install", - *config.get( - "pip_install_arguments", c2cciutils.configuration.AUDIT_SNYK_PIP_INSTALL_ARGUMENTS_DEFAULT - ), - f"--requirement={file}", - ] - ) - if proc.returncode != 0: - print(f"::error::With error from: {file}") - install_success &= proc.returncode == 0 - - for file in ( - subprocess.run(["git", "ls-files", "Pipfile", "*/Pipfile"], stdout=subprocess.PIPE, check=True) - .stdout.decode() - .strip() - .split("\n") - ): - if not file: - continue - if file in config.get( - "files_no_install", c2cciutils.configuration.AUDIT_SNYK_FILES_NO_INSTALL_DEFAULT - ): - continue - if not one_done: - print("::group::Install dependencies") - one_done = True - print(f"::notice::Install from: {file}") - directory = os.path.dirname(os.path.abspath(file)) - - sys.stdout.flush() - sys.stderr.flush() - proc = subprocess.run( # pylint: disable=subprocess-run-check - [ - "pipenv", - "sync", - *config.get( - "pipenv_sync_arguments", c2cciutils.configuration.AUDIT_SNYK_PIPENV_SYNC_ARGUMENTS_DEFAULT - ), - ], - cwd=directory, - ) - if proc.returncode != 0: - print(f"::error::With error from: {file}") - install_success &= proc.returncode == 0 - - if one_done: - print("::endgroup::") - if not install_success: - print("::error::Error while installing the dependencies") - - snyk_exec, env = c2cciutils.snyk_exec() - if not args.fix: - command = [snyk_exec, "monitor", f"--target-reference={args.branch}"] + config.get( - "monitor_arguments", c2cciutils.configuration.AUDIT_SNYK_MONITOR_ARGUMENTS_DEFAULT - ) - print(f"::group::Run: {' '.join(command)}") - sys.stdout.flush() - sys.stderr.flush() - subprocess.run(command, env=env) # pylint: disable=subprocess-run-check - print("::endgroup::") - - command = [snyk_exec, "test"] + config.get( - "test_arguments", c2cciutils.configuration.AUDIT_SNYK_TEST_ARGUMENTS_DEFAULT - ) - print(f"::group::Run: {' '.join(command)}") - sys.stdout.flush() - sys.stderr.flush() - test_proc = subprocess.run(command, env=env) # pylint: disable=subprocess-run-check - print("::endgroup::") - if test_proc.returncode != 0: - test_success = False - print("::error::With error") - - # Clean all the changes to isolate the fix diff - subprocess.run(["git", "reset", "--hard"], check=True) - - command = [snyk_exec, "fix"] + config.get( - "fix_arguments", c2cciutils.configuration.AUDIT_SNYK_FIX_ARGUMENTS_DEFAULT - ) - print(f"::group::Run: {' '.join(command)}") - sys.stdout.flush() - sys.stderr.flush() - snyk_fix_proc = subprocess.run( # pylint: disable=subprocess-run-check - command, env={**env, "FORCE_COLOR": "false"}, stdout=subprocess.PIPE, encoding="utf-8" - ) - snyk_fix_message = snyk_fix_proc.stdout.strip() - print("::endgroup::") - - has_diff = False - if not args.fix: - current_branch = c2cciutils.get_branch(args.branch) - fix_github_create_pull_request_arguments = config.get( - "fix_github_create_pull_request_arguments", - c2cciutils.configuration.AUDIT_SNYK_FIX_PULL_REQUEST_ARGUMENTS_DEFAULT, - ) - has_diff = c2cciutils.create_pull_request_if_needed( - current_branch, - f"snyk-fix/{current_branch}", - "Snyk auto fix", - [f"--body={snyk_fix_message}", *fix_github_create_pull_request_arguments], - ) - - return install_success and test_success and not has_diff diff --git a/c2cciutils/configuration.py b/c2cciutils/configuration.py index d093e4e1e..c8102f522 100644 --- a/c2cciutils/configuration.py +++ b/c2cciutils/configuration.py @@ -4,275 +4,63 @@ from typing import Any, Literal, TypedDict, Union -AUDIT_DEFAULT = {"snyk": True} -""" Default value of the field path 'configuration audit' """ - -AUDIT_SNYK_FILES_NO_INSTALL_DEFAULT: list[Any] = [] -""" Default value of the field path 'Audit Snyk config files_no_install' """ - - -AUDIT_SNYK_FIX_ARGUMENTS_DEFAULT = ["--all-projects"] -""" Default value of the field path 'Audit Snyk config fix_arguments' """ - - -AUDIT_SNYK_FIX_PULL_REQUEST_ARGUMENTS_DEFAULT = ["--fill", "--label=dependencies"] -""" Default value of the field path 'Audit Snyk config fix_github_create_pull_request_arguments' """ - - -AUDIT_SNYK_MONITOR_ARGUMENTS_DEFAULT = ["--all-projects"] -""" Default value of the field path 'Audit Snyk config monitor_arguments' """ - - -AUDIT_SNYK_PIPENV_SYNC_ARGUMENTS_DEFAULT: list[Any] = [] -""" Default value of the field path 'Audit Snyk config pipenv_sync_arguments' """ - - -AUDIT_SNYK_PIP_INSTALL_ARGUMENTS_DEFAULT = ["--user"] -""" Default value of the field path 'Audit Snyk config pip_install_arguments' """ - - -AUDIT_SNYK_TEST_ARGUMENTS_DEFAULT = ["--all-projects", "--fail-on=all", "--severity-threshold=medium"] -""" Default value of the field path 'Audit Snyk config test_arguments' """ - - -class Audit(TypedDict, total=False): +class Configuration(TypedDict, total=False): """ - Audit. + configuration. - The audit configuration - - default: - snyk: true + C2C CI utils configuration file """ - snyk: "AuditWithSnyk" + print_versions: "PrintVersions" """ - Audit with Snyk. - - The audit Snyk configuration - - Aggregation type: oneOf - Subtype: "AuditSnykConfig" - """ - - -class AuditSnykConfig(TypedDict, total=False): - """ - Audit Snyk config. - - The audit Pipfile configuration - """ - - test_arguments: list[str] - """ - audit snyk test arguments. - - The Snyk test arguments - - default: - - --all-projects - - --fail-on=all - - --severity-threshold=medium - """ - - monitor_arguments: list[str] - """ - audit snyk monitor arguments. - - The Snyk monitor arguments - - default: - - --all-projects - """ - - fix_arguments: list[str] - """ - audit snyk fix arguments. - - The Snyk fix arguments - - default: - - --all-projects - """ - - fix_github_create_pull_request_arguments: list[str] - """ - audit snyk fix pull request arguments. - - The Snyk fix pull request extra arguments - - default: - - --fill - - --label=dependencies - """ - - pip_install_arguments: list[str] - """ - audit snyk pip install arguments. - - The Snyk pip install arguments - - default: - - --user - """ - - pipenv_sync_arguments: list[str] - """ - audit snyk pipenv sync arguments. - - The Snyk pipenv sync arguments + Print versions. - default: - [] + The print versions configuration """ - files_no_install: list[str] + publish: "Publish" """ - audit snyk files no install. + Publish. - The list of files to not install + The publishing configurations default: - [] + docker: + images: + helm: + folders: + versions: + - version_tag + pypi: + packages: + versions: + - version_tag """ - -AuditWithSnyk = Union["AuditSnykConfig", bool] -""" -Audit with Snyk. - -The audit Snyk configuration - -Aggregation type: oneOf -Subtype: "AuditSnykConfig" -""" - - -CODESPELL_ARGUMENTS_DEFAULT = ["--quiet-level=2", "--check-filenames", "--ignore-words-list=ro"] -""" Default value of the field path 'Codespell arguments' """ - - -CODESPELL_DICTIONARIES_DEFAULT = ["clear", "rare", "informal", "code", "names", "en-GB_to_en-US"] -""" Default value of the field path 'Codespell internal_dictionaries' """ - - -CODESPELL_IGNORE_REGULAR_EXPRESSION_DEFAULT = ["(.*/)?poetry\\.lock", "(.*/)?package-lock\\.json"] -""" Default value of the field path 'Codespell ignore_re' """ - - -class Codespell(TypedDict, total=False): + version: "Version" """ - Codespell. + Version. - The codespell check configuration + The version configurations """ - internal_dictionaries: list[str] + k8s: "K8SConfiguration" """ - codespell dictionaries. - - List of argument that will be added to the codespell command + K8s configuration. default: - - clear - - rare - - informal - - code - - names - - en-GB_to_en-US - """ - - arguments: list[str] + {} """ - codespell arguments. - List of argument that will be added to the codespell command - - default: - - --quiet-level=2 - - --check-filenames - - --ignore-words-list=ro + dpkg: "Dpkg" """ + dpkg. - ignore_re: list[str] - r""" - codespell ignore regular expression. - - List of regular expression that should be ignored - - default: - - (.*/)?poetry\.lock - - (.*/)?package-lock\.json + The configuration use t manage the dpkg packages """ -# | configuration. -# | -# | C2C CI utils configuration file -Configuration = TypedDict( - "Configuration", - { - # | Print versions. - # | - # | The print versions configuration - "print_versions": "PrintVersions", - # | Codespell. - # | - # | The codespell check configuration - "codespell": "Codespell", - # | Audit. - # | - # | The audit configuration - # | - # | default: - # | snyk: true - "audit": "Audit", - # | Pull request checks. - # | - # | The PR check configuration - # | - # | default: - # | add_issue_link: true - # | commits_messages: true - # | commits_spell: true - # | pull_request_labels: true - # | pull_request_spell: true - "pr-checks": "PullRequestChecks", - # | Publish. - # | - # | The publishing configurations - # | - # | default: - # | docker: - # | images: - # | helm: - # | folders: - # | versions: - # | - version_tag - # | pypi: - # | packages: - # | versions: - # | - version_tag - "publish": "Publish", - # | Version. - # | - # | The version configurations - "version": "Version", - # | K8s configuration. - # | - # | default: - # | {} - "k8s": "K8SConfiguration", - # | dpkg. - # | - # | The configuration use t manage the dpkg packages - "dpkg": "Dpkg", - }, - total=False, -) - - DB_CONFIGURATION_DEFAULT: dict[str, Any] = {} """ Default value of the field path 'K8s configuration db' """ @@ -508,48 +296,6 @@ class K8SConfiguration(TypedDict, total=False): """ Default value of the field path 'publish_pypi' """ -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_FIRST_CAPITAL_DEFAULT = True -""" Default value of the field path 'pull request checks commits messages configuration check_first_capital' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_FIXUP_DEFAULT = True -""" Default value of the field path 'pull request checks commits messages configuration check_fixup' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_MIN_HEAD_LENGTH_DEFAULT = 5 -""" Default value of the field path 'pull request checks commits messages configuration min_head_length' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_NO_MERGE_COMMITS_DEFAULT = True -""" Default value of the field path 'pull request checks commits messages configuration check_no_merge_commits' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_NO_OWN_REVERT_DEFAULT = True -""" Default value of the field path 'pull request checks commits messages configuration check_no_own_revert' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_ONLY_HEAD_DEFAULT = True -""" Default value of the field path 'pull request checks commits spelling configuration only_head' """ - - -PULL_REQUEST_CHECKS_COMMITS_MESSAGES_SQUASH_DEFAULT = True -""" Default value of the field path 'pull request checks commits messages configuration check_squash' """ - - -PULL_REQUEST_CHECKS_DEFAULT = { - "commits_messages": True, - "commits_spell": True, - "pull_request_spell": True, - "pull_request_labels": True, - "add_issue_link": True, -} -""" Default value of the field path 'configuration pr-checks' """ - - -PULL_REQUEST_CHECKS_ONLY_HEAD_DEFAULT = True -""" Default value of the field path 'pull request checks pull request spelling configuration only_head' """ - - class PrintVersions(TypedDict, total=False): """ Print versions. @@ -894,190 +640,6 @@ class PublishPypiPackage(TypedDict, total=False): """ The command used to do the build """ -class PullRequestChecks(TypedDict, total=False): - """ - Pull request checks. - - The PR check configuration - - default: - add_issue_link: true - commits_messages: true - commits_spell: true - pull_request_labels: true - pull_request_spell: true - """ - - commits_messages: "PullRequestChecksCommitsMessages" - """ - pull request checks commits messages. - - Check the pull request commits messages - - Aggregation type: oneOf - Subtype: "PullRequestChecksCommitsMessagesConfiguration" - """ - - commits_spell: "PullRequestChecksCommitsSpelling" - """ - pull request checks commits spelling. - - Aggregation type: oneOf - Subtype: "PullRequestChecksCommitsSpellingConfiguration" - """ - - pull_request_spell: "PullRequestChecksPullRequestSpelling" - """ - pull request checks pull request spelling. - - Aggregation type: oneOf - Subtype: "PullRequestChecksPullRequestSpellingConfiguration" - """ - - pull_request_labels: "PullRequestChecksRequestLabels" - """ - pull request checks request labels. - - According the create changelog configuration - """ - - add_issue_link: "PullRequestChecksAddIssueLink" - """ pull request checks add issue link. """ - - -PullRequestChecksAddIssueLink = bool -""" pull request checks add issue link. """ - - -PullRequestChecksCommitsMessages = Union["PullRequestChecksCommitsMessagesConfiguration", bool] -""" -pull request checks commits messages. - -Check the pull request commits messages - -Aggregation type: oneOf -Subtype: "PullRequestChecksCommitsMessagesConfiguration" -""" - - -class PullRequestChecksCommitsMessagesConfiguration(TypedDict, total=False): - """ - pull request checks commits messages configuration. - - The commit message check configuration - """ - - check_fixup: bool - """ - pull request checks commits messages fixup. - - Check that we don't have one fixup commit in the pull request - - default: True - """ - - check_squash: bool - """ - pull request checks commits messages squash. - - Check that we don't have one squash commit in the pull request - - default: True - """ - - check_first_capital: bool - """ - pull request checks commits messages first capital. - - Check that the all the commits message starts with a capital letter - - default: True - """ - - min_head_length: int - """ - pull request checks commits messages min head length. - - Check that the commits message head is at least this long, use 0 to disable - - default: 5 - """ - - check_no_merge_commits: bool - """ - pull request checks commits messages no merge commits. - - Check that we don't have merge commits in the pull request - - default: True - """ - - check_no_own_revert: bool - """ - pull request checks commits messages no own revert. - - Check that we don't have reverted one of our commits in the pull request - - default: True - """ - - -PullRequestChecksCommitsSpelling = Union["PullRequestChecksCommitsSpellingConfiguration", bool] -""" -pull request checks commits spelling. - -Aggregation type: oneOf -Subtype: "PullRequestChecksCommitsSpellingConfiguration" -""" - - -class PullRequestChecksCommitsSpellingConfiguration(TypedDict, total=False): - """ - pull request checks commits spelling configuration. - - Configuration used to check the spelling of the commits - """ - - only_head: bool - """ - pull request checks commits messages only head. - - default: True - """ - - -PullRequestChecksPullRequestSpelling = Union["PullRequestChecksPullRequestSpellingConfiguration", bool] -""" -pull request checks pull request spelling. - -Aggregation type: oneOf -Subtype: "PullRequestChecksPullRequestSpellingConfiguration" -""" - - -class PullRequestChecksPullRequestSpellingConfiguration(TypedDict, total=False): - """ - pull request checks pull request spelling configuration. - - Configuration used to check the spelling of the title and body of the pull request - """ - - only_head: bool - """ - pull request checks only head. - - default: True - """ - - -PullRequestChecksRequestLabels = bool -""" -pull request checks request labels. - -According the create changelog configuration -""" - - class Version(TypedDict, total=False): """ Version. diff --git a/c2cciutils/pr_checks.py b/c2cciutils/pr_checks.py deleted file mode 100644 index 728874a5b..000000000 --- a/c2cciutils/pr_checks.py +++ /dev/null @@ -1,286 +0,0 @@ -""" -The pull request checking functions. - -Commits, messages and labels. -""" - -import os -import re -import subprocess # nosec -from tempfile import NamedTemporaryFile -from typing import Any, Optional - -import requests -import yaml - -import c2cciutils.configuration - - -def _commit_intro(need_separator: bool, commit: dict[str, Any]) -> bool: - head = commit["commit"]["message"].split("\n")[0] - if need_separator: - print("-" * 30) - print(f'{commit["commit"]["tree"]["sha"]}: {head}') - return True - - -def print_event(github_event: dict[str, Any], **kwargs: Any) -> bool: - """ - Print the github object. - """ - del kwargs - print(yaml.dump(github_event, default_flow_style=False, Dumper=yaml.SafeDumper)) - return True - - -def commits_messages( - config: c2cciutils.configuration.PullRequestChecksCommitsMessagesConfiguration, - commits: list[dict[str, Any]], - **kwargs: Any, -) -> bool: - """ - Check the commits messages. - - - They should start with a capital letter. - - They should not be too short. - - They should not be a squash or fixup commit. - - They should not be a merge commit. - - They should not be a revert commit. - """ - del kwargs - - need_separator = False - success = True - first_capital = re.compile(r"^[^a-z]") - commit_hash = set() - for commit in commits: - need_head = True - commit_hash.add(commit["sha"]) - message_lines = commit["commit"]["message"].split("\n") - head = message_lines[0] - if config.get( - "check_fixup", c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_FIXUP_DEFAULT - ) and head.startswith("fixup! "): - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print("::error::Fixup message not allowed") - success = False - if config.get( - "check_squash", c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_SQUASH_DEFAULT - ) and head.startswith("squash! "): - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print("::error::Squash message not allowed") - success = False - if ( - config.get( - "check_first_capital", - c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_FIRST_CAPITAL_DEFAULT, - ) - and first_capital.match(head) is None - ): - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print("::error::The first letter of message head should be a capital") - success = False - min_length = config.get( - "min_head_length", - c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_MIN_HEAD_LENGTH_DEFAULT, - ) - if min_length > 0 and len(head) < min_length: - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print(f"The message head should be at least {min_length} characters long") - success = False - if ( - config.get( - "check_no_merge_commits", - c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_NO_MERGE_COMMITS_DEFAULT, - ) - and len(commit["parents"]) != 1 - ): - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print("::error::The merge commit are not allowed") - success = False - if config.get( - "check_no_own_revert", - c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_NO_OWN_REVERT_DEFAULT, - ) and ( - head.startswith("Revert ") - and len(message_lines) == 3 - and message_lines[2].startswith("This reverts commit ") - ): - revert_commit_hash = message_lines[2][len("This reverts commit ") : -1] - if revert_commit_hash in commit_hash: - if need_head: - need_separator = _commit_intro(need_separator, commit) - need_head = False - print(f"Revert own commits is not allowed ({revert_commit_hash})") - success = False - continue - return success - - -def commits_spell( - config: c2cciutils.configuration.PullRequestChecksCommitsSpellingConfiguration, - full_config: c2cciutils.configuration.Configuration, - commits: list[dict[str, Any]], - **kwargs: Any, -) -> bool: - """Check the spelling of the commits body.""" - del kwargs - - spellcheck_cmd = c2cciutils.get_codespell_command(full_config) - - success = True - need_separator = False - for commit in commits: - with NamedTemporaryFile("w+t", encoding="utf-8", suffix=".yaml") as temp_file: - if config.get( - "only_head", c2cciutils.configuration.PULL_REQUEST_CHECKS_COMMITS_MESSAGES_ONLY_HEAD_DEFAULT - ): - head = commit["commit"]["message"].split("\n")[0] - temp_file.write(head) - else: - temp_file.write(commit["commit"]["message"]) - temp_file.flush() - spell = subprocess.run( # nosec # pylint: disable=subprocess-run-check - spellcheck_cmd + [temp_file.name], capture_output=True - ) - if spell.returncode != 0: - need_separator = _commit_intro(need_separator, commit) - print("::error::Code spell error") - print(spell.stderr) - print(spell.stdout) - success = False - return success - - -def pull_request_spell( - config: c2cciutils.configuration.PullRequestChecksPullRequestSpellingConfiguration, - full_config: c2cciutils.configuration.Configuration, - github_event: dict[str, Any], - **kwargs: Any, -) -> bool: - """Check the spelling of the pull request title and message.""" - del kwargs - - spellcheck_cmd = c2cciutils.get_codespell_command(full_config) - - with NamedTemporaryFile("w+t") as temp_file: - temp_file.write(github_event["event"]["pull_request"]["title"]) - temp_file.write("\n") - if not config.get( - "only_head", c2cciutils.configuration.PULL_REQUEST_CHECKS_ONLY_HEAD_DEFAULT - ) and github_event["event"]["pull_request"].get("body"): - temp_file.write("\n") - temp_file.write(github_event["event"]["pull_request"]["body"]) - temp_file.write("\n") - temp_file.flush() - spell = subprocess.run( # nosec # pylint: disable=subprocess-run-check - spellcheck_cmd + [temp_file.name], capture_output=True - ) - if spell.returncode != 0: - print("::error::Code spell error in pull request") - print(spell.stderr) - print(spell.stdout) - return False - return True - - -def pull_request_labels(github_event: dict[str, Any], **kwargs: Any) -> bool: - """Check it the label are set correctly for the changelog generation.""" - del kwargs - - if github_event["actor"] == "renovate[bot]": - return True - - if not os.path.exists(".github/changelog-config.yaml"): - return True - - required_labels = [] - with open(".github/changelog-config.yaml", encoding="utf-8") as changelog_config_file: - changelog_config = yaml.load(changelog_config_file, Loader=yaml.SafeLoader) - for section in changelog_config.values(): - if "labels" in section: - required_labels.extend(section["labels"]) - - print(f"Required one onf the following labels: {', '.join(required_labels)}") - - if required_labels: - labels = [ - label["name"] - for label in github_event["event"]["pull_request"]["labels"] - if label["name"] in required_labels - ] - if len(labels) == 0: - print(f"::error::No required label found: {', '.join(required_labels)}") - return False - if len(labels) > 1: - print(f"::error::Too many required labels found: {', '.join(labels)}") - return False - return True - - -GET_ISSUE_RE = [ - re.compile(r"^([A-Z]{3,6}-[0-9]+)-.*$"), - re.compile(r"^([a-z]{3,6}-[0-9]+)-.*$"), - re.compile(r"^.*-([A-Z]{3,6}-[0-9]+)$"), - re.compile(r"^.*-([a-z]{3,6}-[0-9]+)$"), -] - - -def _get_issue_number(name: str) -> Optional[str]: - for re_ in GET_ISSUE_RE: - match = re_.match(name) - if match is not None: - return match.group(1) - return None - - -def add_issue_link(github_event: dict[str, Any], **kwargs: Any) -> bool: - """Add a comment with the link to Jira if needed.""" - del kwargs - - issue_number = _get_issue_number(github_event["event"]["pull_request"]["head"]["ref"]) - - if issue_number is None: - return True - - issue_number = issue_number.upper() - - body = github_event["event"]["pull_request"].get("body") or "" - if issue_number in body.upper(): - return True - - comments_response = requests.get( - github_event["event"]["pull_request"]["_links"]["comments"]["href"], - timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")), - headers=c2cciutils.add_authorization_header({}), - ) - c2cciutils.check_response(comments_response) - comments = comments_response.json() - - for comment in comments: - if issue_number in comment.get("body", "").upper(): - return True - - response = requests.post( - github_event["event"]["pull_request"]["_links"]["comments"]["href"], - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {github_event['token']}", - }, - json={"body": f"See also: [{issue_number}](https://jira.camptocamp.com/browse/{issue_number})"}, - timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")), - ) - - if not response.ok: - print(f"Unable to add the comment: {response.text}") - return response.ok diff --git a/c2cciutils/publish.py b/c2cciutils/publish.py index 58a16b281..3ae07c5e6 100644 --- a/c2cciutils/publish.py +++ b/c2cciutils/publish.py @@ -10,11 +10,11 @@ import re import subprocess # nosec import sys +import tomllib import uuid from typing import Optional import ruamel.yaml -import tomlkit from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow @@ -289,8 +289,8 @@ def pip( if os.path.exists(os.path.join(cwd, "pyproject.toml")): use_poetry = False if "build_command" not in package: - with open(os.path.join(cwd, "pyproject.toml"), encoding="utf-8") as project_file: - pyproject = tomlkit.load(project_file) + with open(os.path.join(cwd, "pyproject.toml"), "rb") as project_file: + pyproject = tomllib.load(project_file) re_splitter = re.compile(r"[<>=]+") for requirement in pyproject.get("build-system", {}).get("requires", []): requirement_split = re_splitter.split(requirement) diff --git a/c2cciutils/schema.json b/c2cciutils/schema.json index 6e768193d..a4bbe9ea2 100644 --- a/c2cciutils/schema.json +++ b/c2cciutils/schema.json @@ -6,192 +6,6 @@ "description": "C2C CI utils configuration file", "additionalProperties": false, "definitions": { - "audit_snyk": { - "title": "Audit with Snyk", - "description": "The audit Snyk configuration", - "oneOf": [ - { - "title": "Audit Snyk config", - "description": "The audit Pipfile configuration", - "type": "object", - "properties": { - "test_arguments": { - "description": "The Snyk test arguments", - "title": "audit snyk test arguments", - "default": ["--all-projects", "--fail-on=all", "--severity-threshold=medium"], - "type": "array", - "items": { "type": "string" } - }, - "monitor_arguments": { - "description": "The Snyk monitor arguments", - "title": "audit snyk monitor arguments", - "default": ["--all-projects"], - "type": "array", - "items": { "type": "string" } - }, - "fix_arguments": { - "description": "The Snyk fix arguments", - "title": "audit snyk fix arguments", - "default": ["--all-projects"], - "type": "array", - "items": { "type": "string" } - }, - "fix_github_create_pull_request_arguments": { - "description": "The Snyk fix pull request extra arguments", - "title": "audit snyk fix pull request arguments", - "default": ["--fill", "--label=dependencies"], - "type": "array", - "items": { "type": "string" } - }, - "pip_install_arguments": { - "description": "The Snyk pip install arguments", - "title": "audit snyk pip install arguments", - "default": ["--user"], - "type": "array", - "items": { "type": "string" } - }, - "pipenv_sync_arguments": { - "description": "The Snyk pipenv sync arguments", - "title": "audit snyk pipenv sync arguments", - "default": [], - "type": "array", - "items": { "type": "string" } - }, - "files_no_install": { - "description": "The list of files to not install", - "title": "audit snyk files no install", - "default": [], - "type": "array", - "items": { "type": "string" } - } - } - }, - { "type": "boolean" } - ] - }, - "codespell": { - "title": "Codespell", - "description": "The codespell check configuration", - "type": "object", - "properties": { - "internal_dictionaries": { - "title": "codespell dictionaries", - "description": "List of argument that will be added to the codespell command", - "default": ["clear", "rare", "informal", "code", "names", "en-GB_to_en-US"], - "type": "array", - "items": { "type": "string" } - }, - "arguments": { - "title": "codespell arguments", - "description": "List of argument that will be added to the codespell command", - "default": ["--quiet-level=2", "--check-filenames", "--ignore-words-list=ro"], - "type": "array", - "items": { "type": "string" } - }, - "ignore_re": { - "title": "codespell ignore regular expression", - "description": "List of regular expression that should be ignored", - "default": ["(.*/)?poetry\\.lock", "(.*/)?package-lock\\.json"], - "type": "array", - "items": { "type": "string" } - } - } - }, - "pr_checks_commits_messages": { - "title": "pull request checks commits messages", - "description": "Check the pull request commits messages", - "oneOf": [ - { - "title": "pull request checks commits messages configuration", - "description": "The commit message check configuration", - "type": "object", - "properties": { - "check_fixup": { - "title": "pull request checks commits messages fixup", - "description": "Check that we don't have one fixup commit in the pull request", - "default": true, - "type": "boolean" - }, - "check_squash": { - "title": "pull request checks commits messages squash", - "description": "Check that we don't have one squash commit in the pull request", - "default": true, - "type": "boolean" - }, - "check_first_capital": { - "title": "pull request checks commits messages first capital", - "description": "Check that the all the commits message starts with a capital letter", - "default": true, - "type": "boolean" - }, - "min_head_length": { - "title": "pull request checks commits messages min head length", - "description": "Check that the commits message head is at least this long, use 0 to disable", - "default": 5, - "type": "integer" - }, - "check_no_merge_commits": { - "title": "pull request checks commits messages no merge commits", - "description": "Check that we don't have merge commits in the pull request", - "default": true, - "type": "boolean" - }, - "check_no_own_revert": { - "title": "pull request checks commits messages no own revert", - "description": "Check that we don't have reverted one of our commits in the pull request", - "default": true, - "type": "boolean" - } - } - }, - { "type": "boolean" } - ] - }, - "pr_checks_commits_spell": { - "title": "pull request checks commits spelling", - "oneOf": [ - { - "title": "pull request checks commits spelling configuration", - "description": "Configuration used to check the spelling of the commits", - "type": "object", - "properties": { - "only_head": { - "title": "pull request checks commits messages only head", - "default": true, - "type": "boolean" - } - } - }, - { "type": "boolean" } - ] - }, - "pr_checks_pull_request_spell": { - "title": "pull request checks pull request spelling", - "oneOf": [ - { - "title": "pull request checks pull request spelling configuration", - "description": "Configuration used to check the spelling of the title and body of the pull request", - "type": "object", - "properties": { - "only_head": { - "title": "pull request checks only head", - "default": true, - "type": "boolean" - } - } - }, - { "type": "boolean" } - ] - }, - "pr_checks_pull_request_labels": { - "title": "pull request checks request labels", - "description": "According the create changelog configuration", - "type": "boolean" - }, - "pr_checks_add_issue_link": { - "title": "pull request checks add issue link", - "type": "boolean" - }, "publish_docker": { "title": "Publish Docker", "description": "The configuration used to publish on Docker", @@ -498,37 +312,6 @@ }, "properties": { "print_versions": { "$ref": "#/definitions/print_versions" }, - "codespell": { "$ref": "#/definitions/codespell" }, - "audit": { - "title": "Audit", - "description": "The audit configuration", - "type": "object", - "default": { - "snyk": true - }, - "properties": { - "snyk": { "$ref": "#/definitions/audit_snyk" } - } - }, - "pr-checks": { - "title": "Pull request checks", - "description": "The PR check configuration", - "type": "object", - "default": { - "commits_messages": true, - "commits_spell": true, - "pull_request_spell": true, - "pull_request_labels": true, - "add_issue_link": true - }, - "properties": { - "commits_messages": { "$ref": "#/definitions/pr_checks_commits_messages" }, - "commits_spell": { "$ref": "#/definitions/pr_checks_commits_spell" }, - "pull_request_spell": { "$ref": "#/definitions/pr_checks_pull_request_spell" }, - "pull_request_labels": { "$ref": "#/definitions/pr_checks_pull_request_labels" }, - "add_issue_link": { "$ref": "#/definitions/pr_checks_add_issue_link" } - } - }, "publish": { "title": "Publish", "description": "The publishing configurations", diff --git a/c2cciutils/scripts/audit.py b/c2cciutils/scripts/audit.py deleted file mode 100644 index 134078ec9..000000000 --- a/c2cciutils/scripts/audit.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 - -""" -The audit main function. -""" - -import argparse -import os -import sys - -import c2cciutils.audit -import c2cciutils.env - - -def main() -> None: - """ - Run the audit. - """ - parser = argparse.ArgumentParser(description="Run the audit of c2cciutils.") - parser.add_argument("--branch", help="The branch to audit, not defined means autodetect") - parser.add_argument("--check", help="Runs only the specified check") - parser.add_argument("--fix", action="store_true", help="Fix issues") - - args = parser.parse_args() - - full_config = c2cciutils.get_config() - c2cciutils.env.print_environment(full_config) - - config = full_config.get("audit", {}) - success = True - for key, conf in config.items(): - if conf is not False and (args.check is None or args.check == key): - audit = getattr(c2cciutils.audit, key) - print(f"Run audit {key}") - success &= audit({} if conf is True else conf, full_config, args) - if not success and os.environ.get("TEST") != "TRUE": - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/c2cciutils/scripts/main.py b/c2cciutils/scripts/main.py index ef8e1eb49..c2f01de40 100644 --- a/c2cciutils/scripts/main.py +++ b/c2cciutils/scripts/main.py @@ -19,7 +19,6 @@ def main() -> None: parser = argparse.ArgumentParser(description="Some utils of c2cciutils.") parser.add_argument("--get-config", action="store_true", help="display the current config") parser.add_argument("--version", action="store_true", help="display the current version") - parser.add_argument("--ls-files-mime", help="List all the files with the specified mime type") args = parser.parse_args() @@ -30,10 +29,6 @@ def main() -> None: version = pkg_resources.get_distribution("c2cciutils").version print(f"c2cciutils {version}") - if args.ls_files_mime: - for file_name in c2cciutils.get_git_files_mime(args.ls_files_mime): - print(file_name) - if __name__ == "__main__": main() diff --git a/c2cciutils/scripts/pr_checks.py b/c2cciutils/scripts/pr_checks.py deleted file mode 100644 index 85c345e6a..000000000 --- a/c2cciutils/scripts/pr_checks.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 - -""" -The pull request checker main function. -""" - -import argparse -import json -import os -import sys -import traceback - -import requests - -import c2cciutils -import c2cciutils.env -import c2cciutils.pr_checks - - -def main() -> None: - """ - Run the checks. - """ - parser = argparse.ArgumentParser(description="Run the pull request checks of c2cciutils.") - parser.add_argument("--stop", action="store_true", help="stop on first error") - parser.add_argument("--check", help="runs only the specified check") - - args = parser.parse_args() - - full_config = c2cciutils.get_config() - c2cciutils.env.print_environment(full_config) - - github_event = json.loads(os.environ["GITHUB_EVENT"]) - - commits_response = requests.get( - github_event["event"]["pull_request"]["_links"]["commits"]["href"], - timeout=int(os.environ.get("C2CCIUTILS_TIMEOUT", "30")), - headers=c2cciutils.add_authorization_header({}), - ) - c2cciutils.check_response(commits_response) - commits = commits_response.json() - - config = full_config["pr-checks"] - - check_args = { - "args": args, - "full_config": full_config, - "commits": commits, - "github_event": github_event, - } - - success = True - for key, conf in config.items(): - if conf is not False and (args.check is None or args.check == key): - check = getattr(c2cciutils.pr_checks, key) - print(f"::group::Run check {key}") - try: - if not check(config={} if conf is True else conf, **check_args): - success = False - print("::endgroup::") - if args.stop: - sys.exit(1) - print("::error::With error") - else: - print("::endgroup::") - except Exception: # pylint: disable=broad-except - traceback.print_exc() - success = False - print("::endgroup::") - if args.stop: - sys.exit(1) - print("::error::With exception") - if not success: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/config.md b/config.md index 4dfd2a79c..ffc7df633 100644 --- a/config.md +++ b/config.md @@ -5,15 +5,6 @@ _C2C CI utils configuration file_ ## Properties - **`print_versions`**: Refer to _[#/definitions/print_versions](#definitions/print_versions)_. -- **`codespell`**: Refer to _[#/definitions/codespell](#definitions/codespell)_. -- **`audit`** _(object)_: The audit configuration. Default: `{"snyk": true}`. - - **`snyk`**: Refer to _[#/definitions/audit_snyk](#definitions/audit_snyk)_. -- **`pr-checks`** _(object)_: The PR check configuration. Default: `{"commits_messages": true, "commits_spell": true, "pull_request_spell": true, "pull_request_labels": true, "add_issue_link": true}`. - - **`commits_messages`**: Refer to _[#/definitions/pr_checks_commits_messages](#definitions/pr_checks_commits_messages)_. - - **`commits_spell`**: Refer to _[#/definitions/pr_checks_commits_spell](#definitions/pr_checks_commits_spell)_. - - **`pull_request_spell`**: Refer to _[#/definitions/pr_checks_pull_request_spell](#definitions/pr_checks_pull_request_spell)_. - - **`pull_request_labels`**: Refer to _[#/definitions/pr_checks_pull_request_labels](#definitions/pr_checks_pull_request_labels)_. - - **`add_issue_link`**: Refer to _[#/definitions/pr_checks_add_issue_link](#definitions/pr_checks_add_issue_link)_. - **`publish`** _(object)_: The publishing configurations. Default: `{"pypi": {"versions": ["version_tag"], "packages": ""}, "docker": {"images": ""}, "helm": {"versions": ["version_tag"], "folders": ""}}`. - **`docker`**: Refer to _[#/definitions/publish_docker](#definitions/publish_docker)_. - **`pypi`**: Refer to _[#/definitions/publish_pypi](#definitions/publish_pypi)_. @@ -38,53 +29,6 @@ _C2C CI utils configuration file_ ## Definitions -- **`audit_snyk`**: The audit Snyk configuration. - - **One of** - - _object_: The audit Pipfile configuration. - - **`test_arguments`** _(array)_: The Snyk test arguments. Default: `["--all-projects", "--fail-on=all", "--severity-threshold=medium"]`. - - **Items** _(string)_ - - **`monitor_arguments`** _(array)_: The Snyk monitor arguments. Default: `["--all-projects"]`. - - **Items** _(string)_ - - **`fix_arguments`** _(array)_: The Snyk fix arguments. Default: `["--all-projects"]`. - - **Items** _(string)_ - - **`fix_github_create_pull_request_arguments`** _(array)_: The Snyk fix pull request extra arguments. Default: `["--fill", "--label=dependencies"]`. - - **Items** _(string)_ - - **`pip_install_arguments`** _(array)_: The Snyk pip install arguments. Default: `["--user"]`. - - **Items** _(string)_ - - **`pipenv_sync_arguments`** _(array)_: The Snyk pipenv sync arguments. Default: `[]`. - - **Items** _(string)_ - - **`files_no_install`** _(array)_: The list of files to not install. Default: `[]`. - - **Items** _(string)_ - - _boolean_ -- **`codespell`** _(object)_: The codespell check configuration. - - **`internal_dictionaries`** _(array)_: List of argument that will be added to the codespell command. Default: `["clear", "rare", "informal", "code", "names", "en-GB_to_en-US"]`. - - **Items** _(string)_ - - **`arguments`** _(array)_: List of argument that will be added to the codespell command. Default: `["--quiet-level=2", "--check-filenames", "--ignore-words-list=ro"]`. - - **Items** _(string)_ - - **`ignore_re`** _(array)_: List of regular expression that should be ignored. Default: `["(.*/)?poetry\\.lock", "(.*/)?package-lock\\.json"]`. - - **Items** _(string)_ -- **`pr_checks_commits_messages`**: Check the pull request commits messages. - - **One of** - - _object_: The commit message check configuration. - - **`check_fixup`** _(boolean)_: Check that we don't have one fixup commit in the pull request. Default: `true`. - - **`check_squash`** _(boolean)_: Check that we don't have one squash commit in the pull request. Default: `true`. - - **`check_first_capital`** _(boolean)_: Check that the all the commits message starts with a capital letter. Default: `true`. - - **`min_head_length`** _(integer)_: Check that the commits message head is at least this long, use 0 to disable. Default: `5`. - - **`check_no_merge_commits`** _(boolean)_: Check that we don't have merge commits in the pull request. Default: `true`. - - **`check_no_own_revert`** _(boolean)_: Check that we don't have reverted one of our commits in the pull request. Default: `true`. - - _boolean_ -- **`pr_checks_commits_spell`** - - **One of** - - _object_: Configuration used to check the spelling of the commits. - - **`only_head`** _(boolean)_: Default: `true`. - - _boolean_ -- **`pr_checks_pull_request_spell`** - - **One of** - - _object_: Configuration used to check the spelling of the title and body of the pull request. - - **`only_head`** _(boolean)_: Default: `true`. - - _boolean_ -- **`pr_checks_pull_request_labels`** _(boolean)_: According the create changelog configuration. -- **`pr_checks_add_issue_link`** _(boolean)_ - **`publish_docker`**: The configuration used to publish on Docker. - **One of** - _object_: The configuration used to publish on Docker. diff --git a/poetry.lock b/poetry.lock index 3f8afc85b..7eb1ebc35 100644 --- a/poetry.lock +++ b/poetry.lock @@ -319,23 +319,6 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] -[[package]] -name = "codespell" -version = "2.3.0" -description = "Codespell" -optional = true -python-versions = ">=3.8" -files = [ - {file = "codespell-2.3.0-py3-none-any.whl", hash = "sha256:a9c7cef2501c9cfede2110fd6d4e5e62296920efe9abfb84648df866e47f58d1"}, - {file = "codespell-2.3.0.tar.gz", hash = "sha256:360c7d10f75e65f67bad720af7007e1060a5d395670ec11a7ed1fed9dd17471f"}, -] - -[package.extras] -dev = ["Pygments", "build", "chardet", "pre-commit", "pytest", "pytest-cov", "pytest-dependency", "ruff", "tomli", "twine"] -hard-encoding-detection = ["chardet"] -toml = ["tomli"] -types = ["chardet (>=5.1.0)", "mypy", "pytest", "pytest-cov", "pytest-dependency"] - [[package]] name = "colorama" version = "0.4.6" @@ -1593,17 +1576,6 @@ trove-classifiers = ">=2022.6.26" [package.extras] test = ["setuptools (>=60)", "zest.releaser[recommended]"] -[[package]] -name = "python-magic" -version = "0.4.27" -description = "File type identification using libmagic" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b"}, - {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, -] - [[package]] name = "pywin32-ctypes" version = "0.2.3" @@ -2104,28 +2076,6 @@ files = [ [package.dependencies] urllib3 = ">=2" -[[package]] -name = "types-setuptools" -version = "75.1.0.20240917" -description = "Typing stubs for setuptools" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-setuptools-75.1.0.20240917.tar.gz", hash = "sha256:12f12a165e7ed383f31def705e5c0fa1c26215dd466b0af34bd042f7d5331f55"}, - {file = "types_setuptools-75.1.0.20240917-py3-none-any.whl", hash = "sha256:06f78307e68d1bbde6938072c57b81cf8a99bc84bd6dc7e4c5014730b097dc0c"}, -] - -[[package]] -name = "types-toml" -version = "0.10.8.20240310" -description = "Typing stubs for toml" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-toml-0.10.8.20240310.tar.gz", hash = "sha256:3d41501302972436a6b8b239c850b26689657e25281b48ff0ec06345b8830331"}, - {file = "types_toml-0.10.8.20240310-py3-none-any.whl", hash = "sha256:627b47775d25fa29977d9c70dc0cbab3f314f32c8d8d0c012f2ef5de7aaec05d"}, -] - [[package]] name = "typing-extensions" version = "4.12.2" @@ -2187,12 +2137,12 @@ type = ["pytest-mypy"] [extras] audit = [] checks = [] -pr-checks = ["codespell"] -publish = ["google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", "id", "tomlkit", "twine"] +pr-checks = [] +publish = ["google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", "id", "twine"] publish-plugins = [] version = ["multi-repo-automation"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "6f60e9aae0f334cf0f8eb36867117f0b7fdf82059d27a85ff82da99c95ae70f6" +content-hash = "f3d500fde213d74e71e47038aaaa1b066b321f264df0987b99381c6cd65c06a8" diff --git a/pyproject.toml b/pyproject.toml index dff1a3a83..e92363d04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ use_parentheses = true line_length = 110 [tool.mypy] -python_version = "3.9" +python_version = "3.11" ignore_missing_imports = true warn_redundant_casts = true warn_unused_ignores = true @@ -51,8 +51,6 @@ c2cciutils-publish = "c2cciutils.scripts.publish:main" c2cciutils-version = "c2cciutils.scripts.version:main" c2cciutils-clean = "c2cciutils.scripts.clean:main" c2cciutils-checks = "c2cciutils.scripts.env:main" -c2cciutils-pull-request-checks = "c2cciutils.scripts.pr_checks:main" -c2cciutils-audit = "c2cciutils.scripts.audit:main" c2cciutils-google-calendar = "c2cciutils.publish:main_calendar" c2cciutils-k8s-install = "c2cciutils.scripts.k8s.install:main" c2cciutils-k8s-db = "c2cciutils.scripts.k8s.db:main" @@ -66,21 +64,15 @@ c2cciutils-docker-versions-gen = "c2cciutils.scripts.docker_versions_gen:main" [tool.poetry.dependencies] python = ">=3.9,<4.0" -markdown-table = "2020.12.3" -python-magic = "0.4.27" requests = "2.32.3" -Markdown = "3.7" google-api-python-client = { version = "2.148.0", optional = true } google-auth-httplib2 = { version = "0.2.0", optional = true } google-auth-oauthlib = { version = "1.2.1", optional = true } "ruamel.yaml" = "0.18.6" defusedxml = "0.7.1" twine = { version = "5.1.1", optional = true } -codespell = { version = "2.3.0", optional = true } -toml = "0.10.2" debian-inspector = "31.1.0" PyYAML = "6.0.2" -tomlkit = { version = "0.13.2", optional = true } multi-repo-automation = { version="1.3.0", optional = true } security-md = "0.2.3" id = { version="1.4.0", optional = true } @@ -88,22 +80,18 @@ id = { version="1.4.0", optional = true } [tool.poetry.extras] audit = [] checks = [] -publish = ["twine", "google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", "tomlkit", "id"] +publish = ["twine", "google-api-python-client", "google-auth-httplib2", "google-auth-oauthlib", "id"] publish_plugins = [] -pr_checks = ["codespell"] +pr_checks = [] version = ["multi-repo-automation"] [tool.poetry.group.dev.dependencies] -# pylint = "2.15.6" prospector = { version = "1.12.0", extras = ["with-bandit", "with-mypy", "with-pyroma"] } prospector-profile-duplicated = "1.6.0" prospector-profile-utils = "1.9.1" types-requests = "2.32.0.20240914" types-pyyaml = "6.0.12.20240917" -types-setuptools = "75.1.0.20240917" types-markdown = "3.7.0.20240822" -types-toml = "0.10.8.20240310" -typing-extensions = "4.12.2" [build-system] requires = [