Skip to content

Commit

Permalink
Merge pull request #1999 from camptocamp/rm-audit
Browse files Browse the repository at this point in the history 
Remove audit, cleanup
  • Loading branch information
@sbrunner
sbrunner authored Oct 28, 2024
2 parents 52a415b + 9722160 commit 16db5a0
Show file tree
Hide file tree
Showing 13 changed files with 39 additions and 1,624 deletions.
19 changes: 2 additions & 17 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,6 @@ see the [documentation](https://github.com/camptocamp/c2cciutils/wiki/Publishing
When we create a tag by default with the `changelog` workflow a release is created on GitHub, a changelog is
generated and added to the release.

## Security

The security is managed by the `c2cciutils-audit` command with Snyk, it will audit the dependencies of the project on every
stabilization branches, if possible a pull request is created automatically to update the dependencies.

When we publish a Docker image the generated image is monitored by Snyk, this means that Snyk will search
for all the dependencies and send the list to the Snyk web site to be monitored.
We also do a test of the image and log the result (This will never cause the build to fail).

## Checks

C2C CI utils will no more provide a tool to do a check of the project, this is replaced by `pre-commit`,
Expand Down Expand Up @@ -53,21 +44,16 @@ workflow will delete the workflows older than 500 days.

C2cciutils make easier to have those workflows in a project:

- `audit.yaml`: Audit the stabilization branches of the application against vulnerabilities in the python and node dependency
- `auto-review.yaml`: Auto review the Renovate pull requests
- `backport.yaml`: Trigger the backports (work with labels)
- `clean.yaml`: Clean the Docker images related on a deleted feature branch
- `main.yaml`: Main workflow especially with the c2cciutils-checks command
- `changelog.yaml`: Generate the changelog and create the release on GitHub
- `delete-old-workflows-run.yaml`: Delete the old workflows
- `pr-checks.yaml`: Run the checks on the pull requests

All the provided commands used in the workflow:

- `c2cciutils`: some generic tools.
- `c2cciutils-version`: Create a new version of the project.
- `c2cciutils-checks`: Run the checks on the code (those checks don't need any project dependencies).
- `c2cciutils-audit`: Do the audit, the main difference with checks is that it can change between runs on the same code.
- `c2cciutils-env`: Print some environment information.
- `c2cciutils-publish`: Publish the project.
- `c2cciutils-clean`: Delete Docker images on Docker Hub after corresponding branch have been deleted.

Expand Down Expand Up @@ -128,7 +114,6 @@ You can override the configuration with the file `ci/config.yaml`.
At the base of the configuration you have:

- `version`: Contains some regular expressions to find the versions branches and tags, and to convert them into application versions.
- `audit`: The audit configuration, see `c2cciutils/audit.py` for more information.
- `publish`: The publishing configuration, see `c2cciutils/publish.py` for more information.

Many actions can be disabled by setting the corresponding configuration part to `False`.
Expand Down Expand Up @@ -239,7 +224,7 @@ To make it working in the `Dockerfile` you should have in the `poetry` stage:

```Dockerfile
ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev
RUN poetry export --extras=checks --extras=publish --extras=audit --output=requirements.txt \
RUN poetry export --extras=checks --extras=publish --output=requirements.txt \
&& poetry export --with=dev --output=requirements-dev.txt
```

Expand Down
212 changes: 0 additions & 212 deletions c2cciutils/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from re import Match, Pattern
from typing import Any, Optional, TypedDict, cast

import magic
import requests
import ruamel.yaml

Expand Down Expand Up @@ -126,8 +125,6 @@ def get_config() -> c2cciutils.configuration.Configuration:

default_config = {
"publish": publish_config,
"pr-checks": c2cciutils.configuration.PULL_REQUEST_CHECKS_DEFAULT,
"audit": c2cciutils.configuration.AUDIT_DEFAULT,
}
merge(default_config, config)

Expand Down Expand Up @@ -411,160 +408,6 @@ def graphql(query_file: str, variables: dict[str, Any], default: Any = None) ->
return cast(dict[str, Any], json_response["data"])


def get_git_files_mime(
mime_type: Optional[list[str]] = None,
extensions: Optional[list[str]] = None,
ignore_patterns_re: Optional[list[str]] = None,
) -> list[str]:
"""
Get list of paths from git with all the files that have the specified mime type.
Arguments:
mime_type: The considered MIME type
extensions: The considered extensions
ignore_patterns_re: A list of regular expressions of files that we should ignore
"""
if mime_type is None:
mime_type = ["text/x-python", "text/x-script.python"]
if extensions is None:
extensions = [".py"]
ignore_patterns_compiled = [re.compile(p) for p in ignore_patterns_re or []]
result = []

for filename in subprocess.check_output(["git", "ls-files"]).decode().strip().split("\n"):
if os.path.isfile(filename) and (
os.path.splitext(filename)[1] in extensions or magic.from_file(filename, mime=True) in mime_type
):
accept = True
for pattern in ignore_patterns_compiled:
if pattern.search(filename):
accept = False
break
if accept:
result.append(filename)
return result


def get_branch(branch: Optional[str], master_branch: str = "master") -> str:
"""
Get the branch name.
Arguments:
branch: The forced to use branch name
master_branch: The master branch name, can be used as default value
Return the branch name
"""
if branch is not None:
return branch
try:
branch = (
subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"], check=True, stdout=subprocess.PIPE)
.stdout.decode()
.strip()
)
except subprocess.CalledProcessError as exception:
print(f"Error getting branch: {exception}")
branch = "HEAD"

if branch == "HEAD":
branch = os.environ.get("GITHUB_HEAD_REF", master_branch)
assert branch is not None
return branch


def get_based_on_master(
repo: list[str],
override_current_branch: Optional[str],
master_branch: str,
config: c2cciutils.configuration.Configuration,
) -> bool:
"""
Check that we are not on a release branch (to avoid errors in versions check).
This function will check the last 20 commits in current branch,
and for each other branch (max 50) check if any commit in last 10 commits is the current one.
Arguments:
repo: The repository [<organization>, <name>]
override_current_branch: The branch to use instead of the current one
master_branch: The master branch name
config: The full configuration
"""
if os.environ.get("GITHUB_REF", "").startswith("refs/tags/"):
# The tags are never consider as based on master
return False
current_branch = get_branch(override_current_branch, master_branch)
if current_branch == master_branch:
return True
branches_re = compile_re(config["version"].get("branch_to_version_re", []))
if does_match(current_branch, branches_re):
return False
if os.environ.get("GITHUB_BASE_REF"):
return os.environ.get("GITHUB_BASE_REF") == master_branch
commits_repository_json = graphql(
"commits.graphql", {"name": repo[1], "owner": repo[0], "branch": current_branch}
).get("repository", {})
commits_json = (
commits_repository_json.get("ref", {}).get("target", {}).get("history", {}).get("nodes", [])
if commits_repository_json.get("ref")
else []
)
branches_json = [
branch
for branch in (
graphql("branches.graphql", {"name": repo[1], "owner": repo[0]})["repository"]["refs"]["nodes"]
)
if branch["name"] != current_branch and does_match(branch["name"], branches_re)
]
based_branch = master_branch
found = False
for commit in commits_json:
for branch in branches_json:
commits = [
branch_commit
for branch_commit in branch["target"]["history"]["nodes"]
if commit["oid"] == branch_commit["oid"]
]
if commits:
based_branch = branch["name"]
found = True
break
if found:
break
return based_branch == master_branch


def get_codespell_command(config: c2cciutils.configuration.Configuration, fix: bool = False) -> list[str]:
"""
Get the codespell command.
Arguments:
config: The full configuration
fix: If we should fix the errors
"""
codespell_config = config.get("codespell", {})
codespell_config = codespell_config if isinstance(codespell_config, dict) else {}
command = ["codespell"]
if fix:
command.append("--write-changes")
for spell_ignore_file in (
".github/spell-ignore-words.txt",
"spell-ignore-words.txt",
".spell-ignore-words.txt",
):
if os.path.exists(spell_ignore_file):
command.append(f"--ignore-words={spell_ignore_file}")
break
dictionaries = codespell_config.get(
"internal_dictionaries", c2cciutils.configuration.CODESPELL_DICTIONARIES_DEFAULT
)
if dictionaries:
command.append("--builtin=" + ",".join(dictionaries))
command += codespell_config.get("arguments", c2cciutils.configuration.CODESPELL_ARGUMENTS_DEFAULT)
return command


def snyk_exec() -> tuple[str, dict[str, str]]:
"""Get the Snyk cli executable path."""
if not os.path.exists(os.path.join(os.path.dirname(__file__), "node_modules")):
Expand All @@ -580,58 +423,3 @@ def snyk_exec() -> tuple[str, dict[str, str]]:
subprocess.run(["snyk", "config", "set", f"org={env['SNYK_ORG']}"], check=True, env=env)

return os.path.join(os.path.dirname(os.path.abspath(__file__)), "node_modules/snyk/bin/snyk"), env


def create_pull_request_if_needed(
current_branch: str,
new_branch: str,
commit_message: str,
pull_request_extra_arguments: Optional[list[str]] = None,
) -> bool:
"""
Create a pull request if there are changes.
"""
if pull_request_extra_arguments is None:
pull_request_extra_arguments = ["--fill"]

diff_proc = subprocess.run(["git", "diff", "--quiet"]) # pylint: disable=subprocess-run-check
if diff_proc.returncode != 0:
print("::group::Diff")
sys.stdout.flush()
sys.stderr.flush()
subprocess.run(["git", "diff"], check=True)
print("::endgroup::")

git_hash = subprocess.run(
["git", "rev-parse", "HEAD"], check=True, stdout=subprocess.PIPE, encoding="utf-8"
).stdout.strip()
subprocess.run(["git", "checkout", "-b", new_branch], check=True)
subprocess.run(["git", "add", "--all"], check=True)
subprocess.run(["git", "commit", f"--message={commit_message}"], check=True)
if os.environ.get("TEST") != "TRUE":
subprocess.run(
["git", "push", "--force", "origin", new_branch],
check=True,
)
env = os.environ.copy()
if "GH_TOKEN" not in env:
if "GITHUB_TOKEN" in env:
env["GH_TOKEN"] = env["GITHUB_TOKEN"]
else:
env["GH_TOKEN"] = str(c2cciutils.gopass("gs/ci/github/token/gopass"))
subprocess.run(
[
"gh",
"pr",
"create",
f"--base={current_branch}",
*pull_request_extra_arguments,
],
check=True,
env=env,
)
else:
subprocess.run(["git", "reset", "--hard"], check=True)
subprocess.run(["git", "checkout", git_hash], check=True)

return diff_proc.returncode != 0
Loading

0 comments on commit 16db5a0

Please sign in to comment.