-
Notifications
You must be signed in to change notification settings - Fork 149
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Carmen Bianca BAKKER <carmenbianca@fsfe.org>
- Loading branch information
1 parent
c0d9448
commit 9d7da72
Showing
5 changed files
with
152 additions
and
85 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,105 +1,138 @@ | ||
| #!/usr/bin/env python3 | ||
| # SPDX-FileCopyrightText: 2023 DB Systel GmbH | ||
| # SPDX-FileCopyrightText: 2023 Carmen Bianca BAKKER <carmenbianca@fsfe.org> | ||
| # | ||
| # SPDX-License-Identifier: GPL-3.0-or-later | ||
|
|
||
| """Lint 3rd party repositories""" | ||
|
|
||
| import argparse | ||
| import json | ||
| import os | ||
| import shutil | ||
| import subprocess | ||
| import sys | ||
| import tempfile | ||
| from pathlib import Path | ||
|
|
||
| from git import Repo | ||
|
|
||
| CLONEDIR = "third-party" | ||
| REPOS = { | ||
| "fsfe/reuse-example": {}, | ||
| "curl/curl": {}, | ||
| "spdx/license-list-XML": {"ignore-failure": True}, | ||
| CLONE_DIR = Path(tempfile.gettempdir()) / "reuse-third-party" | ||
| DEFAULT_REPOS = { | ||
| "https://github.com/fsfe/reuse-example": {}, | ||
| "https://github.com/curl/curl": {}, | ||
| "https://github.com/spdx/license-list-XML": {"expect-failure": True}, | ||
| } | ||
|
|
||
|
|
||
| # Fetch arguments | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument( | ||
| "-d", | ||
| "--debug", | ||
| action="store_true", | ||
| help="print DEBUG messages", | ||
| ) | ||
| parser.add_argument( | ||
| "-f", | ||
| "--force", | ||
| action="store_true", | ||
| help="force re-clone of third-party repositories", | ||
| ) | ||
| args = parser.parse_args() | ||
|
|
||
|
|
||
| def main(args_): | ||
| def rm_fr(path): | ||
| """Force-remove directory.""" | ||
| path = Path(path) | ||
| if path.exists(): | ||
| shutil.rmtree(path) | ||
|
|
||
|
|
||
| def lint_repo(repo, force_clone=False, expect_failure=False, json=False): | ||
| """Meta function to clone and lint a repository, start to finish.""" | ||
| # The sanitation only works on Linux. If we want to do this 'properly', we | ||
| # should use the pathvalidate dependency. | ||
| repo_dir = Path(f"{CLONE_DIR}/{repo.replace('/', '_')}") | ||
|
|
||
| if force_clone: | ||
| rm_fr(repo_dir) | ||
|
|
||
| # Clone repo | ||
| if not repo_dir.exists(): | ||
| print(f"[INFO] Cloning {repo} to {repo_dir}") | ||
| repo_git = Repo.clone_from( | ||
| repo, | ||
| repo_dir, | ||
| # Shallow clone. | ||
| depth=1, | ||
| ) | ||
| else: | ||
| print(f"[INFO] Not cloning {repo} as it exists locally.") | ||
| repo_git = Repo(repo_dir) | ||
|
|
||
| # Get last commit of repo | ||
| repo_sha = repo_git.head.object.hexsha | ||
|
|
||
| # Lint repo | ||
| print(f"[INFO] Start linting of {repo} (commit {repo_sha})") | ||
| lint_result = subprocess.run( | ||
| ["reuse", "--root", repo_dir, "lint", "--json"], | ||
| capture_output=True, | ||
| check=False, | ||
| ) | ||
| if json: | ||
| print(lint_result.stdout.decode("utf-8")) | ||
| print() | ||
| if lint_result.returncode != 0 and not expect_failure: | ||
| print(f"[ERROR] Linting {repo} failed unexpectedly") | ||
| elif lint_result.returncode == 0 and expect_failure: | ||
| print(f"[ERROR] Linting {repo} succeeded unexpectedly") | ||
| elif lint_result.returncode != 0 and expect_failure: | ||
| print(f"[OK] Linting {repo} failed expectedly") | ||
| elif lint_result.returncode == 0 and not expect_failure: | ||
| print(f"[OK] Linting {repo} succeeded expectedly") | ||
| return lint_result | ||
|
|
||
|
|
||
| def main(args): | ||
| """Main function""" | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument( | ||
| "-f", | ||
| "--force", | ||
| action="store_true", | ||
| help="force re-clone of third-party repositories", | ||
| ) | ||
| parser.add_argument( | ||
| "--json", | ||
| action="store_true", | ||
| help="show json output of lint", | ||
| ) | ||
| parser.add_argument( | ||
| "--expect-failure", | ||
| action="store_true", | ||
| help="expect the lint to fail", | ||
| ) | ||
| mutex_group = parser.add_mutually_exclusive_group(required=True) | ||
| mutex_group.add_argument( | ||
| "repo", | ||
| help="link to repository", | ||
| nargs="?", | ||
| ) | ||
| mutex_group.add_argument( | ||
| "--defaults", | ||
| action="store_true", | ||
| help="run against some default repositories", | ||
| ) | ||
| args = parser.parse_args() | ||
|
|
||
| total_lint_fails = 0 | ||
| for repo, settings in REPOS.items(): | ||
| repo_dir = f"{CLONEDIR}/{repo}" | ||
| ignore_failure = settings.get("ignore-failure", False) | ||
| error = False | ||
|
|
||
| # Delete local directory if it already exists | ||
| if os.path.isdir(repo_dir) and args_.force: | ||
| shutil.rmtree(repo_dir) | ||
|
|
||
| # Clone repo | ||
| if not os.path.isdir(repo_dir): | ||
| print(f"[INFO] Cloning {repo} to {repo_dir}") | ||
| repo_git = Repo.clone_from( | ||
| f"https://github.com/{repo}", repo_dir, filter=["tree:0"] | ||
| if args.defaults: | ||
| for repo, settings in DEFAULT_REPOS.items(): | ||
| expect_failure = ( | ||
| settings.get("expect-failure") or args.expect_failure | ||
| ) | ||
| result = lint_repo( | ||
| repo, | ||
| force_clone=args.force, | ||
| expect_failure=expect_failure, | ||
| json=args.json, | ||
| ) | ||
| else: | ||
| print(f"[INFO] Not cloning {repo} as it exists locally.") | ||
| repo_git = Repo(repo_dir) | ||
|
|
||
| # Get last commit of repo | ||
| repo_sha = repo_git.head.object.hexsha | ||
|
|
||
| # Lint repo | ||
| print(f"[INFO] Start linting of {repo} (commit {repo_sha})") | ||
| lint_ret = subprocess.run( | ||
| ["reuse", "--root", repo_dir, "lint", "--json"], | ||
| capture_output=True, | ||
| check=False, | ||
| if result.returncode and not expect_failure: | ||
| total_lint_fails += 1 | ||
| else: | ||
| result = lint_repo( | ||
| args.repo, | ||
| force_clone=args.force, | ||
| expect_failure=args.expect_failure, | ||
| json=args.json, | ||
| ) | ||
|
|
||
| # Analyse output | ||
| # Lint fails unexpectedly | ||
| if lint_ret.returncode != 0 and not ignore_failure: | ||
| error = True | ||
| print(f"[ERROR] Linting {repo} failed unexpectedly") | ||
| # Lint succeeds unexpectedly | ||
| elif lint_ret.returncode == 0 and ignore_failure: | ||
| error = True | ||
| print(f"[ERROR] Linting {repo} succeeded unexpectedly") | ||
| # Lint fails expectedly | ||
| elif lint_ret.returncode != 0 and ignore_failure: | ||
| print(f"[OK] Linting {repo} failed expectedly") | ||
| # Lint succeeds expectedly | ||
| elif lint_ret.returncode == 0 and not ignore_failure: | ||
| print(f"[OK] Linting {repo} succeeded expectedly") | ||
|
|
||
| # Print lint summary in case of error | ||
| if args_.debug or error: | ||
| summary = json.loads(lint_ret.stdout)["summary"] | ||
| print(json.dumps(summary, indent=2)) | ||
|
|
||
| # Increment total error counter | ||
| if error: | ||
| if result.returncode and not args.expect_failure: | ||
| total_lint_fails += 1 | ||
|
|
||
| return total_lint_fails | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| args = parser.parse_args() | ||
| sys.exit(main(args)) | ||
| sys.exit(main(sys.argv[1:])) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -159,6 +159,3 @@ prof/ | |
| .direnv/ | ||
| result | ||
| .envrc | ||
|
|
||
| # Third-party repo testing (.github/workflows/third_party_lint.py) | ||
| third-party/ | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters