Refactor third_party_lint.py

Signed-off-by: Carmen Bianca BAKKER <carmenbianca@fsfe.org>

.github/workflows/third_party_lint.py

@@ -6,100 +6,131 @@
 """Lint 3rd party repositories"""
 
 import argparse
-import json
-import os
 import shutil
 import subprocess
 import sys
+from pathlib import Path
 
 from git import Repo
 
-CLONEDIR = "third-party"
-REPOS = {
-    "fsfe/reuse-example": {},
-    "curl/curl": {},
-    "spdx/license-list-XML": {"ignore-failure": True},
+CLONE_DIR = "third-party"
+DEFAULT_REPOS = {
+    "https://github.com/fsfe/reuse-example": {},
+    "https://github.com/curl/curl": {},
+    "https://github.com/spdx/license-list-XML": {"expect-failure": True},
 }
 
 
-# Fetch arguments
-parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument(
-    "-d",
-    "--debug",
-    action="store_true",
-    help="print DEBUG messages",
-)
-parser.add_argument(
-    "-f",
-    "--force",
-    action="store_true",
-    help="force re-clone of third-party repositories",
-)
-args = parser.parse_args()
-
-
-def main(args_):
+def rm_fr(path):
+    """Force-remove directory."""
+    path = Path(path)
+    if path.exists():
+        shutil.rmtree(path)
+
+
+def lint_repo(repo, force_clone=False, expect_failure=False, json=False):
+    """Meta function to clone and lint a repository, start to finish."""
+    repo_dir = Path(f"{CLONE_DIR}/{repo}")
+
+    if force_clone:
+        rm_fr(repo_dir)
+
+    # Clone repo
+    if not repo_dir.exists():
+        print(f"[INFO] Cloning {repo} to {repo_dir}")
+        repo_git = Repo.clone_from(
+            repo,
+            repo_dir,
+            # Shallow clone.
+            depth=1,
+        )
+    else:
+        print(f"[INFO] Not cloning {repo} as it exists locally.")
+        repo_git = Repo(repo_dir)
+
+    # Get last commit of repo
+    repo_sha = repo_git.head.object.hexsha
+
+    # Lint repo
+    print(f"[INFO] Start linting of {repo} (commit {repo_sha})")
+    lint_result = subprocess.run(
+        ["reuse", "--root", repo_dir, "lint", "--json"],
+        capture_output=True,
+        check=False,
+    )
+    if json:
+        print(lint_result.stdout.decode("utf-8"))
+        print()
+    if lint_result.returncode != 0 and not expect_failure:
+        print(f"[ERROR] Linting {repo} failed unexpectedly")
+    elif lint_result.returncode == 0 and expect_failure:
+        print(f"[ERROR] Linting {repo} succeeded unexpectedly")
+    elif lint_result.returncode != 0 and expect_failure:
+        print(f"[OK] Linting {repo} failed expectedly")
+    elif lint_result.returncode == 0 and not expect_failure:
+        print(f"[OK] Linting {repo} succeeded expectedly")
+    return lint_result
+
+
+def main(args):
     """Main function"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="force re-clone of third-party repositories",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="show json output of lint",
+    )
+    parser.add_argument(
+        "--expect-failure",
+        action="store_true",
+        help="expect the lint to fail",
+    )
+    mutex_group = parser.add_mutually_exclusive_group(required=True)
+    mutex_group.add_argument(
+        "repo",
+        help="link to repository",
+        nargs="?",
+    )
+    mutex_group.add_argument(
+        "--defaults",
+        action="store_true",
+        help="run against some default repositories",
+    )
+    args = parser.parse_args()
+
     total_lint_fails = 0
-    for repo, settings in REPOS.items():
-        repo_dir = f"{CLONEDIR}/{repo}"
-        ignore_failure = settings.get("ignore-failure", False)
-        error = False
-
-        # Delete local directory if it already exists
-        if os.path.isdir(repo_dir) and args_.force:
-            shutil.rmtree(repo_dir)
-
-        # Clone repo
-        if not os.path.isdir(repo_dir):
-            print(f"[INFO] Cloning {repo} to {repo_dir}")
-            repo_git = Repo.clone_from(
-                f"https://github.com/{repo}", repo_dir, filter=["tree:0"]
+    if args.defaults:
+        for repo, settings in DEFAULT_REPOS.items():
+            expect_failure = (
+                settings.get("expect-failure") or args.expect_failure
+            )
+            result = lint_repo(
+                repo,
+                force_clone=args.force,
+                expect_failure=expect_failure,
+                json=args.json,
             )
-        else:
-            print(f"[INFO] Not cloning {repo} as it exists locally.")
-            repo_git = Repo(repo_dir)
-
-        # Get last commit of repo
-        repo_sha = repo_git.head.object.hexsha
-
-        # Lint repo
-        print(f"[INFO] Start linting of {repo} (commit {repo_sha})")
-        lint_ret = subprocess.run(
-            ["reuse", "--root", repo_dir, "lint", "--json"],
-            capture_output=True,
-            check=False,
+            print(expect_failure)
+            print(result.returncode)
+            if result.returncode and not expect_failure:
+                total_lint_fails += 1
+    else:
+        result = lint_repo(
+            args.repo,
+            force_clone=args.force,
+            expect_failure=args.expect_failure,
+            json=args.json,
         )
-
-        # Analyse output
-        # Lint fails unexpectedly
-        if lint_ret.returncode != 0 and not ignore_failure:
-            error = True
-            print(f"[ERROR] Linting {repo} failed unexpectedly")
-        # Lint succeeds unexpectedly
-        elif lint_ret.returncode == 0 and ignore_failure:
-            error = True
-            print(f"[ERROR] Linting {repo} succeeded unexpectedly")
-        # Lint fails expectedly
-        elif lint_ret.returncode != 0 and ignore_failure:
-            print(f"[OK] Linting {repo} failed expectedly")
-        # Lint succeeds expectedly
-        elif lint_ret.returncode == 0 and not ignore_failure:
-            print(f"[OK] Linting {repo} succeeded expectedly")
-
-        # Print lint summary in case of error
-        if args_.debug or error:
-            summary = json.loads(lint_ret.stdout)["summary"]
-            print(json.dumps(summary, indent=2))
-
-        # Increment total error counter
-        if error:
+        if result.returncode and not args.expect_failure:
             total_lint_fails += 1
-
     return total_lint_fails
 
 
 if __name__ == "__main__":
-    args = parser.parse_args()
-    sys.exit(main(args))
+    sys.exit(main(sys.argv[1:]))

.github/workflows/third_party_lint.yaml

@@ -18,6 +18,15 @@ on:
 jobs:
   third-party-lint:
     runs-on: ubuntu-latest
+    strategy:
+      # do not abort the whole test job if one combination in the matrix fails
+      fail-fast: false
+      matrix:
+        repo:
+          [
+            "https://github.com/fsfe/reuse-example",
+            "https://github.com/curl/curl",
+          ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
@@ -30,4 +39,29 @@ jobs:
           poetry install --no-interaction
       # Clone and lint repositories
       - name: Clone and lint repositories
-        run: poetry run python .github/workflows/third_party_lint.py --debug
+        run:
+          poetry run python .github/workflows/third_party_lint.py --json ${{
+          matrix.repo }}
+
+  third-party-lint-expect-failure:
+    runs-on: ubuntu-latest
+    strategy:
+      # do not abort the whole test job if one combination in the matrix fails
+      fail-fast: false
+      matrix:
+        repo: ["https://github.com/spdx/license-list-XML"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --no-interaction
+      # Clone and lint repositories
+      - name: Clone and lint repositories
+        run:
+          poetry run python .github/workflows/third_party_lint.py --json
+          --expect-failure ${{ matrix.repo }}

Makefile

@@ -52,7 +52,7 @@ reuse: dist ## check with self
 
 .PHONY: lint-third-party
 lint-third-party: ## Lint selected third-party repositories to compare with expected output
-	poetry run python3 .github/workflows/third_party_lint.py --debug
+	poetry run python3 .github/workflows/third_party_lint.py --defaults --json
 
 .PHONY: docs
 docs: ## generate Sphinx HTML documentation, including API docs

src/reuse/_main.py

@@ -296,4 +296,4 @@ def main(args: Optional[List[str]] = None, out: IO[str] = sys.stdout) -> int:
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())