From fc06a4c55ecf2d9892b8eb1c601e626d7e8490f0 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 11:30:55 +0200 Subject: [PATCH 01/25] Add mypy dev dependency Signed-off-by: Carmen Bianca BAKKER --- poetry.lock | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 780e7088b..85f51d272 100644 --- a/poetry.lock +++ b/poetry.lock @@ -356,6 +356,25 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "mypy" +version = "1.3.0" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -803,7 +822,7 @@ testing = ["big-o", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3b18cf7227bb5966c1ac57abf8c77d011ab51087eb644a82f5cb39535fabf192" +content-hash = "1f89d525e12adc31ca438130ab341b7a400bf9231ed609db94e54191f0cb26b9" [metadata.files] alabaster = [ @@ -1066,6 +1085,34 @@ mccabe = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +mypy = [ + {file = "mypy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eb485cea53f4f5284e5baf92902cd0088b24984f4209e25981cc359d64448d"}, + {file = "mypy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c99c3ecf223cf2952638da9cd82793d8f3c0c5fa8b6ae2b2d9ed1e1ff51ba85"}, + {file = "mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:550a8b3a19bb6589679a7c3c31f64312e7ff482a816c96e0cecec9ad3a7564dd"}, + {file = "mypy-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cbc07246253b9e3d7d74c9ff948cd0fd7a71afcc2b77c7f0a59c26e9395cb152"}, + {file = "mypy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a22435632710a4fcf8acf86cbd0d69f68ac389a3892cb23fbad176d1cddaf228"}, + {file = "mypy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e33bb8b2613614a33dff70565f4c803f889ebd2f859466e42b46e1df76018dd"}, + {file = "mypy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d23370d2a6b7a71dc65d1266f9a34e4cde9e8e21511322415db4b26f46f6b8c"}, + {file = "mypy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658fe7b674769a0770d4b26cb4d6f005e88a442fe82446f020be8e5f5efb2fae"}, + {file = "mypy-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d29e324cdda61daaec2336c42512e59c7c375340bd202efa1fe0f7b8f8ca"}, + {file = "mypy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d0b6c62206e04061e27009481cb0ec966f7d6172b5b936f3ead3d74f29fe3dcf"}, + {file = "mypy-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:76ec771e2342f1b558c36d49900dfe81d140361dd0d2df6cd71b3db1be155409"}, + {file = "mypy-1.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc95f8386314272bbc817026f8ce8f4f0d2ef7ae44f947c4664efac9adec929"}, + {file = "mypy-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:faff86aa10c1aa4a10e1a301de160f3d8fc8703b88c7e98de46b531ff1276a9a"}, + {file = "mypy-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8c5979d0deb27e0f4479bee18ea0f83732a893e81b78e62e2dda3e7e518c92ee"}, + {file = "mypy-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c5d2cc54175bab47011b09688b418db71403aefad07cbcd62d44010543fc143f"}, + {file = "mypy-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:87df44954c31d86df96c8bd6e80dfcd773473e877ac6176a8e29898bfb3501cb"}, + {file = "mypy-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473117e310febe632ddf10e745a355714e771ffe534f06db40702775056614c4"}, + {file = "mypy-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:74bc9b6e0e79808bf8678d7678b2ae3736ea72d56eede3820bd3849823e7f305"}, + {file = "mypy-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:44797d031a41516fcf5cbfa652265bb994e53e51994c1bd649ffcd0c3a7eccbf"}, + {file = "mypy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ddae0f39ca146972ff6bb4399f3b2943884a774b8771ea0a8f50e971f5ea5ba8"}, + {file = "mypy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c4c42c60a8103ead4c1c060ac3cdd3ff01e18fddce6f1016e08939647a0e703"}, + {file = "mypy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86c2c6852f62f8f2b24cb7a613ebe8e0c7dc1402c61d36a609174f63e0ff017"}, + {file = "mypy-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f9dca1e257d4cc129517779226753dbefb4f2266c4eaad610fc15c6a7e14283e"}, + {file = "mypy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:95d8d31a7713510685b05fbb18d6ac287a56c8f6554d88c19e73f724a445448a"}, + {file = "mypy-1.3.0-py3-none-any.whl", hash = "sha256:a8763e72d5d9574d45ce5881962bc8e9046bf7b375b0abf031f3e6811732a897"}, + {file = "mypy-1.3.0.tar.gz", hash = "sha256:e1f4d16e296f5135624b34e8fb741eb0eadedca90862405b1f1fde2040b9bd11"}, +] mypy-extensions = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, diff --git a/pyproject.toml b/pyproject.toml index bf1ac12be..c7b1be581 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ pre-commit = "^2.9.0" pylint = "^2.12.2" pytest = ">=6.0.0" pytest-cov = ">=2.10.0" +mypy = "^1.0" [tool.poetry.scripts] reuse = 'reuse._main:main' From 08f73baa97bdc9fae7f3918ac8bc96b87392850e Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 11:44:06 +0200 Subject: [PATCH 02/25] Add mypy to pre-commit Signed-off-by: Carmen Bianca BAKKER --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7f5fe9d5..cfb914c3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,10 @@ repos: - id: isort name: isort (pyi) types: [pyi] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.2.0 + hooks: + - id: mypy - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.7.1 hooks: From 2927f7094a03624beb2efacefd4931d77c069e15 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 13:53:09 +0200 Subject: [PATCH 03/25] Mypy: ignore missing imports for certain modules Signed-off-by: Carmen Bianca BAKKER --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c7b1be581..fc65a0f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,3 +95,12 @@ line_length = 80 [tool.pytest.ini_options] addopts = "--doctest-modules" + +[[tool.mypy.overrides]] +module = [ + "binaryornot.check", + "boolean.boolean", + "license_expression", + "pkg_resources", +] +ignore_missing_imports = true From 93f86c3752f0cf40eebbc2219040fecdd5735928 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 16:47:22 +0200 Subject: [PATCH 04/25] Mypy: disallow untyped and incomplete function defs Signed-off-by: Carmen Bianca BAKKER --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fc65a0f94..f8608a8af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,11 @@ line_length = 80 [tool.pytest.ini_options] addopts = "--doctest-modules" +[[tool.mypy.overrides]] +module = "reuse.*" +disallow_untyped_defs = true +disallow_incomplete_defs = true + [[tool.mypy.overrides]] module = [ "binaryornot.check", From f908109fe3dffcfd2530d3e94a196718506255c2 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 02:36:40 +0200 Subject: [PATCH 05/25] Exclude small scripts from mypy type checking Signed-off-by: Carmen Bianca BAKKER --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f8608a8af..4aa302397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,12 @@ line_length = 80 [tool.pytest.ini_options] addopts = "--doctest-modules" +[tool.mypy] +exclude = [ + '^_build\.py$', + '^conf\.py$', +] + [[tool.mypy.overrides]] module = "reuse.*" disallow_untyped_defs = true From 6404d6a7ebd0b913cc77053252664999d122e42a Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 02:40:48 +0200 Subject: [PATCH 06/25] Set default files to type check Signed-off-by: Carmen Bianca BAKKER --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4aa302397..b004a3815 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,10 @@ line_length = 80 addopts = "--doctest-modules" [tool.mypy] +files = [ + "src/reuse/**.py", + "tests/**.py", +] exclude = [ '^_build\.py$', '^conf\.py$', From b5d05eb88bf178165c3c57247d3badf8582f74ff Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 13:48:46 +0200 Subject: [PATCH 07/25] Add mypy instruction to CONTRIBUTING Signed-off-by: Carmen Bianca BAKKER --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3485d82cd..8d37dcdc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,6 +51,7 @@ Next, you'll find the following commands handy: - `poetry run reuse` - `poetry run pytest` - `poetry run pylint src` +- `poetry run mypy` - `make docs` ## Development conventions From c918059472fb30c07a340982284a54ce1463f69b Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 15:04:45 +0200 Subject: [PATCH 08/25] Add py.typed marker Signed-off-by: Carmen Bianca BAKKER --- src/reuse/py.typed | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/reuse/py.typed diff --git a/src/reuse/py.typed b/src/reuse/py.typed new file mode 100644 index 000000000..19800f554 --- /dev/null +++ b/src/reuse/py.typed @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2023 Carmen Bianca BAKKER +# +# SPDX-License-Identifier: GPL-3.0-or-later From 394663a73f09ebb130192a7ab42057239827c0b7 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 13:46:48 +0200 Subject: [PATCH 09/25] Add GitHub job for mypy Signed-off-by: Carmen Bianca BAKKER --- .github/workflows/test.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 377d144b2..870f8975b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -78,6 +78,22 @@ jobs: poetry run isort --check src/ tests/ poetry run black . + mypy: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: | + pip install poetry + poetry install --no-interaction + - name: Test typing with mypy + run: | + poetry run mypy + prettier: runs-on: ubuntu-20.04 container: node:latest From a3f092bc03fd21f92099637879f39bd609cde4e4 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 12:33:28 +0200 Subject: [PATCH 10/25] Make _util mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/_util.py | 98 +++++++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/src/reuse/_util.py b/src/reuse/_util.py index 7b4ce30dd..bd7b1062c 100644 --- a/src/reuse/_util.py +++ b/src/reuse/_util.py @@ -20,13 +20,14 @@ import subprocess import sys from argparse import ArgumentTypeError +from collections import Counter from difflib import SequenceMatcher from gettext import gettext as _ from hashlib import sha1 from itertools import chain from os import PathLike from pathlib import Path -from typing import BinaryIO, Dict, Iterator, List, Optional, Set +from typing import IO, Any, BinaryIO, Dict, Iterator, List, Optional, Set, Union from boolean.boolean import Expression, ParseError from debian.copyright import Copyright @@ -36,6 +37,9 @@ from ._licenses import ALL_NON_DEPRECATED_MAP from .comment import _all_style_classes +# TODO: When removing Python 3.8 support, use PathLike[str] +StrPath = Union[str, PathLike] + GIT_EXE = shutil.which("git") HG_EXE = shutil.which("hg") @@ -139,18 +143,21 @@ def setup_logging(level: int = logging.WARNING) -> None: def execute_command( - command: List[str], logger: logging.Logger, cwd: PathLike = None, **kwargs + command: List[str], + logger: logging.Logger, + cwd: Optional[StrPath] = None, + **kwargs: Any, ) -> subprocess.CompletedProcess: """Run the given command with subprocess.run. Forward kwargs. Silence output into a pipe unless kwargs override it. """ logger.debug("running '%s'", " ".join(command)) - stdout = kwargs.get("stdout", subprocess.PIPE) - stderr = kwargs.get("stderr", subprocess.PIPE) + stdout: Union[None, int, IO[Any]] = kwargs.get("stdout", subprocess.PIPE) + stderr: Union[None, int, IO[Any]] = kwargs.get("stderr", subprocess.PIPE) return subprocess.run( - map(str, command), + list(map(str, command)), stdout=stdout, stderr=stderr, check=False, @@ -159,7 +166,7 @@ def execute_command( ) -def find_licenses_directory(root: PathLike) -> Optional[Path]: +def find_licenses_directory(root: StrPath) -> Path: """Find the licenses directory from CWD or *root*. In the following order: - LICENSES/ in *root*. @@ -175,26 +182,30 @@ def find_licenses_directory(root: PathLike) -> Optional[Path]: licenses_path = cwd / "LICENSES" if root: - licenses_path = root / "LICENSES" + licenses_path = Path(root) / "LICENSES" elif cwd.name == "LICENSES": licenses_path = cwd return licenses_path -def decoded_text_from_binary(binary_file: BinaryIO, size: int = None) -> str: +def decoded_text_from_binary( + binary_file: BinaryIO, size: Optional[int] = None +) -> str: """Given a binary file object, detect its encoding and return its contents as a decoded string. Do not throw any errors if the encoding contains errors: Just replace the false characters. If *size* is specified, only read so many bytes. """ + if size is None: + size = -1 rawdata = binary_file.read(size) result = rawdata.decode("utf-8", errors="replace") return result.replace("\r\n", "\n") -def _determine_license_path(path: PathLike) -> Path: +def _determine_license_path(path: StrPath) -> Path: """Given a path FILE, return FILE.license if it exists, otherwise return FILE. """ @@ -204,7 +215,7 @@ def _determine_license_path(path: PathLike) -> Path: return license_path -def _determine_license_suffix_path(path: PathLike) -> Path: +def _determine_license_suffix_path(path: StrPath) -> Path: """Given a path FILE or FILE.license, return FILE.license.""" path = Path(path) if path.suffix == ".license": @@ -212,9 +223,7 @@ def _determine_license_suffix_path(path: PathLike) -> Path: return Path(f"{path}.license") -def _copyright_from_dep5( - path: PathLike, dep5_copyright: Copyright -) -> ReuseInfo: +def _copyright_from_dep5(path: StrPath, dep5_copyright: Copyright) -> ReuseInfo: """Find the reuse information of *path* in the dep5 Copyright object.""" result = dep5_copyright.find_files_paragraph(Path(path).as_posix()) @@ -222,8 +231,12 @@ def _copyright_from_dep5( return ReuseInfo() return ReuseInfo( - spdx_expressions=set(map(_LICENSING.parse, [result.license.synopsis])), - copyright_lines=set(map(str.strip, result.copyright.splitlines())), + spdx_expressions=set( + map(_LICENSING.parse, [result.license.synopsis]) # type: ignore + ), + copyright_lines=set( + map(str.strip, result.copyright.splitlines()) # type:ignore + ), source_type=SourceType.DEP5_FILE, ) @@ -254,6 +267,8 @@ def merge_copyright_lines(copyright_lines: Set[str]) -> Set[str]: into a range. If a same statement uses multiple prefixes, use only the most frequent one. """ + # pylint: disable=too-many-locals + # TODO: Rewrite this function. It's a bit of a mess. copyright_in = [] for line in copyright_lines: for pattern in _COPYRIGHT_PATTERNS: @@ -269,35 +284,37 @@ def merge_copyright_lines(copyright_lines: Set[str]) -> Set[str]: } ) - copyright_out = [] - for statement in {item["statement"] for item in copyright_in}: + copyright_out = set() + for line_info in copyright_in: + statement = str(line_info["statement"]) copyright_list = [ item for item in copyright_in if item["statement"] == statement ] - prefixes = [item["prefix"] for item in copyright_list] # Get the style of the most common prefix - prefix = max(set(prefixes), key=prefixes.count) + prefix = str( + Counter([item["prefix"] for item in copyright_list]).most_common(1)[ + 0 + ][0] + ) style = "spdx" - # pylint: disable=consider-using-dict-items - for sty in _COPYRIGHT_STYLES: - if prefix == _COPYRIGHT_STYLES[sty]: - style = sty + for key, value in _COPYRIGHT_STYLES.items(): + if prefix == value: + style = key break # get year range if any - years = [] + years: List[str] = [] for copy in copyright_list: years += copy["year"] - if len(years) == 0: - year = None - elif min(years) == max(years): + year: Optional[str] = None + if min(years) == max(years): year = min(years) else: year = f"{min(years)} - {max(years)}" - copyright_out.append(make_copyright_line(statement, year, style)) + copyright_out.add(make_copyright_line(statement, year, style)) return copyright_out @@ -308,7 +325,7 @@ def extract_reuse_info(text: str) -> ReuseInfo: :raises ParseError: if an SPDX expression could not be parsed """ text = filter_ignore_block(text) - spdx_tags: Dict[str, str] = {} + spdx_tags: Dict[str, Set[str]] = {} for tag, pattern in _SPDX_TAGS.items(): spdx_tags[tag] = set(find_spdx_tag(text, pattern)) # License expressions and copyright matches are special cases. @@ -334,7 +351,7 @@ def extract_reuse_info(text: str) -> ReuseInfo: return ReuseInfo( spdx_expressions=expressions, copyright_lines=copyright_matches, - **spdx_tags, + **spdx_tags, # type: ignore ) @@ -419,7 +436,7 @@ def make_copyright_line( return f"{copyright_prefix} {statement}" -def _checksum(path: PathLike) -> str: +def _checksum(path: StrPath) -> str: path = Path(path) file_sha1 = sha1() @@ -433,7 +450,12 @@ def _checksum(path: PathLike) -> str: class PathType: """Factory for creating Paths""" - def __init__(self, mode="r", force_file=False, force_directory=False): + def __init__( + self, + mode: str = "r", + force_file: bool = False, + force_directory: bool = False, + ): if mode in ("r", "r+", "w"): self._mode = mode else: @@ -445,7 +467,7 @@ def __init__(self, mode="r", force_file=False, force_directory=False): "'force_file' and 'force_directory' cannot both be True" ) - def _check_read(self, path): + def _check_read(self, path: Path) -> None: if path.exists() and os.access(path, os.R_OK): if self._force_file and not path.is_file(): raise ArgumentTypeError(_("'{}' is not a file").format(path)) @@ -456,7 +478,7 @@ def _check_read(self, path): return raise ArgumentTypeError(_("can't open '{}'").format(path)) - def _check_write(self, path): + def _check_write(self, path: Path) -> None: if path.is_dir(): raise ArgumentTypeError( _("can't write to directory '{}'").format(path) @@ -467,7 +489,7 @@ def _check_write(self, path): return raise ArgumentTypeError(_("can't write to '{}'").format(path)) - def __call__(self, string): + def __call__(self, string: str) -> Path: path = Path(string) try: @@ -494,7 +516,7 @@ def spdx_identifier(text: str) -> Expression: def similar_spdx_identifiers(identifier: str) -> List[str]: """Given an incorrect SPDX identifier, return a list of similar ones.""" - suggestions = [] + suggestions: List[str] = [] if identifier in ALL_NON_DEPRECATED_MAP: return suggestions @@ -509,7 +531,9 @@ def similar_spdx_identifiers(identifier: str) -> List[str]: return suggestions -def print_incorrect_spdx_identifier(identifier: str, out=sys.stdout) -> None: +def print_incorrect_spdx_identifier( + identifier: str, out: IO[str] = sys.stdout +) -> None: """Print out that *identifier* is not valid, and follow up with some suggestions. """ From a21c34f8db2d5cfcf43c125e28f995c8f9debe0f Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 11:40:30 +0200 Subject: [PATCH 11/25] Make comment mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/comment.py | 69 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/src/reuse/comment.py b/src/reuse/comment.py index 1bfeaf738..242987abc 100644 --- a/src/reuse/comment.py +++ b/src/reuse/comment.py @@ -23,7 +23,7 @@ import logging import operator from textwrap import dedent -from typing import List, NamedTuple +from typing import List, NamedTuple, Type _LOGGER = logging.getLogger(__name__) @@ -49,6 +49,7 @@ class MultiLineSegments(NamedTuple): class CommentStyle: """Base class for comment style.""" + SHORTHAND = "" SINGLE_LINE = "" INDENT_AFTER_SINGLE = "" # (start, middle, end) @@ -57,7 +58,7 @@ class CommentStyle: INDENT_BEFORE_MIDDLE = "" INDENT_AFTER_MIDDLE = "" INDENT_BEFORE_END = "" - SHEBANGS = [] + SHEBANGS: List[str] = [] @classmethod def can_handle_single(cls) -> bool: @@ -142,7 +143,7 @@ def _parse_comment_single(cls, text: str) -> str: """ if not cls.can_handle_single(): raise CommentParseError(f"{cls} cannot parse single-line comments") - result = [] + result_lines = [] for line in text.splitlines(): if not line.startswith(cls.SINGLE_LINE): @@ -150,8 +151,8 @@ def _parse_comment_single(cls, text: str) -> str: f"'{line}' does not start with a comment marker" ) line = line.lstrip(cls.SINGLE_LINE) - result.append(line) - result = "\n".join(result) + result_lines.append(line) + result = "\n".join(result_lines) return dedent(result) @classmethod @@ -181,14 +182,14 @@ def _parse_comment_multi(cls, text: str) -> str: if not cls.can_handle_multi(): raise CommentParseError(f"{cls} cannot parse multi-line comments") - result = [] + result_lines = [] try: first, *lines, last = text.splitlines() last_is_first = False except ValueError: first = text lines = [] - last = None # Set this later. + last = "" # Set this later. last_is_first = True if not first.startswith(cls.MULTI_LINE.start): @@ -200,7 +201,7 @@ def _parse_comment_multi(cls, text: str) -> str: for line in lines: line = cls._remove_middle_marker(line) - result.append(line) + result_lines.append(line) if last_is_first: last = first @@ -213,7 +214,7 @@ def _parse_comment_multi(cls, text: str) -> str: last = last.rstrip() last = cls._remove_middle_marker(last) - result = "\n".join(result) + result = "\n".join(result_lines) result = dedent(result) return "\n".join(item for item in (first, result, last) if item) @@ -258,7 +259,7 @@ def comment_at_first_character(cls, text: str) -> str: class AppleScriptCommentStyle(CommentStyle): """AppleScript comment style.""" - _shorthand = "applescript" + SHORTHAND = "applescript" SINGLE_LINE = "--" INDENT_AFTER_SINGLE = " " @@ -268,7 +269,7 @@ class AppleScriptCommentStyle(CommentStyle): class AspxCommentStyle(CommentStyle): """ASPX comment style.""" - _shorthand = "aspx" + SHORTHAND = "aspx" MULTI_LINE = MultiLineSegments("<%--", "", "--%>") @@ -276,7 +277,7 @@ class AspxCommentStyle(CommentStyle): class BatchFileCommentStyle(CommentStyle): """Windows batch file comment style.""" - _shorthand = "bat" + SHORTHAND = "bat" SINGLE_LINE = "REM" INDENT_AFTER_SINGLE = " " @@ -285,7 +286,7 @@ class BatchFileCommentStyle(CommentStyle): class BibTexCommentStyle(CommentStyle): """BibTex comment style.""" - _shorthand = "bibtex" + SHORTHAND = "bibtex" MULTI_LINE = MultiLineSegments("@Comment{", "", "}") @@ -293,7 +294,7 @@ class BibTexCommentStyle(CommentStyle): class CCommentStyle(CommentStyle): """C comment style.""" - _shorthand = "c" + SHORTHAND = "c" SINGLE_LINE = "//" INDENT_AFTER_SINGLE = " " @@ -310,7 +311,7 @@ class CCommentStyle(CommentStyle): class CssCommentStyle(CommentStyle): """CSS comment style.""" - _shorthand = "css" + SHORTHAND = "css" MULTI_LINE = MultiLineSegments("/*", "*", "*/") INDENT_BEFORE_MIDDLE = " " @@ -337,7 +338,7 @@ def comment_at_first_character(cls, text: str) -> str: class FortranCommentStyle(CommentStyle): """Fortran comment style.""" - _shorthand = "f" + SHORTHAND = "f" SINGLE_LINE = "c" INDENT_AFTER_SINGLE = " " @@ -346,7 +347,7 @@ class FortranCommentStyle(CommentStyle): class FtlCommentStyle(CommentStyle): """FreeMarker Template Language comment style.""" - _shorthand = "ftl" + SHORTHAND = "ftl" MULTI_LINE = MultiLineSegments("<#--", "", "-->") @@ -354,7 +355,7 @@ class FtlCommentStyle(CommentStyle): class HandlebarsCommentStyle(CommentStyle): """Handlebars comment style.""" - _shorthand = "handlebars" + SHORTHAND = "handlebars" MULTI_LINE = MultiLineSegments("{{!--", "", "--}}") @@ -362,7 +363,7 @@ class HandlebarsCommentStyle(CommentStyle): class HaskellCommentStyle(CommentStyle): """Haskell comment style.""" - _shorthand = "haskell" + SHORTHAND = "haskell" SINGLE_LINE = "--" INDENT_AFTER_SINGLE = " " @@ -371,7 +372,7 @@ class HaskellCommentStyle(CommentStyle): class HtmlCommentStyle(CommentStyle): """HTML comment style.""" - _shorthand = "html" + SHORTHAND = "html" MULTI_LINE = MultiLineSegments("") SHEBANGS = [" List[CommentStyle]: +def _all_style_classes() -> List[Type[CommentStyle]]: """Return a list of all defined style classes, excluding the base class.""" result = [] for key, value in globals().items(): @@ -779,11 +780,9 @@ def _all_style_classes() -> List[CommentStyle]: return sorted(result, key=operator.attrgetter("__name__")) -# pylint: disable=protected-access - _result = _all_style_classes() _result.remove(EmptyCommentStyle) _result.remove(UncommentableCommentStyle) #: A map of human-friendly names against style classes. -NAME_STYLE_MAP = {style._shorthand: style for style in _result} +NAME_STYLE_MAP = {style.SHORTHAND: style for style in _result} From 21906d74d1f86ef877fb05eafc475b7ed85bd7c8 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 13:27:11 +0200 Subject: [PATCH 12/25] Make vcs mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/project.py | 3 ++- src/reuse/vcs.py | 54 ++++++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/reuse/project.py b/src/reuse/project.py index cc0b346ce..e68147dcb 100644 --- a/src/reuse/project.py +++ b/src/reuse/project.py @@ -232,10 +232,11 @@ def reuse_info_of(self, path: PathLike) -> ReuseInfo: source_path=source_path, source_type=source_type ) - def relative_from_root(self, path: Path) -> Path: + def relative_from_root(self, path: PathLike) -> Path: """If the project root is /tmp/project, and *path* is /tmp/project/src/file, then return src/file. """ + path = Path(path) try: return path.relative_to(self.root) except ValueError: diff --git a/src/reuse/vcs.py b/src/reuse/vcs.py index 5b46e04b4..0e8eea720 100644 --- a/src/reuse/vcs.py +++ b/src/reuse/vcs.py @@ -6,14 +6,18 @@ """This module deals with version control systems.""" +from __future__ import annotations + import logging import os from abc import ABC, abstractmethod -from os import PathLike from pathlib import Path -from typing import Optional, Set +from typing import TYPE_CHECKING, Optional, Set + +from ._util import GIT_EXE, HG_EXE, StrPath, execute_command -from ._util import GIT_EXE, HG_EXE, execute_command +if TYPE_CHECKING: + from .project import Project _LOGGER = logging.getLogger(__name__) @@ -22,16 +26,16 @@ class VCSStrategy(ABC): """Strategy pattern for version control systems.""" @abstractmethod - def __init__(self, project: "Project"): + def __init__(self, project: Project): self.project = project @abstractmethod - def is_ignored(self, path: PathLike) -> bool: + def is_ignored(self, path: StrPath) -> bool: """Is *path* ignored by the VCS?""" @classmethod @abstractmethod - def in_repo(cls, directory: PathLike) -> bool: + def in_repo(cls, directory: StrPath) -> bool: """Is *directory* inside of the VCS repository? :raises NotADirectoryError: if directory is not a directory. @@ -39,7 +43,7 @@ def in_repo(cls, directory: PathLike) -> bool: @classmethod @abstractmethod - def find_root(cls, cwd: PathLike = None) -> Optional[Path]: + def find_root(cls, cwd: Optional[StrPath] = None) -> Optional[Path]: """Try to find the root of the project from *cwd*. If none is found, return None. @@ -50,19 +54,19 @@ def find_root(cls, cwd: PathLike = None) -> Optional[Path]: class VCSStrategyNone(VCSStrategy): """Strategy that is used when there is no VCS.""" - def __init__(self, project: "Project"): + def __init__(self, project: Project): # pylint: disable=useless-super-delegation super().__init__(project) - def is_ignored(self, path: PathLike) -> bool: + def is_ignored(self, path: StrPath) -> bool: return False @classmethod - def in_repo(cls, directory: PathLike) -> bool: + def in_repo(cls, directory: StrPath) -> bool: return False @classmethod - def find_root(cls, cwd: PathLike = None) -> Optional[Path]: + def find_root(cls, cwd: Optional[StrPath] = None) -> Optional[Path]: return None @@ -80,7 +84,7 @@ def _find_all_ignored_files(self) -> Set[Path]: ignored, don't return all files inside of it. """ command = [ - GIT_EXE, + str(GIT_EXE), "ls-files", "--exclude-standard", "--ignored", @@ -97,32 +101,32 @@ def _find_all_ignored_files(self) -> Set[Path]: all_files = result.stdout.decode("utf-8").split("\0") return {Path(file_) for file_ in all_files} - def is_ignored(self, path: PathLike) -> bool: + def is_ignored(self, path: StrPath) -> bool: path = self.project.relative_from_root(path) return path in self._all_ignored_files @classmethod - def in_repo(cls, directory: PathLike) -> bool: + def in_repo(cls, directory: StrPath) -> bool: if directory is None: directory = Path.cwd() if not Path(directory).is_dir(): raise NotADirectoryError() - command = [GIT_EXE, "status"] + command = [str(GIT_EXE), "status"] result = execute_command(command, _LOGGER, cwd=directory) return not result.returncode @classmethod - def find_root(cls, cwd: PathLike = None) -> Optional[Path]: + def find_root(cls, cwd: Optional[StrPath] = None) -> Optional[Path]: if cwd is None: cwd = Path.cwd() if not Path(cwd).is_dir(): raise NotADirectoryError() - command = [GIT_EXE, "rev-parse", "--show-toplevel"] + command = [str(GIT_EXE), "rev-parse", "--show-toplevel"] result = execute_command(command, _LOGGER, cwd=cwd) if not result.returncode: @@ -135,7 +139,7 @@ def find_root(cls, cwd: PathLike = None) -> Optional[Path]: class VCSStrategyHg(VCSStrategy): """Strategy that is used for Mercurial.""" - def __init__(self, project: "Project"): + def __init__(self, project: Project): super().__init__(project) if not HG_EXE: raise FileNotFoundError("Could not find binary for Mercurial") @@ -146,7 +150,7 @@ def _find_all_ignored_files(self) -> Set[Path]: is ignored, don't return all files inside of it. """ command = [ - HG_EXE, + str(HG_EXE), "status", "--ignored", # terse is marked 'experimental' in the hg help but is documented @@ -161,32 +165,32 @@ def _find_all_ignored_files(self) -> Set[Path]: all_files = result.stdout.decode("utf-8").split("\0") return {Path(file_) for file_ in all_files} - def is_ignored(self, path: PathLike) -> bool: + def is_ignored(self, path: StrPath) -> bool: path = self.project.relative_from_root(path) return path in self._all_ignored_files @classmethod - def in_repo(cls, directory: PathLike) -> bool: + def in_repo(cls, directory: StrPath) -> bool: if directory is None: directory = Path.cwd() if not Path(directory).is_dir(): raise NotADirectoryError() - command = [HG_EXE, "root"] + command = [str(HG_EXE), "root"] result = execute_command(command, _LOGGER, cwd=directory) return not result.returncode @classmethod - def find_root(cls, cwd: PathLike = None) -> Optional[Path]: + def find_root(cls, cwd: Optional[StrPath] = None) -> Optional[Path]: if cwd is None: cwd = Path.cwd() if not Path(cwd).is_dir(): raise NotADirectoryError() - command = [HG_EXE, "root"] + command = [str(HG_EXE), "root"] result = execute_command(command, _LOGGER, cwd=cwd) if not result.returncode: @@ -196,7 +200,7 @@ def find_root(cls, cwd: PathLike = None) -> Optional[Path]: return None -def find_root(cwd: PathLike = None) -> Optional[Path]: +def find_root(cwd: Optional[StrPath] = None) -> Optional[Path]: """Try to find the root of the project from *cwd*. If none is found, return None. From 3a92cddf0176db330fafa67e04e4cf2495af61e6 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 14:22:16 +0200 Subject: [PATCH 13/25] Make project mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/project.py | 38 ++++++++++++++++++++++---------------- src/reuse/vcs.py | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/reuse/project.py b/src/reuse/project.py index e68147dcb..14deda66c 100644 --- a/src/reuse/project.py +++ b/src/reuse/project.py @@ -12,7 +12,7 @@ import os from gettext import gettext as _ from pathlib import Path -from typing import Dict, Iterator, Optional +from typing import Dict, Iterator, Optional, Union, cast from boolean.boolean import ParseError from debian.copyright import Copyright @@ -32,14 +32,20 @@ _HEADER_BYTES, GIT_EXE, HG_EXE, - PathLike, + StrPath, _contains_snippet, _copyright_from_dep5, _determine_license_path, decoded_text_from_binary, extract_reuse_info, ) -from .vcs import VCSStrategyGit, VCSStrategyHg, VCSStrategyNone, find_root +from .vcs import ( + VCSStrategy, + VCSStrategyGit, + VCSStrategyHg, + VCSStrategyNone, + find_root, +) _LOGGER = logging.getLogger(__name__) @@ -51,7 +57,7 @@ class Project: def __init__( self, - root: PathLike, + root: StrPath, include_submodules: bool = False, include_meson_subprojects: bool = False, ): @@ -60,7 +66,7 @@ def __init__( raise NotADirectoryError(f"{self._root} is no valid path") if GIT_EXE and VCSStrategyGit.in_repo(self._root): - self.vcs_strategy = VCSStrategyGit(self) + self.vcs_strategy: VCSStrategy = VCSStrategyGit(self) elif HG_EXE and VCSStrategyHg.in_repo(self._root): self.vcs_strategy = VCSStrategyHg(self) else: @@ -72,14 +78,14 @@ def __init__( ) self.vcs_strategy = VCSStrategyNone(self) - self.licenses_without_extension = {} + self.licenses_without_extension: Dict[str, Path] = {} self.license_map = LICENSE_MAP.copy() # TODO: Is this correct? self.license_map.update(EXCEPTION_MAP) self.licenses = self._licenses() # Use '0' as None, because None is a valid value... - self._copyright_val = 0 + self._copyright_val: Optional[Union[int, Copyright]] = 0 self.include_submodules = include_submodules meson_build_path = self._root / "meson.build" @@ -88,7 +94,7 @@ def __init__( include_meson_subprojects and uses_meson ) - def all_files(self, directory: PathLike = None) -> Iterator[Path]: + def all_files(self, directory: Optional[StrPath] = None) -> Iterator[Path]: """Yield all files in *directory* and its subdirectories. The files that are not yielded are: @@ -101,8 +107,8 @@ def all_files(self, directory: PathLike = None) -> Iterator[Path]: directory = self.root directory = Path(directory) - for root, dirs, files in os.walk(directory): - root = Path(root) + for root_str, dirs, files in os.walk(directory): + root = Path(root_str) _LOGGER.debug("currently walking in '%s'", root) # Don't walk ignored directories @@ -141,7 +147,7 @@ def all_files(self, directory: PathLike = None) -> Iterator[Path]: _LOGGER.debug("yielding '%s'", the_file) yield the_file - def reuse_info_of(self, path: PathLike) -> ReuseInfo: + def reuse_info_of(self, path: StrPath) -> ReuseInfo: """Return REUSE info of *path*. This function will return any REUSE information that it can find, both @@ -232,7 +238,7 @@ def reuse_info_of(self, path: PathLike) -> ReuseInfo: source_path=source_path, source_type=source_type ) - def relative_from_root(self, path: PathLike) -> Path: + def relative_from_root(self, path: StrPath) -> Path: """If the project root is /tmp/project, and *path* is /tmp/project/src/file, then return src/file. """ @@ -304,17 +310,17 @@ def _copyright(self) -> Optional[Copyright]: # this line under each exception. if not self._copyright_val: self._copyright_val = None - return self._copyright_val + return cast(Optional[Copyright], self._copyright_val) def _licenses(self) -> Dict[str, Path]: """Return a dictionary of all licenses in the project, with their SPDX identifiers as names and paths as values. """ - license_files = {} + license_files: Dict[str, Path] = {} directory = str(self.root / "LICENSES/**") - for path in glob.iglob(directory, recursive=True): - path = Path(path) + for path_str in glob.iglob(directory, recursive=True): + path = Path(path_str) # For some reason, LICENSES/** is resolved even though it # doesn't exist. I have no idea why. Deal with that here. if not Path(path).exists() or Path(path).is_dir(): diff --git a/src/reuse/vcs.py b/src/reuse/vcs.py index 0e8eea720..3ae70b117 100644 --- a/src/reuse/vcs.py +++ b/src/reuse/vcs.py @@ -73,7 +73,7 @@ def find_root(cls, cwd: Optional[StrPath] = None) -> Optional[Path]: class VCSStrategyGit(VCSStrategy): """Strategy that is used for Git.""" - def __init__(self, project): + def __init__(self, project: Project): super().__init__(project) if not GIT_EXE: raise FileNotFoundError("Could not find binary for Git") From f9108d608686b704b97409b33f965ccbf073f46a Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 14:49:07 +0200 Subject: [PATCH 14/25] Make header mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/_util.py | 2 +- src/reuse/header.py | 103 ++++++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/src/reuse/_util.py b/src/reuse/_util.py index bd7b1062c..4b29ff0ae 100644 --- a/src/reuse/_util.py +++ b/src/reuse/_util.py @@ -235,7 +235,7 @@ def _copyright_from_dep5(path: StrPath, dep5_copyright: Copyright) -> ReuseInfo: map(_LICENSING.parse, [result.license.synopsis]) # type: ignore ), copyright_lines=set( - map(str.strip, result.copyright.splitlines()) # type:ignore + map(str.strip, result.copyright.splitlines()) # type: ignore ), source_type=SourceType.DEP5_FILE, ) diff --git a/src/reuse/header.py b/src/reuse/header.py index 26fcc1057..8725c94f6 100644 --- a/src/reuse/header.py +++ b/src/reuse/header.py @@ -21,11 +21,20 @@ import os import re import sys -from argparse import ArgumentParser +from argparse import ArgumentParser, Namespace from gettext import gettext as _ -from os import PathLike from pathlib import Path -from typing import Iterable, List, NamedTuple, Optional, Sequence, Tuple +from typing import ( + IO, + Iterable, + NamedTuple, + Optional, + Sequence, + Set, + Tuple, + Type, + cast, +) from binaryornot.check import is_binary from boolean.boolean import ParseError @@ -37,6 +46,7 @@ from ._util import ( _COPYRIGHT_STYLES, PathType, + StrPath, _determine_license_path, _determine_license_suffix_path, contains_reuse_info, @@ -82,9 +92,9 @@ class MissingReuseInfo(Exception): # TODO: Add a template here maybe. def _create_new_header( reuse_info: ReuseInfo, - template: Template = None, + template: Optional[Template] = None, template_is_commented: bool = False, - style: CommentStyle = None, + style: Optional[Type[CommentStyle]] = None, force_multi: bool = False, ) -> str: """Format a new header from scratch. @@ -96,7 +106,7 @@ def _create_new_header( if template is None: template = DEFAULT_TEMPLATE if style is None: - style = PythonCommentStyle + style = cast(Type[CommentStyle], PythonCommentStyle) rendered = template.render( copyright_lines=sorted(reuse_info.copyright_lines), @@ -132,10 +142,10 @@ def _create_new_header( # pylint: disable=too-many-arguments def create_header( reuse_info: ReuseInfo, - header: str = None, - template: Template = None, + header: Optional[str] = None, + template: Optional[Template] = None, template_is_commented: bool = False, - style: CommentStyle = None, + style: Optional[Type[CommentStyle]] = None, force_multi: bool = False, merge_copyrights: bool = False, ) -> str: @@ -203,7 +213,7 @@ def _indices_of_newlines(text: str) -> Sequence[int]: def _find_first_spdx_comment( - text: str, style: CommentStyle = None + text: str, style: Optional[Type[CommentStyle]] = None ) -> _TextSections: """Find the first SPDX comment in the file. Return a tuple with everything preceding the comment, the comment itself, and everything following it. @@ -247,9 +257,9 @@ def _extract_shebang(prefix: str, text: str) -> Tuple[str, str]: def find_and_replace_header( text: str, reuse_info: ReuseInfo, - template: Template = None, + template: Optional[Template] = None, template_is_commented: bool = False, - style: CommentStyle = None, + style: Optional[Type[CommentStyle]] = None, force_multi: bool = False, merge_copyrights: bool = False, ) -> str: @@ -324,9 +334,9 @@ def find_and_replace_header( def add_new_header( text: str, reuse_info: ReuseInfo, - template: Template = None, + template: Optional[Template] = None, template_is_commented: bool = False, - style: CommentStyle = None, + style: Optional[Type[CommentStyle]] = None, force_multi: bool = False, merge_copyrights: bool = False, ) -> str: @@ -365,11 +375,15 @@ def add_new_header( return new_text -def _get_comment_style(path: Path) -> Optional[CommentStyle]: +def _get_comment_style(path: StrPath) -> Optional[Type[CommentStyle]]: """Return value of CommentStyle detected for *path* or None.""" + path = Path(path) style = FILENAME_COMMENT_STYLE_MAP_LOWERCASE.get(path.name.lower()) if style is None: - style = EXTENSION_COMMENT_STYLE_MAP_LOWERCASE.get(path.suffix.lower()) + style = cast( + Optional[Type[CommentStyle]], + EXTENSION_COMMENT_STYLE_MAP_LOWERCASE.get(path.suffix.lower()), + ) return style @@ -382,11 +396,11 @@ def _is_uncommentable(path: Path) -> bool: def _verify_paths_line_handling( - paths: List[Path], + paths: Iterable[Path], parser: ArgumentParser, force_single: bool, force_multi: bool, -): +) -> None: """This function aborts the parser when *force_single* or *force_multi* is used, but the file type does not support that type of comment style. """ @@ -410,7 +424,9 @@ def _verify_paths_line_handling( ) -def _verify_paths_comment_style(paths: List[Path], parser: ArgumentParser): +def _verify_paths_comment_style( + paths: Iterable[Path], parser: ArgumentParser +) -> None: unrecognised_files = [] for path in paths: @@ -459,30 +475,30 @@ def _find_template(project: Project, name: str) -> Template: def _add_header_to_file( - path: PathLike, + path: StrPath, reuse_info: ReuseInfo, - template: Template, + template: Optional[Template], template_is_commented: bool, style: Optional[str], force_multi: bool = False, skip_existing: bool = False, merge_copyrights: bool = False, replace: bool = True, - out=sys.stdout, + out: IO[str] = sys.stdout, ) -> int: """Helper function.""" - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-locals result = 0 if style is not None: - style = NAME_STYLE_MAP[style] + comment_style: Optional[Type[CommentStyle]] = NAME_STYLE_MAP.get(style) else: - style = _get_comment_style(path) - if style is None: - out.write(_("Skipped unrecognised file {path}").format(path=path)) - out.write("\n") - return result + comment_style = _get_comment_style(path) + if comment_style is None: + out.write(_("Skipped unrecognised file {path}").format(path=path)) + out.write("\n") + return result - with path.open("r", encoding="utf-8", newline="") as fp: + with open(path, "r", encoding="utf-8", newline="") as fp: text = fp.read() # Ideally, this check is done elsewhere. But that would necessitate reading @@ -508,7 +524,7 @@ def _add_header_to_file( reuse_info, template=template, template_is_commented=template_is_commented, - style=style, + style=comment_style, force_multi=force_multi, merge_copyrights=merge_copyrights, ) @@ -518,7 +534,7 @@ def _add_header_to_file( reuse_info, template=template, template_is_commented=template_is_commented, - style=style, + style=comment_style, force_multi=force_multi, merge_copyrights=merge_copyrights, ) @@ -539,7 +555,7 @@ def _add_header_to_file( out.write("\n") result = 1 else: - with path.open("w", encoding="utf-8", newline=line_ending) as fp: + with open(path, "w", encoding="utf-8", newline=line_ending) as fp: fp.write(output) # TODO: This may need to be rephrased more elegantly. out.write(_("Successfully changed header of {path}").format(path=path)) @@ -548,7 +564,9 @@ def _add_header_to_file( return result -def _verify_write_access(paths: Iterable[PathLike], parser: ArgumentParser): +def _verify_write_access( + paths: Iterable[StrPath], parser: ArgumentParser +) -> None: not_writeable = [ str(path) for path in paths if not os.access(path, os.W_OK) ] @@ -558,7 +576,7 @@ def _verify_write_access(paths: Iterable[PathLike], parser: ArgumentParser): ) -def add_arguments(parser) -> None: +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to parser.""" parser.add_argument( "--copyright", @@ -666,7 +684,7 @@ def add_arguments(parser) -> None: parser.add_argument("path", action="store", nargs="+", type=PathType("r")) -def run(args, project: Project, out=sys.stdout) -> int: +def run(args: Namespace, project: Project, out: IO[str] = sys.stdout) -> int: """Add headers to files.""" # pylint: disable=too-many-branches,too-many-locals,too-many-statements if "addheader" in args.parser.prog.split(): @@ -709,7 +727,7 @@ def run(args, project: Project, out=sys.stdout) -> int: args.force_dot_license = True if args.recursive: - paths = set() + paths: Set[Path] = set() all_files = [path.resolve() for path in project.all_files()] for path in args.path: if path.is_file(): @@ -739,19 +757,22 @@ def run(args, project: Project, out=sys.stdout) -> int: if not args.skip_unrecognised: _verify_paths_comment_style(paths, args.parser) - template = None + template: Optional[Template] = None commented = False if args.template: try: - template = _find_template(project, args.template) + template = cast(Template, _find_template(project, args.template)) except TemplateNotFound: args.parser.error( _("template {template} could not be found").format( template=args.template ) ) + # This code is never reached, but mypy is not aware that + # parser.error quits the program. + raise - if ".commented" in Path(template.name).suffixes: + if ".commented" in Path(cast(str, template.name)).suffixes: commented = True year = None @@ -761,7 +782,7 @@ def run(args, project: Project, out=sys.stdout) -> int: elif args.year: year = args.year.pop() else: - year = datetime.date.today().year + year = str(datetime.date.today().year) expressions = set(args.license) if args.license is not None else set() copyright_style = ( From 3890dcfd07d1c51538e5cce88798739d2e3744e3 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 14:58:07 +0200 Subject: [PATCH 15/25] Make _main mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/_main.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/reuse/_main.py b/src/reuse/_main.py index 5c1282515..64bc4e45f 100644 --- a/src/reuse/_main.py +++ b/src/reuse/_main.py @@ -10,7 +10,7 @@ import logging import sys from gettext import gettext as _ -from typing import List +from typing import IO, Callable, List, Optional, Type, cast from . import ( __REUSE_version__, @@ -238,14 +238,14 @@ def parser() -> argparse.ArgumentParser: def add_command( # pylint: disable=too-many-arguments,redefined-builtin - subparsers, + subparsers: argparse._SubParsersAction, name: str, - add_arguments_func, - run_func, - formatter_class=None, - description: str = None, - help: str = None, - aliases: list = None, + add_arguments_func: Callable[[argparse.ArgumentParser], None], + run_func: Callable[[argparse.Namespace, Project, IO[str]], int], + formatter_class: Optional[Type[argparse.HelpFormatter]] = None, + description: Optional[str] = None, + help: Optional[str] = None, + aliases: Optional[List[str]] = None, ) -> None: """Add a subparser for a command.""" if formatter_class is None: @@ -262,10 +262,10 @@ def add_command( # pylint: disable=too-many-arguments,redefined-builtin subparser.set_defaults(parser=subparser) -def main(args: List[str] = None, out=sys.stdout) -> int: +def main(args: Optional[List[str]] = None, out: IO[str] = sys.stdout) -> int: """Main entry function.""" if args is None: - args = sys.argv[1:] + args = cast(List[str], sys.argv[1:]) main_parser = parser() parsed_args = main_parser.parse_args(args) From b0a754012e838045ac4431e36ba3723a38ced00c Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 14:59:29 +0200 Subject: [PATCH 16/25] Make init mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/__init__.py | 16 ++++++++-------- src/reuse/init.py | 15 ++++++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/reuse/__init__.py b/src/reuse/__init__.py index ba741adac..623bfb4ed 100644 --- a/src/reuse/__init__.py +++ b/src/reuse/__init__.py @@ -22,7 +22,7 @@ from dataclasses import dataclass, field from enum import Enum, auto from importlib.metadata import PackageNotFoundError, version -from typing import NamedTuple, Optional, Set, Type +from typing import Any, Dict, NamedTuple, Optional, Set, Type from boolean.boolean import Expression @@ -109,7 +109,7 @@ class ReuseInfo: source_path: Optional[str] = None source_type: Optional[SourceType] = None - def _check_nonexistent(self, **kwargs) -> None: + def _check_nonexistent(self, **kwargs: Any) -> None: nonexistent_attributes = set(kwargs) - set(self.__dict__) if nonexistent_attributes: raise KeyError( @@ -117,7 +117,7 @@ def _check_nonexistent(self, **kwargs) -> None: f" {self.__class__}: {', '.join(nonexistent_attributes)}" ) - def copy(self, **kwargs) -> Type["ReuseInfo"]: + def copy(self, **kwargs: Any) -> "ReuseInfo": """Return a copy of ReuseInfo, replacing the values of attributes with the values from *kwargs*. """ @@ -125,9 +125,9 @@ def copy(self, **kwargs) -> Type["ReuseInfo"]: new_kwargs = {} for key, value in self.__dict__.items(): new_kwargs[key] = kwargs.get(key, value) - return self.__class__(**new_kwargs) + return self.__class__(**new_kwargs) # type: ignore - def union(self, value) -> Type["ReuseInfo"]: + def union(self, value: "ReuseInfo") -> "ReuseInfo": """Return a new instance of ReuseInfo where all Set attributes are equal to the union of the set in *self* and the set in *value*. @@ -147,16 +147,16 @@ def union(self, value) -> Type["ReuseInfo"]: new_kwargs[key] = attr_val.union(other_val) else: new_kwargs[key] = attr_val - return self.__class__(**new_kwargs) + return self.__class__(**new_kwargs) # type: ignore def contains_copyright_or_licensing(self) -> bool: """Either *spdx_expressions* or *copyright_lines* is non-empty.""" return bool(self.spdx_expressions or self.copyright_lines) - def __bool__(self): + def __bool__(self) -> bool: return any(self.__dict__.values()) - def __or__(self, value) -> Type["ReuseInfo"]: + def __or__(self, value: "ReuseInfo") -> "ReuseInfo": return self.union(value) diff --git a/src/reuse/init.py b/src/reuse/init.py index 509984c83..6928acff0 100644 --- a/src/reuse/init.py +++ b/src/reuse/init.py @@ -5,10 +5,11 @@ """Functions for REUSE-ifying a project.""" import sys +from argparse import ArgumentParser, Namespace from gettext import gettext as _ from inspect import cleandoc from pathlib import Path -from typing import List +from typing import IO, List from urllib.error import URLError from ._licenses import ALL_NON_DEPRECATED_MAP @@ -18,7 +19,7 @@ from .vcs import find_root -def prompt_licenses(out=sys.stdout) -> List[str]: +def prompt_licenses(out: IO[str] = sys.stdout) -> List[str]: """Prompt the user for a list of licenses.""" first = _( "What license is your project under? " @@ -28,7 +29,7 @@ def prompt_licenses(out=sys.stdout) -> List[str]: "What other license is your project under? " "Provide the SPDX License Identifier." ) - licenses = [] + licenses: List[str] = [] while True: if not licenses: @@ -49,7 +50,7 @@ def prompt_licenses(out=sys.stdout) -> List[str]: licenses.append(result) -def add_arguments(parser): +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to parser.""" parser.add_argument( "path", @@ -59,7 +60,11 @@ def add_arguments(parser): ) -def run(args, project: Project, out=sys.stdout): +def run( + args: Namespace, + project: Project, + out: IO[str] = sys.stdout, +) -> int: """List all non-compliant files.""" # pylint: disable=too-many-statements,unused-argument if args.path: From 8ba2fe3aabd5407250dd130fe0ae4e2f1bc19382 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 15:13:03 +0200 Subject: [PATCH 17/25] Make conftest mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- tests/conftest.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3c594b10d..601f5ecc4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ from inspect import cleandoc from io import StringIO from pathlib import Path -from typing import Optional +from typing import Generator from unittest.mock import create_autospec import pytest @@ -72,7 +72,7 @@ def git_exe() -> str: """Run the test with git.""" if not GIT_EXE: pytest.skip("cannot run this test without git") - yield GIT_EXE + return str(GIT_EXE) @pytest.fixture() @@ -80,11 +80,11 @@ def hg_exe() -> str: """Run the test with mercurial (hg).""" if not HG_EXE: pytest.skip("cannot run this test without mercurial") - yield HG_EXE + return str(HG_EXE) @pytest.fixture(params=[True, False]) -def multiprocessing(request, monkeypatch) -> bool: +def multiprocessing(request, monkeypatch) -> Generator[bool, None, None]: """Run the test with or without multiprocessing.""" if not request.param: monkeypatch.delattr(mp, "Pool") @@ -92,7 +92,7 @@ def multiprocessing(request, monkeypatch) -> bool: @pytest.fixture(params=[True, False]) -def add_license_concluded(request) -> bool: +def add_license_concluded(request) -> Generator[bool, None, None]: yield request @@ -169,7 +169,7 @@ def _repo_contents( @pytest.fixture() -def git_repository(fake_repository: Path, git_exe: Optional[str]) -> Path: +def git_repository(fake_repository: Path, git_exe: str) -> Path: """Create a git repository with ignored files.""" os.chdir(fake_repository) _repo_contents(fake_repository) @@ -197,7 +197,7 @@ def git_repository(fake_repository: Path, git_exe: Optional[str]) -> Path: @pytest.fixture() -def hg_repository(fake_repository: Path, hg_exe: Optional[str]) -> Path: +def hg_repository(fake_repository: Path, hg_exe: str) -> Path: """Create a mercurial repository with ignored files.""" os.chdir(fake_repository) _repo_contents( @@ -225,7 +225,7 @@ def hg_repository(fake_repository: Path, hg_exe: Optional[str]) -> Path: @pytest.fixture() def submodule_repository( - git_repository: Path, git_exe: Optional[str], tmpdir_factory + git_repository: Path, git_exe: str, tmpdir_factory ) -> Path: """Create a git repository that contains a submodule.""" header = cleandoc( From 8b726a5e32c5e0249e022a5805ef352f6beb08c6 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sun, 9 Apr 2023 15:30:23 +0200 Subject: [PATCH 18/25] Make other tests mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- tests/test_lint.py | 7 ++++--- tests/test_main.py | 10 +++++++--- tests/test_project.py | 7 ++++--- tests/test_report.py | 7 ++++--- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 6439006b0..357ac18ef 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -7,6 +7,7 @@ import shutil import sys +from importlib import import_module import pytest @@ -15,14 +16,14 @@ from reuse.report import ProjectReport try: - import posix as is_posix + IS_POSIX = bool(import_module("posix")) except ImportError: - is_posix = False + IS_POSIX = False cpython = pytest.mark.skipif( sys.implementation.name != "cpython", reason="only CPython supported" ) -posix = pytest.mark.skipif(not is_posix, reason="Windows not supported") +posix = pytest.mark.skipif(not IS_POSIX, reason="Windows not supported") # REUSE-IgnoreStart diff --git a/tests/test_main.py b/tests/test_main.py index 0faed6df5..342b4b6cf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -16,7 +16,7 @@ import re from inspect import cleandoc from pathlib import Path -from typing import Optional +from typing import Generator, Optional from unittest.mock import create_autospec from urllib.error import URLError @@ -31,7 +31,9 @@ @pytest.fixture(params=[True, False]) -def optional_git_exe(request, monkeypatch) -> Optional[str]: +def optional_git_exe( + request, monkeypatch +) -> Generator[Optional[str], None, None]: """Run the test with or without git.""" exe = GIT_EXE if request.param else "" monkeypatch.setattr("reuse.project.GIT_EXE", exe) @@ -40,7 +42,9 @@ def optional_git_exe(request, monkeypatch) -> Optional[str]: @pytest.fixture(params=[True, False]) -def optional_hg_exe(request, monkeypatch) -> Optional[str]: +def optional_hg_exe( + request, monkeypatch +) -> Generator[Optional[str], None, None]: """Run the test with or without mercurial.""" exe = HG_EXE if request.param else "" monkeypatch.setattr("reuse.project.HG_EXE", exe) diff --git a/tests/test_project.py b/tests/test_project.py index ae41389e2..59ab96f08 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -8,6 +8,7 @@ import os import shutil +from importlib import import_module from inspect import cleandoc from pathlib import Path from textwrap import dedent @@ -18,11 +19,11 @@ from reuse.project import Project try: - import posix as is_posix + IS_POSIX = bool(import_module("posix")) except ImportError: - is_posix = False + IS_POSIX = False -posix = pytest.mark.skipif(not is_posix, reason="Windows not supported") +posix = pytest.mark.skipif(not IS_POSIX, reason="Windows not supported") TESTS_DIRECTORY = Path(__file__).parent.resolve() RESOURCES_DIRECTORY = TESTS_DIRECTORY / "resources" diff --git a/tests/test_report.py b/tests/test_report.py index 4bc2a23f0..ba2fc3147 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -9,6 +9,7 @@ import os import sys +from importlib import import_module import pytest @@ -16,14 +17,14 @@ from reuse.report import FileReport, ProjectReport try: - import posix as is_posix + IS_POSIX = bool(import_module("posix")) except ImportError: - is_posix = False + IS_POSIX = False cpython = pytest.mark.skipif( sys.implementation.name != "cpython", reason="only CPython supported" ) -posix = pytest.mark.skipif(not is_posix, reason="Windows not supported") +posix = pytest.mark.skipif(not IS_POSIX, reason="Windows not supported") # REUSE-IgnoreStart From 7e1b376294c94b279fac55fd0aed33cc329605b5 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 01:56:41 +0200 Subject: [PATCH 19/25] Make download mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/download.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/reuse/download.py b/src/reuse/download.py index b4ad5517d..4be10dace 100644 --- a/src/reuse/download.py +++ b/src/reuse/download.py @@ -9,15 +9,17 @@ import logging import sys import urllib.request +from argparse import ArgumentParser, Namespace from gettext import gettext as _ -from os import PathLike from pathlib import Path +from typing import IO from urllib.error import URLError from urllib.parse import urljoin from ._licenses import ALL_NON_DEPRECATED_MAP from ._util import ( PathType, + StrPath, find_licenses_directory, print_incorrect_spdx_identifier, ) @@ -49,12 +51,12 @@ def download_license(spdx_identifier: str) -> str: raise URLError("Status code was not 200") -def _path_to_license_file(spdx_identifier: str, root: PathLike) -> Path: +def _path_to_license_file(spdx_identifier: str, root: StrPath) -> Path: licenses_path = find_licenses_directory(root=root) return licenses_path / "".join((spdx_identifier, ".txt")) -def put_license_in_file(spdx_identifier: str, destination: PathLike) -> None: +def put_license_in_file(spdx_identifier: str, destination: StrPath) -> None: """Download a license and put it in the destination file. This function exists solely for convenience. @@ -77,7 +79,7 @@ def put_license_in_file(spdx_identifier: str, destination: PathLike) -> None: fp.write(text) -def add_arguments(parser) -> None: +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to parser.""" parser.add_argument( "license", @@ -95,10 +97,10 @@ def add_arguments(parser) -> None: ) -def run(args, project: Project, out=sys.stdout) -> int: +def run(args: Namespace, project: Project, out: IO[str] = sys.stdout) -> int: """Download license and place it in the LICENSES/ directory.""" - def _already_exists(path: PathLike): + def _already_exists(path: StrPath) -> None: out.write( _("Error: {spdx_identifier} already exists.").format( spdx_identifier=path @@ -106,7 +108,7 @@ def _already_exists(path: PathLike): ) out.write("\n") - def _could_not_download(identifier: str): + def _could_not_download(identifier: str) -> None: out.write(_("Error: Failed to download license.")) out.write(" ") if identifier not in ALL_NON_DEPRECATED_MAP: @@ -115,7 +117,7 @@ def _could_not_download(identifier: str): out.write(_("Is your internet connection working?")) out.write("\n") - def _successfully_downloaded(destination: PathLike): + def _successfully_downloaded(destination: StrPath) -> None: out.write( _("Successfully downloaded {spdx_identifier}.").format( spdx_identifier=destination From de584f444a6d4eb696f16d23262cb6a43f56e346 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 01:59:16 +0200 Subject: [PATCH 20/25] Make spdx mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/spdx.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/reuse/spdx.py b/src/reuse/spdx.py index e94bddc21..20cb2739f 100644 --- a/src/reuse/spdx.py +++ b/src/reuse/spdx.py @@ -8,7 +8,9 @@ import contextlib import logging import sys +from argparse import ArgumentParser, Namespace from gettext import gettext as _ +from typing import IO from . import _IGNORE_SPDX_PATTERNS from ._util import PathType @@ -18,7 +20,7 @@ _LOGGER = logging.getLogger(__name__) -def add_arguments(parser) -> None: +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to the parser.""" parser.add_argument( "--output", "-o", dest="file", action="store", type=PathType("w") @@ -43,7 +45,7 @@ def add_arguments(parser) -> None: ) -def run(args, project: Project, out=sys.stdout) -> int: +def run(args: Namespace, project: Project, out: IO[str] = sys.stdout) -> int: """Print the project's bill of materials.""" # The SPDX spec mandates that a creator must be specified when a license # conclusion is made, so here we enforce that. More context: From 30ea22afda6cd333bc6d59f17d241f94c59ed2e3 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 02:01:37 +0200 Subject: [PATCH 21/25] Make _format mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/_format.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/reuse/_format.py b/src/reuse/_format.py index c90e3e02c..afc8bc3d8 100644 --- a/src/reuse/_format.py +++ b/src/reuse/_format.py @@ -5,19 +5,20 @@ """Formatting functions primarily for the CLI.""" from textwrap import fill, indent +from typing import Iterator WIDTH = 78 INDENT = 2 -def fill_paragraph(text, width=WIDTH, indent_width=0): +def fill_paragraph(text: str, width: int = WIDTH, indent_width: int = 0) -> str: """Wrap a single paragraph.""" return indent( fill(text.strip(), width=width - indent_width), indent_width * " " ) -def fill_all(text, width=WIDTH, indent_width=0): +def fill_all(text: str, width: int = WIDTH, indent_width: int = 0) -> str: """Wrap all paragraphs.""" return "\n\n".join( fill_paragraph(paragraph, width=width, indent_width=indent_width) @@ -25,7 +26,7 @@ def fill_all(text, width=WIDTH, indent_width=0): ) -def split_into_paragraphs(text): +def split_into_paragraphs(text: str) -> Iterator[str]: """Yield all paragraphs in a text. A paragraph is a piece of text surrounded by empty lines. """ From ad05304d9dd2ee672a5c2e65bd72d1ded392e7db Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 02:10:53 +0200 Subject: [PATCH 22/25] Make _licenses mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/_licenses.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/reuse/_licenses.py b/src/reuse/_licenses.py index b4432c92d..0f9b0d305 100644 --- a/src/reuse/_licenses.py +++ b/src/reuse/_licenses.py @@ -11,6 +11,7 @@ import json import os +from typing import Dict, List, Tuple _BASE_DIR = os.path.dirname(__file__) _RESOURCES_DIR = os.path.join(_BASE_DIR, "resources") @@ -18,7 +19,7 @@ _EXCEPTIONS = os.path.join(_RESOURCES_DIR, "exceptions.json") -def _load_license_list(file_name): +def _load_license_list(file_name: str) -> Tuple[List[int], Dict[str, Dict]]: """Return the licenses list version tuple and a mapping of licenses id->name loaded from a JSON file from https://github.com/spdx/license-list-data @@ -33,7 +34,7 @@ def _load_license_list(file_name): return version, licenses_map -def _load_exception_list(file_name): +def _load_exception_list(file_name: str) -> Tuple[List[int], Dict[str, Dict]]: """Return the exceptions list version tuple and a mapping of exceptions id->name loaded from a JSON file from https://github.com/spdx/license-list-data From 40d1786b10713926327937a2e7b5ee0f2c4e90a4 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Mon, 10 Apr 2023 02:12:47 +0200 Subject: [PATCH 23/25] Make supported_licenses mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/supported_licenses.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/reuse/supported_licenses.py b/src/reuse/supported_licenses.py index e8799e176..736d3df3f 100644 --- a/src/reuse/supported_licenses.py +++ b/src/reuse/supported_licenses.py @@ -6,18 +6,20 @@ """supported-licenses command handler""" import sys +from argparse import ArgumentParser, Namespace +from typing import IO from ._licenses import _LICENSES, _load_license_list from .project import Project # pylint: disable=unused-argument -def add_arguments(parser) -> None: +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to the parser.""" # pylint: disable=unused-argument -def run(args, project: Project, out=sys.stdout): +def run(args: Namespace, project: Project, out: IO[str] = sys.stdout) -> int: """Print the supported SPDX licenses list""" licenses = _load_license_list(_LICENSES)[1] From ecf467e592deeb950097b6b3370d3df6681a5d5c Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sat, 10 Jun 2023 19:54:40 +0200 Subject: [PATCH 24/25] Make report mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/report.py | 101 +++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/src/reuse/report.py b/src/reuse/report.py index 1303f9748..268e48430 100644 --- a/src/reuse/report.py +++ b/src/reuse/report.py @@ -13,13 +13,13 @@ from gettext import gettext as _ from hashlib import md5 from io import StringIO -from os import PathLike, cpu_count +from os import cpu_count from pathlib import Path -from typing import Iterable, List, NamedTuple, Optional, Set +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, cast from uuid import uuid4 from . import __REUSE_version__, __version__ -from ._util import _LICENSING, _checksum +from ._util import _LICENSING, StrPath, _checksum from .project import Project, ReuseInfo _LOGGER = logging.getLogger(__name__) @@ -30,12 +30,14 @@ class _MultiprocessingContainer: """Container that remembers some data in order to generate a FileReport.""" - def __init__(self, project, do_checksum, add_license_concluded): + def __init__( + self, project: Project, do_checksum: bool, add_license_concluded: bool + ): self.project = project self.do_checksum = do_checksum self.add_license_concluded = add_license_concluded - def __call__(self, file_): + def __call__(self, file_: StrPath) -> "_MultiprocessingResult": # pylint: disable=broad-except try: return _MultiprocessingResult( @@ -55,7 +57,7 @@ def __call__(self, file_): class _MultiprocessingResult(NamedTuple): """Result of :class:`MultiprocessingContainer`.""" - path: PathLike + path: StrPath report: Optional["FileReport"] error: Optional[Exception] @@ -64,31 +66,31 @@ class ProjectReport: # pylint: disable=too-many-instance-attributes """Object that holds linting report about the project.""" def __init__(self, do_checksum: bool = True): - self.path = None - self.licenses = {} - self.missing_licenses = {} - self.bad_licenses = {} - self.deprecated_licenses = set() - self.read_errors = set() - self.file_reports = set() - self.licenses_without_extension = {} + self.path: StrPath = "" + self.licenses: Dict[str, Path] = {} + self.missing_licenses: Dict[str, Set[Path]] = {} + self.bad_licenses: Dict[str, Set[Path]] = {} + self.deprecated_licenses: Set[str] = set() + self.read_errors: Set[Path] = set() + self.file_reports: Set[FileReport] = set() + self.licenses_without_extension: Dict[str, Path] = {} self.do_checksum = do_checksum - self._unused_licenses = None - self._used_licenses = None - self._files_without_licenses = None - self._files_without_copyright = None - self._is_compliant = None + self._unused_licenses: Optional[Set[str]] = None + self._used_licenses: Optional[Set[str]] = None + self._files_without_licenses: Optional[Set[Path]] = None + self._files_without_copyright: Optional[Set[Path]] = None + self._is_compliant: Optional[bool] = None - def to_dict_lint(self): + def to_dict_lint(self) -> Dict[str, Any]: """Collects and formats data relevant to linting from report and returns it as a dictionary. :return: Dictionary containing data from the ProjectReport object """ # Setup report data container - data = { + data: Dict[str, Any] = { "non_compliant": { "missing_licenses": self.missing_licenses, "unused_licenses": [str(file) for file in self.unused_licenses], @@ -232,7 +234,7 @@ def generate( cls, project: Project, do_checksum: bool = True, - multiprocessing: bool = cpu_count() > 1, + multiprocessing: bool = cpu_count() > 1, # type: ignore add_license_concluded: bool = False, ) -> "ProjectReport": """Generate a ProjectReport from a Project.""" @@ -249,7 +251,9 @@ def generate( if multiprocessing: with mp.Pool() as pool: - results = pool.map(container, project.all_files()) + results: Iterable[_MultiprocessingResult] = pool.map( + container, project.all_files() + ) pool.join() else: results = map(container, project.all_files()) @@ -261,7 +265,7 @@ def generate( _("Could not read '{path}'").format(path=result.path), exc_info=result.error, ) - project_report.read_errors.add(result.path) + project_report.read_errors.add(Path(result.path)) continue _LOGGER.error( _( @@ -269,9 +273,9 @@ def generate( ).format(path=result.path), exc_info=result.error, ) - project_report.read_errors.add(result.path) + project_report.read_errors.add(Path(result.path)) continue - file_report = result.report + file_report = cast(FileReport, result.report) # File report. project_report.file_reports.add(file_report) @@ -329,8 +333,8 @@ def unused_licenses(self) -> Set[str]: return self._unused_licenses @property - def files_without_licenses(self) -> Iterable[PathLike]: - """Iterable of paths that have no license information.""" + def files_without_licenses(self) -> Set[Path]: + """Set of paths that have no license information.""" if self._files_without_licenses is not None: return self._files_without_licenses @@ -343,8 +347,8 @@ def files_without_licenses(self) -> Iterable[PathLike]: return self._files_without_licenses @property - def files_without_copyright(self) -> Iterable[PathLike]: - """Iterable of paths that have no copyright information.""" + def files_without_copyright(self) -> Set[Path]: + """Set of paths that have no copyright information.""" if self._files_without_copyright is not None: return self._files_without_copyright @@ -383,14 +387,19 @@ class _File: # pylint: disable=too-few-public-methods case. """ - def __init__(self, name, spdx_id=None, chk_sum=None): + def __init__( + self, + name: str, + spdx_id: Optional[str] = None, + chk_sum: Optional[str] = None, + ): self.name: str = name - self.spdx_id: str = spdx_id - self.chk_sum: str = chk_sum + self.spdx_id: Optional[str] = spdx_id + self.chk_sum: Optional[str] = chk_sum self.licenses_in_file: List[str] = [] - self.license_concluded: str = None - self.copyright: str = None - self.info: ReuseInfo = None + self.license_concluded: str = "" + self.copyright: str = "" + self.info: ReuseInfo = ReuseInfo() class FileReport: @@ -398,17 +407,15 @@ class FileReport: it also contains SPDX File information in :attr:`spdxfile`. """ - def __init__( - self, name: PathLike, path: PathLike, do_checksum: bool = True - ): - self.spdxfile = _File(name) + def __init__(self, name: StrPath, path: StrPath, do_checksum: bool = True): + self.spdxfile = _File(str(name)) self.path = Path(path) self.do_checksum = do_checksum - self.bad_licenses = set() - self.missing_licenses = set() + self.bad_licenses: Set[str] = set() + self.missing_licenses: Set[str] = set() - def to_dict_lint(self): + def to_dict_lint(self) -> Dict[str, Any]: """Turn the report into a json-like dictionary with exclusively information relevant for linting. """ @@ -432,7 +439,7 @@ def to_dict_lint(self): def generate( cls, project: Project, - path: PathLike, + path: StrPath, do_checksum: bool = True, add_license_concluded: bool = False, ) -> "FileReport": @@ -503,13 +510,13 @@ def generate( report.spdxfile.info = reuse_info return report - def __hash__(self): + def __hash__(self) -> int: if self.spdxfile.chk_sum is not None: return hash(self.spdxfile.name + self.spdxfile.chk_sum) - return super().__hash__(self) + return super().__hash__() -def format_creator(creator: str) -> str: +def format_creator(creator: Optional[str]) -> str: """Render the creator field based on the provided flag""" if creator is None: return "Anonymous ()" From bf40ad415ff63ef14354c020c19636ccb5dad167 Mon Sep 17 00:00:00 2001 From: Carmen Bianca BAKKER Date: Sat, 10 Jun 2023 20:01:29 +0200 Subject: [PATCH 25/25] Make lint mypy-compliant Signed-off-by: Carmen Bianca BAKKER --- src/reuse/lint.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/reuse/lint.py b/src/reuse/lint.py index 980bcf983..6c76701da 100644 --- a/src/reuse/lint.py +++ b/src/reuse/lint.py @@ -9,16 +9,18 @@ import json import sys +from argparse import ArgumentParser, Namespace from gettext import gettext as _ from io import StringIO from pathlib import Path +from typing import IO, Any from . import __REUSE_version__ from .project import Project from .report import ProjectReport -def add_arguments(parser): +def add_arguments(parser: ArgumentParser) -> None: """Add arguments to parser.""" mutex_group = parser.add_mutually_exclusive_group() mutex_group.add_argument( @@ -208,7 +210,7 @@ def format_json(report: ProjectReport) -> str: :return: String (representing JSON) that can be output to sys.stdout """ - def custom_serializer(obj): + def custom_serializer(obj: Any) -> Any: """Custom serializer for the dictionary output of ProjectReport :param obj: Object to be serialized @@ -229,7 +231,7 @@ def custom_serializer(obj): ) -def run(args, project: Project, out=sys.stdout): +def run(args: Namespace, project: Project, out: IO[str] = sys.stdout) -> int: """List all non-compliant files.""" report = ProjectReport.generate( project, do_checksum=False, multiprocessing=not args.no_multiprocessing