Add support for type checking with mypy plugins

mypy_primer/main.py

@@ -18,15 +18,15 @@
 from mypy_primer.globals import ctx, parse_options_and_set_ctx
 from mypy_primer.model import Project, TypeCheckResult
 from mypy_primer.projects import get_projects
-from mypy_primer.type_checker import setup_mypy, setup_pyright, setup_typeshed
+from mypy_primer.type_checker import Checker, setup_mypy, setup_pyright, setup_typeshed
 from mypy_primer.utils import Style, debug_print, line_count, run, strip_colour_code
 
 T = TypeVar("T")
 
 
 async def setup_new_and_old_mypy(
     new_mypy_revision: RevisionLike, old_mypy_revision: RevisionLike
-) -> tuple[Path, Path]:
+) -> tuple[Checker, Checker]:
     new_mypy, old_mypy = await asyncio.gather(
         setup_mypy(
             ctx.get().base_dir / "new_mypy",
@@ -44,8 +44,8 @@ async def setup_new_and_old_mypy(
 
     if ctx.get().debug:
         (new_version, _), (old_version, _) = await asyncio.gather(
-            run([str(new_mypy), "--version"], output=True),
-            run([str(old_mypy), "--version"], output=True),
+            run([str(new_mypy.path), "--version"], output=True),
+            run([str(old_mypy.path), "--version"], output=True),
         )
         debug_print(f"{Style.BLUE}new mypy version: {new_version.stdout.strip()}{Style.RESET}")
         debug_print(f"{Style.BLUE}old mypy version: {old_version.stdout.strip()}{Style.RESET}")
@@ -55,7 +55,7 @@ async def setup_new_and_old_mypy(
 
 async def setup_new_and_old_pyright(
     new_pyright_revision: RevisionLike, old_pyright_revision: RevisionLike
-) -> tuple[Path, Path]:
+) -> tuple[Checker, Checker]:
     new_pyright, old_pyright = await asyncio.gather(
         setup_pyright(
             ctx.get().base_dir / "new_pyright",
@@ -71,8 +71,8 @@ async def setup_new_and_old_pyright(
 
     if ctx.get().debug:
         (new_version, _), (old_version, _) = await asyncio.gather(
-            run([str(new_pyright), "--version"], output=True),
-            run([str(old_pyright), "--version"], output=True),
+            run([str(new_pyright.path), "--version"], output=True),
+            run([str(old_pyright.path), "--version"], output=True),
         )
         debug_print(f"{Style.BLUE}new pyright version: {new_version.stdout.strip()}{Style.RESET}")
         debug_print(f"{Style.BLUE}old pyright version: {old_version.stdout.strip()}{Style.RESET}")
@@ -251,7 +251,7 @@ async def bisect() -> None:
     await asyncio.gather(*[project.setup() for project in projects])
 
     async def run_wrapper(project: Project) -> tuple[str, TypeCheckResult]:
-        return project.name, (await project.run_mypy(str(mypy_exe), typeshed_dir=None))
+        return project.name, (await project.run_mypy(mypy_exe, typeshed_dir=None))
 
     results_fut = await asyncio.gather(*(run_wrapper(project) for project in projects))
     old_results: dict[str, TypeCheckResult] = dict(results_fut)
@@ -307,9 +307,9 @@ async def coverage() -> None:
 
     projects = select_projects()
     if sys.platform == "win32":
-        mypy_python = mypy_exe.parent / "python.exe"
+        mypy_python = mypy_exe.path.parent / "python.exe"
     else:
-        mypy_python = mypy_exe.parent / "python"
+        mypy_python = mypy_exe.path.parent / "python"
 
     assert mypy_python.exists()
 
@@ -355,8 +355,8 @@ async def primer() -> int:
 
     results = [
         project.primer_result(
-            new_type_checker=str(new_type_checker),
-            old_type_checker=str(old_type_checker),
+            new_type_checker=new_type_checker,
+            old_type_checker=old_type_checker,
             new_typeshed=new_typeshed_dir,
             old_typeshed=old_typeshed_dir,
         )

mypy_primer/model.py

@@ -17,6 +17,7 @@
 
 from mypy_primer.git_utils import ensure_repo_at_revision
 from mypy_primer.globals import ctx
+from mypy_primer.type_checker import Checker
 from mypy_primer.utils import BIN_DIR, Style, debug_print, quote_path, run
 
 extra_dataclass_args = {"kw_only": True} if sys.version_info >= (3, 10) else {}
@@ -75,6 +76,11 @@ def name(self) -> str:
     def venv_dir(self) -> Path:
         return ctx.get().projects_dir / f"_{self.name}_venv"
 
+    @property
+    def site_packages(self) -> Path:
+        python = f"python{sys.version_info.major}.{sys.version_info.minor}"
+        return self.venv_dir / "lib" / python / "site-packages"
+
     def expected_success(self, type_checker: str) -> bool:
         if type_checker == "mypy":
             return self.expected_mypy_success
@@ -120,6 +126,8 @@ async def setup(self) -> None:
                 if e.stderr:
                     print(e.stderr)
                 raise RuntimeError(f"pip install failed for {self.name}") from e
+            else:
+                assert not (self.venv_dir / BIN_DIR / "mypy").exists(), "Mypy installed by project"
 
     def get_mypy_cmd(self, mypy: str | Path, additional_flags: Sequence[str] = ()) -> str:
         mypy_cmd = self.mypy_cmd
@@ -140,7 +148,7 @@ def get_mypy_cmd(self, mypy: str | Path, additional_flags: Sequence[str] = ()) -
         )
         return mypy_cmd
 
-    async def run_mypy(self, mypy: str | Path, typeshed_dir: Path | None) -> TypeCheckResult:
+    async def run_mypy(self, mypy: Checker, typeshed_dir: Path | None) -> TypeCheckResult:
         additional_flags = ctx.get().additional_flags.copy()
         env = os.environ.copy()
         env["MYPY_FORCE_COLOR"] = "1"
@@ -153,8 +161,15 @@ async def run_mypy(self, mypy: str | Path, typeshed_dir: Path | None) -> TypeChe
         if "MYPYPATH" in env:
             mypy_path = env["MYPYPATH"].split(os.pathsep) + mypy_path
         env["MYPYPATH"] = os.pathsep.join(mypy_path)
+        pythonpath = env.get("PYTHONPATH", "").split(os.pathsep)
+        pythonpath.insert(0, str(self.site_packages))
+        if mypy.site_packages is not None:
+            pythonpath.insert(0, str(mypy.site_packages))
+        env["PYTHONPATH"] = os.pathsep.join(pythonpath)
 
-        mypy_cmd = self.get_mypy_cmd(mypy, additional_flags)
+        mypy_cmd = self.get_mypy_cmd(mypy.path, additional_flags)
+        if ctx.get().debug:
+            debug_print(f'{Style.BLUE}PYTHONPATH={env["PYTHONPATH"]}{Style.RESET}')
         proc, runtime = await run(
             mypy_cmd,
             shell=True,
@@ -164,7 +179,7 @@ async def run_mypy(self, mypy: str | Path, typeshed_dir: Path | None) -> TypeChe
             env=env,
         )
         if ctx.get().debug:
-            debug_print(f"{Style.BLUE}{mypy} on {self.name} took {runtime:.2f}s{Style.RESET}")
+            debug_print(f"{Style.BLUE}{mypy.path} on {self.name} took {runtime:.2f}s{Style.RESET}")
 
         output = proc.stderr + proc.stdout
 
@@ -203,11 +218,11 @@ def get_pyright_cmd(self, pyright: str | Path, additional_flags: Sequence[str] =
         pyright_cmd = pyright_cmd.format(pyright=pyright)
         return pyright_cmd
 
-    async def run_pyright(self, pyright: str | Path, typeshed_dir: Path | None) -> TypeCheckResult:
+    async def run_pyright(self, pyright: Checker, typeshed_dir: Path | None) -> TypeCheckResult:
         additional_flags: list[str] = []
         if typeshed_dir is not None:
             additional_flags.append(f"--typeshedpath {quote_path(typeshed_dir)}")
-        pyright_cmd = self.get_pyright_cmd(pyright, additional_flags)
+        pyright_cmd = self.get_pyright_cmd(pyright.path, additional_flags)
         if self.pip_cmd:
             activate = (
                 f"source {shlex.quote(str(self.venv_dir / BIN_DIR / 'activate'))}"
@@ -223,15 +238,17 @@ async def run_pyright(self, pyright: str | Path, typeshed_dir: Path | None) -> T
             cwd=ctx.get().projects_dir / self.name,
         )
         if ctx.get().debug:
-            debug_print(f"{Style.BLUE}{pyright} on {self.name} took {runtime:.2f}s{Style.RESET}")
+            debug_print(
+                f"{Style.BLUE}{pyright.path} on {self.name} took {runtime:.2f}s{Style.RESET}"
+            )
 
         output = proc.stderr + proc.stdout
         return TypeCheckResult(
             pyright_cmd, output, not bool(proc.returncode), self.expected_pyright_success, runtime
         )
 
     async def run_typechecker(
-        self, type_checker: str | Path, typeshed_dir: Path | None
+        self, type_checker: Checker, typeshed_dir: Path | None
     ) -> TypeCheckResult:
         if ctx.get().type_checker == "mypy":
             return await self.run_mypy(type_checker, typeshed_dir)
@@ -242,8 +259,8 @@ async def run_typechecker(
 
     async def primer_result(
         self,
-        new_type_checker: str,
-        old_type_checker: str,
+        new_type_checker: Checker,
+        old_type_checker: Checker,
         new_typeshed: Path | None,
         old_typeshed: Path | None,
     ) -> PrimerResult:

mypy_primer/projects.py

@@ -642,15 +642,12 @@ def get_projects() -> list[Project]:
         ),
         Project(
             location="https://github.com/home-assistant/core",
-            mypy_cmd=(
-                "sed -i.bak '/^plugins = pydantic.mypy$/s/^/#/' mypy.ini; {mypy} homeassistant"
-            ),
+            mypy_cmd="{mypy} homeassistant",
             pip_cmd=(
                 "{pip} install attrs pydantic types-setuptools types-atomicwrites types-certifi"
                 " types-croniter types-PyYAML types-requests types-python-slugify types-backports"
             ),
             mypy_cost=70,
-            supported_platforms=["linux", "darwin"],  # hack for sed
         ),
         Project(location="https://github.com/kornia/kornia", mypy_cmd="{mypy} kornia"),
         Project(
@@ -912,6 +909,12 @@ def get_projects() -> list[Project]:
                 " sniffio"
             ),
         ),
+        Project(
+            location="https://github.com/flaeppe/django-choicefield",
+            mypy_cmd="{mypy}",
+            pip_cmd="{pip} install . django-stubs pytest",
+            expected_mypy_success=True,
+        ),
     ]
     assert len(projects) == len({p.name for p in projects})
     for p in projects:

mypy_primer/type_checker.py

@@ -6,19 +6,25 @@
 import sys
 import venv
 from pathlib import Path
+from typing import NamedTuple
 
 from mypy_primer.git_utils import RevisionLike, ensure_repo_at_revision
 from mypy_primer.utils import BIN_DIR, MYPY_EXE_NAME, run
 
 
+class Checker(NamedTuple):
+    path: Path
+    site_packages: Path | None
+
+
 async def setup_mypy(
     mypy_dir: Path,
     revision_like: RevisionLike,
     *,
     repo: str | None,
     mypyc_compile_level: int | None,
     editable: bool = False,
-) -> Path:
+) -> Checker:
     mypy_dir.mkdir(exist_ok=True)
     venv_dir = mypy_dir / "venv"
     venv.create(venv_dir, with_pip=True, clear=True)
@@ -62,15 +68,16 @@ async def setup_mypy(
         # warm up mypy on macos to avoid the first run being slow
         await run([str(mypy_exe), "--version"])
     assert mypy_exe.exists()
-    return mypy_exe
+    python = f"python{sys.version_info.major}.{sys.version_info.minor}"
+    return Checker(path=mypy_exe, site_packages=venv_dir / "lib" / python / "site-packages")
 
 
 async def setup_pyright(
     pyright_dir: Path,
     revision_like: RevisionLike,
     *,
     repo: str | None,
-) -> Path:
+) -> Checker:
     pyright_dir.mkdir(exist_ok=True)
 
     if repo is None:
@@ -82,7 +89,7 @@ async def setup_pyright(
 
     pyright_exe = repo_dir / "packages" / "pyright" / "index.js"
     assert pyright_exe.exists()
-    return pyright_exe
+    return Checker(path=pyright_exe, site_packages=None)
 
 
 async def setup_typeshed(parent_dir: Path, *, repo: str, revision_like: RevisionLike) -> Path: