add some tests for our python code

.github/workflows/build_and_release.yml

@@ -92,6 +92,11 @@ jobs:
           realpath ./.pyprojectx/main >> $GITHUB_PATH
           echo ./node_modules/.bin >> $GITHUB_PATH
 
+      # ideally this should be run with the rest of the pytest tests in the validation workflow,
+      # but they depend on the docstubs generated in the previous job
+      - name: validate docstubs
+        run: pdm run test_python -- -m needs_all_docstubs
+
       - id: current-version
         run: |
           echo ::set-output name=CURRENT_VERSION::$(pdm show --version)

.github/workflows/validation.yml

@@ -73,8 +73,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
 
-      - run: ./pw pdm install
       - run: ./pw pdm use ${{ env.PYTHON_VERSION }} --first
+      - run: ./pw pdm install
+
+      - name: python tests
+        run: ./pw pdm run test_python -- -m "not needs_all_docstubs"
 
       - name: npm test (pyright-internal)
         run: npm test

packages/pyright-internal/src/tests/harness/fourslash/testState.ts

@@ -81,7 +81,7 @@ import {
 import { TestAccessHost } from '../testAccessHost';
 import * as host from '../testHost';
 import { stringify } from '../utils';
-import { createFromFileSystem, distlibFolder, libFolder, typeshedFolder } from '../vfs/factory';
+import { createFromFileSystem, distlibFolder, libFolder } from '../vfs/factory';
 import * as vfs from '../vfs/filesystem';
 import { parseTestData } from './fourSlashParser';
 import {
@@ -1714,9 +1714,9 @@ export class TestState {
         }
 
         configOptions.include.push(getFileSpec(configOptions.projectRoot, '.'));
-        configOptions.exclude.push(getFileSpec(configOptions.projectRoot, typeshedFolder.getFilePath()));
-        configOptions.exclude.push(getFileSpec(configOptions.projectRoot, distlibFolder.getFilePath()));
-        configOptions.exclude.push(getFileSpec(configOptions.projectRoot, libFolder.getFilePath()));
+        // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, typeshedFolder.getFilePath()));
+        // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, distlibFolder.getFilePath()));
+        // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, libFolder.getFilePath()));
 
         if (mountPaths) {
             for (const mountPath of mountPaths.keys()) {

pyproject.toml

@@ -12,6 +12,9 @@ dev = [
     "nodejs-wheel>=20.13.1",
     "pdm-backend>=2.3.0",
     "typing_extensions>=4.12.2",
+    "pytest>=8.2.2",
+    "pytest-github-actions-annotate-failures>=0.2.0",
+    "basedtyping>=0.1.4",
 ]
 # these deps are also needed in build-system.requires. see https://github.com/pdm-project/pdm/issues/2947
 docstubs = [
@@ -31,10 +34,11 @@ typecheck = 'basedpyright'
 ruff_check = { composite = ['ruff check', 'ruff format --check --diff'] }
 ruff_fix = { composite = ['ruff check --fix', 'ruff format --fix'] }
 pylint = 'pylint basedpyright based_build pdm_build.py'
+test_python = 'pytest tests'
 generate_docstubs = { call = "based_build.generate_docstubs:main" }
 
 [tool.pdm.build]
-excludes = ["based_build/"]
+excludes = ["based_build/", "tests/", "pdm_build.py"]
 
 [project]
 name = "basedpyright"
@@ -249,7 +253,7 @@ ignore-overlong-task-comments = true
 [tool.ruff.lint.per-file-ignores]
 "*.pyi" = ["A001", "A002", "N"] # we don't control names in 3rd party modules
 "tests/**/*.py" = [
-    "S101", # Use of assert detected (pytest uses assert statements)
+    "S", # none of these security focused rules are relevant for the tests
 ]
 [tool.ruff.lint.isort]
 combine-as-imports = true
@@ -258,3 +262,10 @@ split-on-trailing-comma = false
 
 [tool.ruff.format]
 skip-magic-trailing-comma = true
+
+[tool.pytest.ini_options]
+xfail_strict = true
+addopts = ['--strict-markers']
+markers = [
+    'needs_all_docstubs', # indicates that the test needs to run after the docstubs are built, rather than during validation
+]

tests/__init__.py

@@ -0,0 +1,2 @@
+"""tests for python code. any tests for typescript code should go in
+`packages/pyright-internal/src/tests` instead"""

tests/test_generate_docstubs.py

@@ -0,0 +1,79 @@
+"""
+these tests assume that the doc stubs have already run (they should get run as part of
+`pdm install`) as they validate that the current state of the `docstubs` folder is correct
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from functools import wraps
+from locale import getpreferredencoding
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable
+
+from pytest import mark
+
+if TYPE_CHECKING:
+    from basedtyping import P, T
+
+
+def needs_all_docstubs(
+    condition_to_run_locally: bool,  # noqa: FBT001
+) -> Callable[[Callable[P, T]], Callable[P, T]]:
+    def decorator(fn: Callable[P, T]) -> Callable[P, T]:
+        @wraps(fn)
+        @mark.needs_all_docstubs
+        @mark.skipif(
+            not condition_to_run_locally and not os.getenv("GITHUB_ACTIONS"),
+            reason="condition not met",
+        )
+        def wrapped(*args: P.args, **kwargs: P.kwargs):
+            return fn(*args, **kwargs)
+
+        return wrapped
+
+    return decorator
+
+
+def read_module_text(name: str):
+    return Path("docstubs/stdlib", name).read_text(
+        encoding=getpreferredencoding(do_setlocale=False)
+    )
+
+
+def test_builtin_docstring():
+    assert '''
+class float:
+    """Convert a string or number to a floating point number, if possible."""
+''' in read_module_text("builtins.pyi")
+
+
+@needs_all_docstubs(sys.platform == "win32")
+def test_windows_only_docstring():
+    assert read_module_text("nt.pyi").startswith('''"""
+This module provides access to operating system functionality that is
+standardized by the C Standard and the POSIX standard (a thinly
+disguised Unix interface).  Refer to the library manual and
+corresponding Unix manual entries for more information on calls.
+"""''')
+
+
+@needs_all_docstubs(sys.platform != "win32")
+def test_linux_or_mac_only_docstring():
+    assert '''
+    def tzset() -> None:
+        """
+        tzset()
+
+        Initialize, or reinitialize, the local timezone to the value stored in
+        os.environ['TZ']. The TZ environment variable should be specified in
+        standard Unix timezone format as documented in the tzset man page
+        (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently
+        fall back to UTC. If the TZ environment variable is not set, the local
+        timezone is set to the systems best guess of wallclock time.
+        Changing the TZ environment variable without calling tzset *may* change
+        the local timezone used by methods such as localtime, but this behaviour
+        should not be relied on.
+        """
+''' in read_module_text("time.pyi")

tests/test_python_command.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from subprocess import run
+
+
+def test_version():
+    """pretty useless test, this is mainly just making sure the python wrapper scripts aren't
+    busted"""
+    result = run(["basedpyright", "--version"], check=True, capture_output=True)
+    assert result.returncode == 0
+    assert result.stdout.startswith(b"basedpyright ")
+    assert b"based on pyright " in result.stdout