From d8257fd2eeb2fe024e6560c68fbb4c2285ad0227 Mon Sep 17 00:00:00 2001 From: detachhead Date: Mon, 17 Jun 2024 20:18:16 +1000 Subject: [PATCH] add some tests for our python code --- .github/workflows/build_and_release.yml | 5 ++ .github/workflows/validation.yml | 5 +- .../src/tests/harness/fourslash/testState.ts | 8 +- pdm.lock | 80 ++++++++++++++++++- pyproject.toml | 14 +++- tests/__init__.py | 2 + tests/test_generate_docstubs.py | 59 ++++++++++++++ tests/test_python_command.py | 12 +++ 8 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_generate_docstubs.py create mode 100644 tests/test_python_command.py diff --git a/.github/workflows/build_and_release.yml b/.github/workflows/build_and_release.yml index ddbd4c2dda..4d88b28c61 100644 --- a/.github/workflows/build_and_release.yml +++ b/.github/workflows/build_and_release.yml @@ -92,6 +92,11 @@ jobs: realpath ./.pyprojectx/main >> $GITHUB_PATH echo ./node_modules/.bin >> $GITHUB_PATH + # ideally this should be run with the rest of the pytest tests in the validation workflow, + # but they depend on the docstubs generated in the previous job + - name: validate docstubs + run: pdm run test_python -- -m needs_all_docstubs + - id: current-version run: | echo ::set-output name=CURRENT_VERSION::$(pdm show --version) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index f464559ed0..198662d5f6 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -73,8 +73,11 @@ jobs: restore-keys: | ${{ runner.os }}-node- - - run: ./pw pdm install - run: ./pw pdm use ${{ env.PYTHON_VERSION }} --first + - run: ./pw pdm install + + - run: python tests + - run: ./pw pdm run test_python -- -m "not needs_all_docstubs" - name: npm test (pyright-internal) run: npm test diff --git a/packages/pyright-internal/src/tests/harness/fourslash/testState.ts b/packages/pyright-internal/src/tests/harness/fourslash/testState.ts index ebe5123cb6..a1bc8fcb97 100644 --- a/packages/pyright-internal/src/tests/harness/fourslash/testState.ts +++ b/packages/pyright-internal/src/tests/harness/fourslash/testState.ts @@ -81,7 +81,7 @@ import { import { TestAccessHost } from '../testAccessHost'; import * as host from '../testHost'; import { stringify } from '../utils'; -import { createFromFileSystem, distlibFolder, libFolder, typeshedFolder } from '../vfs/factory'; +import { createFromFileSystem, distlibFolder, libFolder } from '../vfs/factory'; import * as vfs from '../vfs/filesystem'; import { parseTestData } from './fourSlashParser'; import { @@ -1714,9 +1714,9 @@ export class TestState { } configOptions.include.push(getFileSpec(configOptions.projectRoot, '.')); - configOptions.exclude.push(getFileSpec(configOptions.projectRoot, typeshedFolder.getFilePath())); - configOptions.exclude.push(getFileSpec(configOptions.projectRoot, distlibFolder.getFilePath())); - configOptions.exclude.push(getFileSpec(configOptions.projectRoot, libFolder.getFilePath())); + // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, typeshedFolder.getFilePath())); + // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, distlibFolder.getFilePath())); + // configOptions.exclude.push(getFileSpec(configOptions.projectRoot, libFolder.getFilePath())); if (mountPaths) { for (const mountPath of mountPaths.keys()) { diff --git a/pdm.lock b/pdm.lock index 24e49e132d..908d71225a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "docstubs"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:c7dfb6836249152b93768912c3cd27de5070acb4438875ced9a02458f3c659ae" +content_hash = "sha256:1208dee5238db9803e571ca08ef8ee0d5b4110e297687ccf6897359d6d0fe986" [[package]] name = "astroid" @@ -53,6 +53,18 @@ revision = "20ac945391c14de82cb5281e4ce019f262311633" summary = "" groups = ["docstubs"] +[[package]] +name = "exceptiongroup" +version = "1.2.1" +requires_python = ">=3.7" +summary = "Backport of PEP 654 (exception groups)" +groups = ["dev"] +marker = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + [[package]] name = "importlib-metadata" version = "7.1.0" @@ -68,6 +80,17 @@ files = [ {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" +groups = ["dev"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "isort" version = "5.13.2" @@ -176,6 +199,17 @@ files = [ {file = "nodejs_wheel_binaries-20.14.0.tar.gz", hash = "sha256:38e4b1b4dee08898da5fb9d813d1653162fd0e6f4ec83dc6615b2432f1fa90aa"}, ] +[[package]] +name = "packaging" +version = "24.1" +requires_python = ">=3.8" +summary = "Core utilities for Python packages" +groups = ["dev"] +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + [[package]] name = "pdm-backend" version = "2.3.0" @@ -201,6 +235,17 @@ files = [ {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] +[[package]] +name = "pluggy" +version = "1.5.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + [[package]] name = "pylint" version = "3.2.3" @@ -225,6 +270,39 @@ files = [ {file = "pylint-3.2.3.tar.gz", hash = "sha256:02f6c562b215582386068d52a30f520d84fdbcf2a95fc7e855b816060d048b60"}, ] +[[package]] +name = "pytest" +version = "8.2.2" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +groups = ["dev"] +dependencies = [ + "colorama; sys_platform == \"win32\"", + "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", + "iniconfig", + "packaging", + "pluggy<2.0,>=1.5", + "tomli>=1; python_version < \"3.11\"", +] +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + +[[package]] +name = "pytest-github-actions-annotate-failures" +version = "0.2.0" +requires_python = ">=3.7" +summary = "pytest plugin to annotate failed tests with a workflow command for GitHub Actions" +groups = ["dev"] +dependencies = [ + "pytest>=4.0.0", +] +files = [ + {file = "pytest-github-actions-annotate-failures-0.2.0.tar.gz", hash = "sha256:844ab626d389496e44f960b42f0a72cce29ae06d363426d17ea9ae1b4bef2288"}, + {file = "pytest_github_actions_annotate_failures-0.2.0-py3-none-any.whl", hash = "sha256:8bcef65fed503faaa0524b59cfeccc8995130972dd7b008d64193cc41b9cde85"}, +] + [[package]] name = "pyyaml" version = "6.0.1" diff --git a/pyproject.toml b/pyproject.toml index 6d9749064d..05ee73509e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ dev = [ "nodejs-wheel>=20.13.1", "pdm-backend>=2.3.0", "typing_extensions>=4.12.2", + "pytest>=8.2.2", + "pytest-github-actions-annotate-failures>=0.2.0", ] # these deps are also needed in build-system.requires. see https://github.com/pdm-project/pdm/issues/2947 docstubs = [ @@ -31,10 +33,11 @@ typecheck = 'basedpyright' ruff_check = { composite = ['ruff check', 'ruff format --check --diff'] } ruff_fix = { composite = ['ruff check --fix', 'ruff format --fix'] } pylint = 'pylint basedpyright based_build pdm_build.py' +test_python = 'pytest tests' generate_docstubs = { call = "based_build.generate_docstubs:main" } [tool.pdm.build] -excludes = ["based_build/"] +excludes = ["based_build/", "tests/", "pdm_build.py"] [project] name = "basedpyright" @@ -249,7 +252,7 @@ ignore-overlong-task-comments = true [tool.ruff.lint.per-file-ignores] "*.pyi" = ["A001", "A002", "N"] # we don't control names in 3rd party modules "tests/**/*.py" = [ - "S101", # Use of assert detected (pytest uses assert statements) + "S", # none of these security focused rules are relevant for the tests ] [tool.ruff.lint.isort] combine-as-imports = true @@ -258,3 +261,10 @@ split-on-trailing-comma = false [tool.ruff.format] skip-magic-trailing-comma = true + +[tool.pytest.ini_options] +xfail_strict = true +addopts = ['--strict-markers'] +markers = [ + 'needs_all_docstubs', # indicates that the test needs to run after the docstubs are built, rather than during validation +] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..284220f129 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +"""tests for python code. any tests for typescript code should go in +`packages/pyright-internal/src/tests` instead""" diff --git a/tests/test_generate_docstubs.py b/tests/test_generate_docstubs.py new file mode 100644 index 0000000000..35a5477388 --- /dev/null +++ b/tests/test_generate_docstubs.py @@ -0,0 +1,59 @@ +""" +these tests assume that the doc stubs have already run (they should get run as part of +`pdm install`) as they validate that the current state of the `docstubs` folder is correct +""" + +from __future__ import annotations + +import sys +from locale import getpreferredencoding +from pathlib import Path + +from pytest import mark + +quotes = '"""' + + +def read_module_text(name: str): + return Path("docstubs/stdlib", name).read_text( + encoding=getpreferredencoding(do_setlocale=False) + ) + + +def test_builtin_docstring(): + assert ''' +class float: + """Convert a string or number to a floating point number, if possible.""" +''' in read_module_text("builtins.pyi") + + +@mark.skipif(sys.platform != "win32", reason="windows only") +@mark.needs_all_docstubs +def test_windows_only_docstring(): + assert read_module_text("nt.pyi").startswith('''""" +This module provides access to operating system functionality that is +standardized by the C Standard and the POSIX standard (a thinly +disguised Unix interface). Refer to the library manual and +corresponding Unix manual entries for more information on calls. +"""''') + + +@mark.skipif(sys.platform == "win32", reason="method does not exist on windows") +@mark.needs_all_docstubs +def test_linux_or_mac_only_docstring(): + assert ''' + def tzset() -> None: + """ + tzset() + + Initialize, or reinitialize, the local timezone to the value stored in + os.environ['TZ']. The TZ environment variable should be specified in + standard Unix timezone format as documented in the tzset man page + (eg. 'US/Eastern', 'Europe/Amsterdam'). Unknown timezones will silently + fall back to UTC. If the TZ environment variable is not set, the local + timezone is set to the systems best guess of wallclock time. + Changing the TZ environment variable without calling tzset *may* change + the local timezone used by methods such as localtime, but this behaviour + should not be relied on. + """ +''' in read_module_text("time.pyi") diff --git a/tests/test_python_command.py b/tests/test_python_command.py new file mode 100644 index 0000000000..25e05ef2e8 --- /dev/null +++ b/tests/test_python_command.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from subprocess import run + + +def test_version(): + """pretty useless test, this is mainly just making sure the python wrapper scripts aren't + busted""" + result = run(["basedpyright", "--version"], check=True, capture_output=True) + assert result.returncode == 0 + assert result.stdout.startswith(b"basedpyright ") + assert b"based on pyright " in result.stdout