From bd4446d3ff2eeb16952a20880021196120d74be9 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Sat, 3 Dec 2022 17:03:42 +0000 Subject: [PATCH] refactor(cli): remove pkg in favour of automatically downloading Node (#454) ## Change Summary This PR completely refactors how the Prisma CLI is downloaded / installed / called. We now download Node itself at runtime and use that to install the Prisma CLI and then run it directly with Node as well. This has some significant advantages: - We are now no longer tied to releases of the packaged CLI - These were being released by Luca, who created the Go Client and is no longer at Prisma, on his own free time. - Only major versions were released which means the CLI couldn't be easily tested with the latest changes on the https://github.com/prisma/prisma repository - Prisma Studio can now be launched from the CLI - The TypeScript Client can now be generated from our CLI wrapper - We now longer have to manually download the engine binaries ourselves, that's handled transparently for us - A packaged version of Node no longer has to be installed for each new Prisma version - We now have support for ARM However, this does not come without some concerns: - Size increase - We use https://github.com/ekalinin/nodeenv to download Node at runtime if it isn't already installed. This downloads and creates extra files that are not strictly necessary for our use case. This results in an increase from ~140MB -> ~300MB. - - However this size increase can be reduced by installing [nodejs-bin](https://pypi.org/project/nodejs-bin/) which you can do by providing the `node` extra, e.g. `pip install prisma[node]`. This brings the total download size to be very similar to the packaged CLI. - This concern also doesn't apply in cases where Node is already present. It actually will greatly improve the experience in this case. ## How does it work? We now resolve a Node binary using this flow: - Check if [nodejs-bin](https://pypi.org/project/nodejs-bin/) is installed - Check if Node is installed globally - Downloads Node using https://github.com/ekalinin/nodeenv to a configurable location which defaults to `~/.cache/prisma-nodeenv` The first two steps in this flow can be skipped if you so desire through your `pyproject.toml` file or using environment variables. For example: ```toml [tool.prisma] # skip global node check use_global_node = false # skip nodejs-bin check use_nodejs_bin = false # change nodeenv installation directory nodeenv_cache_dir = '~/.foo/nodeenv' ``` Or using the environment variables, `PRISMA_USE_GLOBAL_NODE`, `PRISMA_USE_NODEJS_BIN` and `PRISMA_NODEENV_CACHE_DIR` respectively. The Prisma CLI is then installed directly from [npm](https://www.npmjs.com/package/prisma) and ran directly using the resolved Node binary. --- .dockerignore | 5 + .github/workflows/test.yml | 19 +- databases/main.py | 8 +- databases/requirements.txt | 1 + docs/reference/config.md | 89 ++-- lib/testing/shared_conftest/__init__.py | 1 + .../shared_conftest/_shared_conftest.py | 13 + pipelines/coverage.nox.py | 5 +- pipelines/requirements/dev.txt | 1 + pipelines/requirements/lint.txt | 1 + pipelines/requirements/mypy.txt | 1 + pipelines/requirements/node.txt | 1 + pipelines/test.nox.py | 3 +- pipelines/utils/utils.py | 12 + requirements/base.txt | 1 + requirements/node.txt | 1 + scripts/docs.py | 16 +- setup.py | 10 +- src/prisma/_compat.py | 12 + src/prisma/_config.py | 64 ++- src/prisma/_proxy.py | 13 +- src/prisma/binaries/__init__.py | 2 - src/prisma/binaries/binaries.py | 74 ---- src/prisma/binaries/binary.py | 52 --- src/prisma/binaries/engine.py | 47 -- src/prisma/binaries/utils.py | 30 -- src/prisma/cli/__init__.py | 1 + src/prisma/cli/_node.py | 405 ++++++++++++++++++ src/prisma/cli/commands/fetch.py | 10 +- src/prisma/cli/commands/version.py | 2 +- src/prisma/cli/prisma.py | 126 +++--- src/prisma/engine/errors.py | 4 +- src/prisma/engine/utils.py | 70 ++- src/prisma/generator/generator.py | 3 + src/prisma/generator/jsonrpc.py | 12 +- src/prisma/generator/models.py | 52 ++- .../generator/templates/client.py.jinja | 5 +- .../generator/templates/engine/query.py.jinja | 9 +- src/prisma/utils.py | 11 + tests/Dockerfile | 10 + tests/integrations/sync/tests/test_binary.py | 14 - tests/test_binaries.py | 31 -- tests/test_cli/test_cli.py | 18 +- tests/test_cli/test_fetch.py | 61 +-- tests/test_cli/test_prisma.py | 21 + tests/test_cli/test_version.py | 14 +- tests/test_config.py | 5 +- tests/test_engine.py | 49 ++- .../test_exhaustive/test_async[client.py].raw | 5 +- .../test_async[enginequery.py].raw | 9 +- .../test_exhaustive/test_sync[client.py].raw | 5 +- .../test_sync[enginequery.py].raw | 9 +- .../exhaustive/test_exhaustive.py | 14 +- tests/test_generation/test_validation.py | 2 +- tests/test_node/__init__.py | 0 tests/test_node/test.js | 2 + tests/test_node/test_node.py | 259 +++++++++++ tests/utils.py | 14 + tests/windows.Dockerfile | 10 + 59 files changed, 1227 insertions(+), 517 deletions(-) create mode 100644 pipelines/requirements/node.txt create mode 100644 requirements/node.txt delete mode 100644 src/prisma/binaries/binaries.py delete mode 100644 src/prisma/binaries/binary.py delete mode 100644 src/prisma/binaries/engine.py delete mode 100644 src/prisma/binaries/utils.py create mode 100644 src/prisma/cli/_node.py delete mode 100644 tests/integrations/sync/tests/test_binary.py delete mode 100644 tests/test_binaries.py create mode 100644 tests/test_cli/test_prisma.py create mode 100644 tests/test_node/__init__.py create mode 100644 tests/test_node/test.js create mode 100644 tests/test_node/test_node.py diff --git a/.dockerignore b/.dockerignore index 58fd2b54f..2aecae64d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,9 @@ !src/ !tests !requirements/ +!databases/ +!lib/ +!noxfile.py !pipelines/ +!pytest.ini +!MANIFEST.in diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21c000d2f..bddbdcefc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: name: test runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python-version: [3.7, 3.8, 3.9, "3.10", "3.11.0-rc.1"] @@ -303,21 +304,23 @@ jobs: strategy: fail-fast: false matrix: - docker-platform: [linux/amd64] - # TODO: Uncomment this to add testing support for arm64 and delete - # the above - # docker-platform: ["linux/amd64", "linux/arm64"] - # TODO: Uncomment this later, Go-based CLI does not run on Alpine - # https://github.com/prisma/prisma-client-go/issues/357 - # python-os-distro: [slim-bullseye, alpine] - python-os-distro: [slim-bullseye] + docker-platform: [linux/amd64, linux/arm64] + python-os-distro: [slim-bullseye, alpine] + exclude: + # Dockerfile is currently broken for alpine / arm64 + # https://github.com/RobertCraigie/prisma-client-py/issues/581 + - docker-platform: linux/arm64 + python-os-distro: alpine steps: - uses: actions/checkout@v3 + # https://github.com/docker/build-push-action/ - name: Set up QEMU uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Docker Build uses: docker/build-push-action@v3 # https://github.com/docker/build-push-action/#inputs diff --git a/databases/main.py b/databases/main.py index aaff09d23..287d84bff 100644 --- a/databases/main.py +++ b/databases/main.py @@ -21,7 +21,11 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined from lib.utils import flatten, escape_path -from pipelines.utils import setup_coverage, get_pkg_location +from pipelines.utils import ( + setup_coverage, + get_pkg_location, + maybe_install_nodejs_bin, +) from prisma._compat import cached_property from .utils import DatabaseConfig @@ -68,6 +72,8 @@ def test( with session.chdir(DATABASES_DIR): # setup env session.install('-r', 'requirements.txt') + maybe_install_nodejs_bin(session) + if inplace: # pragma: no cover # useful for updating the generated code so that Pylance picks it up session.install('-U', '-e', '..') diff --git a/databases/requirements.txt b/databases/requirements.txt index 770a87420..4d42b52c1 100644 --- a/databases/requirements.txt +++ b/databases/requirements.txt @@ -3,6 +3,7 @@ click==8.1.3 coverage==6.5.0 syrupy==3.0.5 dirty-equals==0.5.0 +distro -r ../pipelines/requirements/deps/pyright.txt -r ../pipelines/requirements/deps/pytest.txt diff --git a/docs/reference/config.md b/docs/reference/config.md index ae60393ab..dc7d95d8a 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -199,54 +199,87 @@ Or through environment variables, e.g. `PRISMA_BINARY_CACHE_DIR`. In the case th ### Binary Cache Directory -This option controls where the Prisma Engine and Prisma CLI binaries should be downloaded to. This defaults to a temporary directory that includes the current Prisma Engine version. +This option controls where the Prisma Engine and Prisma CLI binaries should be downloaded to. This defaults to a cache directory that includes the current Prisma Engine version. + +| Option | Environment Variable | Default | +| ------------------ | -------------------------- | ------------------------------------------------------------------------- | +| `binary_cache_dir` | `PRISMA_BINARY_CACHE_DIR` | `/{home}/.cache/prisma-python/binaries/{prisma_version}/{engine_version}` | + +### Home Directory + +This option can be used to change the base directory of the `binary_cache_dir` option without having to worry about versioning the Prisma binaries. This is useful if you need to download the binaries to a local directory. + +| Option | Environment Variable | Default | +| -------- | --------------------- | ------- | +| `home_dir` | `PRISMA_HOME_DIR` | `~` | -| Option | Environment Variable | Default | -| ------------------ | -------------------------- | ----------------------------------------------------- | -| `binary_cache_dir` | `PRISMA_BINARY_CACHE_DIR` | `/{tmp}/prisma/binaries/engines/{engine_version}` | ### Prisma Version -This option controls the version of the Prisma CLI to use. It should be noted that this is intended to be internal and only the pinned prisma version is guaranteed to be supported. +This option controls the version of Prisma to use. It should be noted that this is intended to be internal and only the pinned Prisma version is guaranteed to be supported. | Option | Environment Variable | Default | | ---------------- | --------------------- | -------- | | `prisma_version` | `PRISMA_VERSION` | `3.13.0` | -### Engine Version +### Expected Engine Version -This option controls the version of the [Prisma Engines](https://github.com/prisma/prisma-engines) to use, like `prisma_version` this is intended to be internal and only the pinned engine version is guaranteed to be supported. +This is an internal option that is here as a safeguard for the `prisma_version` option. If you modify the `prisma_version` option then you must also update this option to use the corresponding engine version. You can find a list of engine versions [here](https://github.com/prisma/prisma-engines). -| Option | Environment Variable | Default | -| ---------------- | ----------------------- | ------------------------------------------ | -| `engine_version` | `PRISMA_ENGINE_VERSION` | `efdf9b1183dddfd4258cd181a72125755215ab7b` | +| Option | Environment Variable | Default | +| ------------------------- | -------------------------------- | ------------------------------------------ | +| `expected_engine_version` | `PRISMA_EXPECTED_ENGINE_VERSION` | `efdf9b1183dddfd4258cd181a72125755215ab7b` | -### Prisma URL -This option controls where the Prisma CLI binaries should be downloaded from. If set, this must be a string that takes two format arguments, `version` and `platform`, for example: +### Binary Platform + +This option is useful if you need to make use of the [binaryTargets](https://www.prisma.io/docs/concepts/components/prisma-schema/generators#binary-targets) schema option to build your application on one platform and deploy it on another. + +This allows you to set the current platform dynamically as Prisma Client Python does not have official support for `binaryTargets` and although we do have some safe guards in place to attempt to use the correct binary, it has not been thoroughly tested. + +A list of valid options can be found [here](https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference#binarytargets-options). -``` -https://example.com/prisma-cli-{version}-{platform}.gz -``` -| Option | Environment Variable | Default | -| -------------| -------------------- | --------------------------------------------------------------------------------------- | -| `prisma_url` | `PRISMA_CLI_URL` | `https://prisma-photongo.s3-eu-west-1.amazonaws.com/prisma-cli-{version}-{platform}.gz` | +| Option | Environment Variable | +| ----------------- | ------------------------ | +| `binary_platform` | `PRISMA_BINARY_PLATFORM` | -### Engine URL +### Use Global Node -This option controls where the [Prisma Engine](https://github.com/prisma/prisma-engines) binaries should be downloaded from. If set, this must be a string that takes three positional format arguments, for example: +This option configures whether or not Prisma Client Python will attempt to use the globally installed version of [Node](https://nodejs.org/en/), if it is available, to run the Prisma CLI. + +| Option | Environment Variable | Default | +| ----------------- | ------------------------ | ------- | +| `use_global_node` | `PRISMA_USE_GLOBAL_NODE` | `True` | + +### Use nodejs-bin + +This option configures whether or not Prisma Client Python will attempt to use the installed version of the [nodejs-bin](https://pypi.org/project/nodejs-bin/) package, if it is available, to run the Prisma CLI. + +| Option | Environment Variable | Default | +| ---------------- | ----------------------- | ------- | +| `use_nodejs_bin` | `PRISMA_USE_NODEJS_BIN` | `True` | + +### Extra Nodeenv Arguments + +This option allows you to pass additional arguments to [nodeenv](https://github.com/ekalinin/nodeenv) which is the package we use to automatically download a Node binary to run the CLI with. + +Arguments are passed after the path, for example: ``` -https://example.com/prisma-binaries-mirror/{0}/{1}/{2}.gz +python -m nodeenv ``` -Where: +| Option | Environment Variable | +| -------------------- | --------------------------- | +| `nodeenv_extra_args` | `PRISMA_NODEENV_EXTRA_ARGS` | + +### Nodeenv Cache Directory + +This option configures where Prisma Client Python will store the Node binary that is installed by [nodeenv](https://github.com/ekalinin/nodeenv). -- `0` corresponds to the [engine version](#engine-version) -- `1` corresponds to the current binary platform -- `2` corresponds to the name of the engine being downloaded. +Note that this does not make use of the [Home Directory](#home-directory) option and instead uses the actual user home directory. -| Option | Environment Variable | Default | -| -------------| -------------------- | ------------------------------------------------------- | -| `engine_url` | `PRISMA_ENGINE_URL` | `https://binaries.prisma.sh/all_commits/{0}/{1}/{2}.gz` | +| Option | Environment Variable | Default | +| ------------------- | -------------------------- | --------------------------------- | +| `nodeenv_cache_dir` | `PRISMA_NODEENV_CACHE_DIR` | `~/.cache/prisma-python/nodeenv/` | diff --git a/lib/testing/shared_conftest/__init__.py b/lib/testing/shared_conftest/__init__.py index 2c5d4a722..c4e13acec 100644 --- a/lib/testing/shared_conftest/__init__.py +++ b/lib/testing/shared_conftest/__init__.py @@ -1,6 +1,7 @@ # NOTE: a lot of these are not intended to be imported and referenced directly # and are instead intended to be `*` imported in a `conftest.py` file from ._shared_conftest import ( + setup_env as setup_env, event_loop as event_loop, client_fixture as client_fixture, patch_prisma_fixture as patch_prisma_fixture, diff --git a/lib/testing/shared_conftest/_shared_conftest.py b/lib/testing/shared_conftest/_shared_conftest.py index 0e64405d2..320050c7b 100644 --- a/lib/testing/shared_conftest/_shared_conftest.py +++ b/lib/testing/shared_conftest/_shared_conftest.py @@ -1,6 +1,7 @@ import asyncio import inspect from typing import TYPE_CHECKING, Iterator +from pathlib import Path import pytest @@ -14,6 +15,10 @@ if TYPE_CHECKING: from _pytest.fixtures import FixtureRequest + from _pytest.monkeypatch import MonkeyPatch + + +HOME_DIR = Path.home() @async_fixture(name='client', scope='session') @@ -31,6 +36,14 @@ def event_loop() -> asyncio.AbstractEventLoop: return get_or_create_event_loop() +@pytest.fixture(autouse=True) +def setup_env(monkeypatch: 'MonkeyPatch') -> None: + # Set a custom home directory to use for caching binaries so that + # when we make use of pytest's temporary directory functionality the binaries + # don't have to be downloaded again. + monkeypatch.setenv('PRISMA_HOME_DIR', str(HOME_DIR)) + + @pytest.fixture(name='patch_prisma', autouse=True) def patch_prisma_fixture(request: 'FixtureRequest') -> Iterator[None]: if request_has_client(request): diff --git a/pipelines/coverage.nox.py b/pipelines/coverage.nox.py index 66fdccf63..edf26bcc3 100644 --- a/pipelines/coverage.nox.py +++ b/pipelines/coverage.nox.py @@ -4,7 +4,6 @@ from pathlib import Path import nox -from git.repo import Repo from lib.utils import maybe_decode from pipelines.utils import setup_env, CACHE_DIR, TMP_DIR @@ -18,6 +17,10 @@ @nox.session(name='push-coverage') def push_coverage(session: nox.Session) -> None: + # We have to import `git` here as it will cause an error on machines that don't have + # git installed. This happens in our docker tests. + from git.repo import Repo + session.env['COVERAGE_FILE'] = str(CACHE_DIR / '.coverage') session.install( '-r', diff --git a/pipelines/requirements/dev.txt b/pipelines/requirements/dev.txt index 140b0afb8..ce0967a42 100644 --- a/pipelines/requirements/dev.txt +++ b/pipelines/requirements/dev.txt @@ -6,3 +6,4 @@ twine==4.0.1 typer==0.7.0 rtoml==0.9.0 GitPython +distro diff --git a/pipelines/requirements/lint.txt b/pipelines/requirements/lint.txt index 1c28482db..97dba7c2a 100644 --- a/pipelines/requirements/lint.txt +++ b/pipelines/requirements/lint.txt @@ -1,4 +1,5 @@ -r test.txt +-r node.txt -r deps/pyright.txt interrogate==1.5.0 blue==0.9.1 diff --git a/pipelines/requirements/mypy.txt b/pipelines/requirements/mypy.txt index 780fc1ef9..26654aa39 100644 --- a/pipelines/requirements/mypy.txt +++ b/pipelines/requirements/mypy.txt @@ -1,3 +1,4 @@ -r test.txt +-r node.txt mypy==0.950 types-mock diff --git a/pipelines/requirements/node.txt b/pipelines/requirements/node.txt new file mode 100644 index 000000000..03d4501d3 --- /dev/null +++ b/pipelines/requirements/node.txt @@ -0,0 +1 @@ +nodejs-bin==16.15.1a4 diff --git a/pipelines/test.nox.py b/pipelines/test.nox.py index 24cbf8d87..8e1d8b13b 100644 --- a/pipelines/test.nox.py +++ b/pipelines/test.nox.py @@ -1,6 +1,6 @@ import nox -from pipelines.utils import setup_env +from pipelines.utils import setup_env, maybe_install_nodejs_bin from pipelines.utils.prisma import generate @@ -9,6 +9,7 @@ def test(session: nox.Session) -> None: setup_env(session) session.install('-r', 'pipelines/requirements/test.txt') session.install('.') + maybe_install_nodejs_bin(session) generate(session) diff --git a/pipelines/utils/utils.py b/pipelines/utils/utils.py index 19b35924e..1bb102823 100644 --- a/pipelines/utils/utils.py +++ b/pipelines/utils/utils.py @@ -2,10 +2,12 @@ from pathlib import Path import nox +import distro CACHE_DIR = Path.cwd() / '.cache' TMP_DIR = Path(tempfile.gettempdir()) +PIPELINES_DIR = Path(__file__).parent.parent def get_pkg_location(session: nox.Session, pkg: str) -> str: @@ -44,3 +46,13 @@ def setup_env(session: nox.Session) -> None: ] ] ) + + +def maybe_install_nodejs_bin(session: nox.Session) -> bool: + # nodejs-bin is not available on alpine yet, we need to wait until this fix is released: + # https://github.com/samwillis/nodejs-pypi/issues/11 + if distro.id() == 'alpine': + return False + + session.install('-r', str(PIPELINES_DIR / 'requirements/node.txt')) + return True diff --git a/requirements/base.txt b/requirements/base.txt index d13116ded..961004dbb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,4 +5,5 @@ click>=7.1.2 python-dotenv>=0.12.0 typing-extensions>=3.7 tomlkit +nodeenv cached-property; python_version < '3.8' diff --git a/requirements/node.txt b/requirements/node.txt new file mode 100644 index 000000000..2339a304f --- /dev/null +++ b/requirements/node.txt @@ -0,0 +1 @@ +nodejs-bin diff --git a/scripts/docs.py b/scripts/docs.py index 3a6d9a0a6..5c03e9ecf 100644 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -46,21 +46,13 @@ def main() -> None: r'\1' + '`' + config.prisma_version + '`', content, ) + content = re.sub( - r'(`PRISMA_ENGINE_VERSION` \| )`(.*)`', - r'\1' + '`' + config.engine_version + '`', - content, - ) - content = re.sub( - r'(`PRISMA_CLI_URL` \| )`(.*)`', - r'\1' + '`' + config.prisma_url + '`', - content, - ) - content = re.sub( - r'(`PRISMA_ENGINE_URL` \| )`(.*)`', - r'\1' + '`' + config.engine_url + '`', + r'(`PRISMA_EXPECTED_ENGINE_VERSION` \| )`(.*)`', + r'\1' + '`' + config.expected_engine_version + '`', content, ) + config_doc.write_text(content) diff --git a/setup.py b/setup.py index baa6fad68..7a0e7c264 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,10 @@ def requirements(name: str) -> List[str]: raise RuntimeError('version is not set') +extras = { + 'node': requirements('node.txt'), +} + setup( name='prisma', version=version, @@ -50,8 +54,10 @@ def requirements(name: str) -> List[str]: include_package_data=True, zip_safe=False, extras_require={ - # we define `all` even though it's empty so that we can add to it in the future - 'all': [], + **extras, + 'all': [ + req for requirements in extras.values() for req in requirements + ], }, entry_points={ 'console_scripts': [ diff --git a/src/prisma/_compat.py b/src/prisma/_compat.py index 6f903313e..d6a822ce1 100644 --- a/src/prisma/_compat.py +++ b/src/prisma/_compat.py @@ -3,6 +3,7 @@ from asyncio import get_running_loop as get_running_loop from ._types import CallableT +from .utils import make_optional if TYPE_CHECKING: @@ -49,6 +50,17 @@ def validator( from functools import cached_property as cached_property +if TYPE_CHECKING: + import nodejs as _nodejs + + nodejs = make_optional(_nodejs) +else: + try: + import nodejs + except ImportError: + nodejs = None + + def removeprefix(string: str, prefix: str) -> str: if string.startswith(prefix): return string[len(prefix) :] diff --git a/src/prisma/_config.py b/src/prisma/_config.py index 094554429..8065abad4 100644 --- a/src/prisma/_config.py +++ b/src/prisma/_config.py @@ -1,8 +1,7 @@ from __future__ import annotations -import tempfile from pathlib import Path -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Union, Optional, List import tomlkit from pydantic import BaseSettings, Extra, Field @@ -23,21 +22,17 @@ class DefaultConfig(BaseSettings): ) # Engine binary versions can be found under https://github.com/prisma/prisma-engine/commits/main - engine_version: str = Field( - env='PRISMA_ENGINE_VERSION', + expected_engine_version: str = Field( + env='PRISMA_EXPECTED_ENGINE_VERSION', default='efdf9b1183dddfd4258cd181a72125755215ab7b', ) - # CLI binaries are stored here - prisma_url: str = Field( - env='PRISMA_CLI_URL', - default='https://prisma-photongo.s3-eu-west-1.amazonaws.com/prisma-cli-{version}-{platform}.gz', - ) - - # Engine binaries are stored here - engine_url: str = Field( - env='PRISMA_ENGINE_URL', - default='https://binaries.prisma.sh/all_commits/{0}/{1}/{2}.gz', + # Home directory, used to build the `binary_cache_dir` option by default, useful in multi-user + # or testing environments so that the binaries can be easily cached without having to worry + # about versioning them. + home_dir: Path = Field( + env='PRISMA_HOME_DIR', + default=Path.home(), ) # Where to store the downloaded binaries @@ -46,6 +41,32 @@ class DefaultConfig(BaseSettings): default=None, ) + # Workaround to support setting the binary platform until it can be properly implemented + binary_platform: Optional[str] = Field( + env='PRISMA_BINARY_PLATFORM', default=None + ) + + # Whether or not to use the global node installation (if available) + use_global_node: bool = Field(env='PRISMA_USE_GLOBAL_NODE', default=True) + + # Whether or not to use the `nodejs-bin` package (if installed) + use_nodejs_bin: bool = Field(env='PRISMA_USE_NODEJS_BIN', default=True) + + # Extra arguments to pass to nodeenv, arguments are passed after the path, e.g. python -m nodeenv + nodeenv_extra_args: List[str] = Field( + env='PRISMA_NODEENV_EXTRA_ARGS', + default_factory=list, + ) + + # Where to download nodeenv to, defaults to ~/.cache/prisma-python/nodeenv + nodeenv_cache_dir: Path = Field( + env='PRISMA_NODEENV_CACHE_DIR', + default_factory=lambda: Path.home() + / '.cache' + / 'prisma-python' + / 'nodeenv', + ) + class Config(BaseSettings.Config): extra: Extra = Extra.ignore @@ -67,11 +88,12 @@ class Config(DefaultConfig): def from_base(cls, config: DefaultConfig) -> Config: if config.binary_cache_dir is None: config.binary_cache_dir = ( - Path(tempfile.gettempdir()) - / 'prisma' + config.home_dir + / '.cache' + / 'prisma-python' / 'binaries' - / 'engines' - / config.engine_version + / config.prisma_version + / config.expected_engine_version ) return cls.parse_obj(config.dict()) @@ -90,7 +112,11 @@ def load(cls, path: Path | None = None) -> Config: else: config = {} - return cls.from_base(DefaultConfig.parse_obj(config)) + return cls.parse(**config) + + @classmethod + def parse(cls, **kwargs: object) -> Config: + return cls.from_base(DefaultConfig.parse_obj(kwargs)) class LazyConfigProxy(LazyProxy[Config]): diff --git a/src/prisma/_proxy.py b/src/prisma/_proxy.py index 443e0f828..a70fd821c 100644 --- a/src/prisma/_proxy.py +++ b/src/prisma/_proxy.py @@ -16,18 +16,18 @@ def __init__(self) -> None: self.__proxied: T | None = None def __getattr__(self, attr: str) -> object: - return getattr(self.__get_proxied(), attr) + return getattr(self.__get_proxied__(), attr) def __repr__(self) -> str: - return repr(self.__get_proxied()) + return repr(self.__get_proxied__()) def __str__(self) -> str: - return str(self.__get_proxied()) + return str(self.__get_proxied__()) def __dir__(self) -> Iterable[str]: - return self.__get_proxied().__dir__() + return self.__get_proxied__().__dir__() - def __get_proxied(self) -> T: + def __get_proxied__(self) -> T: proxied = self.__proxied if proxied is not None: return proxied @@ -35,6 +35,9 @@ def __get_proxied(self) -> T: self.__proxied = proxied = self.__load__() return proxied + def __set_proxied__(self, value: T) -> None: + self.__proxied = value + def __as_proxied__(self) -> T: """Helper method that returns the current proxy, typed as the loaded object""" return cast(T, self) diff --git a/src/prisma/binaries/__init__.py b/src/prisma/binaries/__init__.py index af30b1f84..65b66fea3 100644 --- a/src/prisma/binaries/__init__.py +++ b/src/prisma/binaries/__init__.py @@ -1,5 +1,3 @@ # -*- coding: utf-8 -*- -from .engine import * -from .binaries import * from .constants import * diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py deleted file mode 100644 index deaa88e04..000000000 --- a/src/prisma/binaries/binaries.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -import logging -from pathlib import Path -from typing import Optional, List - -import click - -from .binary import Binary -from .engine import Engine -from .constants import PRISMA_CLI_NAME -from .. import config - - -__all__ = ( - 'ENGINES', - 'BINARIES', - 'ensure_cached', - 'remove_all', -) - -log: logging.Logger = logging.getLogger(__name__) - -ENGINES = [ - Engine(name='query-engine', env='PRISMA_QUERY_ENGINE_BINARY'), - Engine(name='migration-engine', env='PRISMA_MIGRATION_ENGINE_BINARY'), - Engine( - name='introspection-engine', env='PRISMA_INTROSPECTION_ENGINE_BINARY' - ), - Engine(name='prisma-fmt', env='PRISMA_FMT_BINARY'), -] - -BINARIES: List[Binary] = [ - *ENGINES, - Binary(name=PRISMA_CLI_NAME, env='PRISMA_CLI_BINARY'), -] - - -def ensure_cached() -> Path: - binaries: List[Binary] = [] - for binary in BINARIES: - path = binary.path - if path.exists(): - log.debug('%s cached at %s', binary.name, path) - else: - log.debug('%s not cached at %s', binary.name, path) - binaries.append(binary) - - if not binaries: - log.debug('All binaries are cached') - return config.binary_cache_dir - - def show_item(item: Optional[Binary]) -> str: - if item is not None: - return binary.name - return '' - - with click.progressbar( - binaries, - label='Downloading binaries', - fill_char=click.style('#', fg='yellow'), - item_show_func=show_item, - ) as iterator: - for binary in iterator: - binary.download() - - return config.binary_cache_dir - - -def remove_all() -> None: - """Remove all downloaded binaries""" - for binary in BINARIES: - if binary.path.exists(): - binary.path.unlink() diff --git a/src/prisma/binaries/binary.py b/src/prisma/binaries/binary.py deleted file mode 100644 index 7f5ab7386..000000000 --- a/src/prisma/binaries/binary.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import logging -from pathlib import Path -from pydantic import BaseModel - -from . import platform -from .utils import download -from .. import config - - -__all__ = ('Binary',) - -log: logging.Logger = logging.getLogger(__name__) - - -class Binary(BaseModel): - name: str - env: str - - def download(self) -> None: - # TODO: respect schema binary options - url = self.url - dest = self.path - - if dest.exists(): - log.debug('%s is cached, skipping download', self.name) - return - - log.debug('Downloading from %s to %s', url, dest) - download(url, str(dest.absolute())) - log.debug('Downloaded %s to %s', self.name, dest.absolute()) - - @property - def url(self) -> str: - return platform.check_for_extension(config.prisma_url).format( - version=config.prisma_version, platform=platform.name() - ) - - @property - def path(self) -> Path: - env = os.environ.get(self.env) - if env is not None: - log.debug( - 'Using environment variable location: %s for %s', - env, - self.name, - ) - return Path(env) - - return config.binary_cache_dir.joinpath( - platform.check_for_extension(self.name) - ) diff --git a/src/prisma/binaries/engine.py b/src/prisma/binaries/engine.py deleted file mode 100644 index 7ecaa09e4..000000000 --- a/src/prisma/binaries/engine.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import logging -from pathlib import Path - -from . import platform -from .binary import Binary -from .. import config - - -__all__ = ('Engine',) - -log: logging.Logger = logging.getLogger(__name__) - - -class Engine(Binary): - name: str - env: str - - @property - def url(self) -> str: - return platform.check_for_extension( - config.engine_url.format( - config.engine_version, platform.binary_platform(), self.name - ) - ) - - @property - def path(self) -> Path: - env = os.environ.get(self.env) - if env is not None: - log.debug( - 'Using environment variable location: %s for %s', - env, - self.name, - ) - return Path(env) - - binary_name = platform.binary_platform() - return Path( - platform.check_for_extension( - str( - config.binary_cache_dir.joinpath( - f'prisma-{self.name}-{binary_name}' - ) - ) - ) - ) diff --git a/src/prisma/binaries/utils.py b/src/prisma/binaries/utils.py deleted file mode 100644 index 7c1995f68..000000000 --- a/src/prisma/binaries/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import gzip -import shutil -from pathlib import Path - -from .._sync_http import client - - -def download(url: str, to: str) -> None: - Path(to).parent.mkdir(parents=True, exist_ok=True) - - tmp = to + '.tmp' - tar = to + '.gz.tmp' - client.download(url, tar) - - # decompress to a tmp file before replacing the original - with gzip.open(tar, 'rb') as f_in: - with open(tmp, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - # chmod +x - status = os.stat(tmp) - os.chmod(tmp, status.st_mode | 0o111) - - # override the original - shutil.copy(tmp, to) - - # remove temporary files - os.remove(tar) - os.remove(tmp) diff --git a/src/prisma/cli/__init__.py b/src/prisma/cli/__init__.py index a5bd848cf..87218c44e 100644 --- a/src/prisma/cli/__init__.py +++ b/src/prisma/cli/__init__.py @@ -1 +1,2 @@ from .cli import * +from ._node import UnknownTargetError as UnknownTargetError diff --git a/src/prisma/cli/_node.py b/src/prisma/cli/_node.py new file mode 100644 index 000000000..9a4f2ddd9 --- /dev/null +++ b/src/prisma/cli/_node.py @@ -0,0 +1,405 @@ +from __future__ import annotations + +import re +import os +import sys +import shutil +import logging +import subprocess +from pathlib import Path +from abc import ABC, abstractmethod +from typing import IO, Union, Any, Mapping, cast +from typing_extensions import Literal + +from pydantic.typing import get_args + +from .. import config +from .._proxy import LazyProxy +from ..binaries import platform +from ..errors import PrismaError +from .._compat import nodejs + + +log: logging.Logger = logging.getLogger(__name__) +File = Union[int, IO[Any]] +Target = Literal['node', 'npm'] + +# taken from https://github.com/prisma/prisma/blob/main/package.json +MIN_NODE_VERSION = (14, 17) + +# mapped the node version above from https://nodejs.org/en/download/releases/ +MIN_NPM_VERSION = (6, 14) + +# we only care about the first two entries in the version number +VERSION_RE = re.compile(r'v?(\d+)(?:\.?(\d+))') + + +# TODO: remove the possibility to get mismatched paths for `node` and `npm` + + +class UnknownTargetError(PrismaError): + def __init__(self, *, target: str) -> None: + super().__init__( + f'Unknown target: {target}; Valid choices are: {", ".join(get_args(cast(type, Target)))}' + ) + + +# TODO: add tests for this error +class MissingNodejsBinError(PrismaError): + def __init__(self) -> None: + super().__init__( + 'Attempted to access a function that requires the `nodejs-bin` package to be installed but it is not.' + ) + + +class Strategy(ABC): + resolver: Literal['nodejs-bin', 'global', 'nodeenv'] + + # TODO: support more options + def run( + self, + *args: str, + check: bool = False, + cwd: Path | None = None, + stdout: File | None = None, + stderr: File | None = None, + env: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[bytes]: + """Call the underlying Node.js binary. + + The interface for this function is very similar to `subprocess.run()`. + """ + return self.__run__( + *args, + check=check, + cwd=cwd, + stdout=stdout, + stderr=stderr, + env=_update_path_env( + env=env, + target_bin=self.target_bin, + ), + ) + + @abstractmethod + def __run__( + self, + *args: str, + check: bool = False, + cwd: Path | None = None, + stdout: File | None = None, + stderr: File | None = None, + env: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[bytes]: + """Call the underlying Node.js binary. + + This should not be directly accessed, the `run()` function should be used instead. + """ + + @property + @abstractmethod + def target_bin(self) -> Path: + """Property containing the location of the `bin` directory for the resolved node installation. + + This is used to dynamically alter the `PATH` environment variable to give the appearance that Node + is installed globally on the machine as this is a requirement of Prisma's installation step, see this + comment for more context: https://github.com/RobertCraigie/prisma-client-py/pull/454#issuecomment-1280059779 + """ + ... + + +class NodeBinaryStrategy(Strategy): + target: Target + resolver: Literal['global', 'nodeenv'] + + def __init__( + self, + *, + path: Path, + target: Target, + resolver: Literal['global', 'nodeenv'], + ) -> None: + self.path = path + self.target = target + self.resolver = resolver + + @property + def target_bin(self) -> Path: + return self.path.parent + + def __run__( + self, + *args: str, + check: bool = False, + cwd: Path | None = None, + stdout: File | None = None, + stderr: File | None = None, + env: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[bytes]: + path = str(self.path.absolute()) + log.debug('Executing binary at %s with args: %s', path, args) + return subprocess.run( + [path, *args], + check=check, + cwd=cwd, + env=env, + stdout=stdout, + stderr=stderr, + ) + + @classmethod + def resolve(cls, target: Target) -> NodeBinaryStrategy: + path = None + if config.use_global_node: + path = _get_global_binary(target) + + if path is not None: + return NodeBinaryStrategy( + path=path, + target=target, + resolver='global', + ) + + return NodeBinaryStrategy.from_nodeenv(target) + + @classmethod + def from_nodeenv(cls, target: Target) -> NodeBinaryStrategy: + cache_dir = config.nodeenv_cache_dir.absolute() + if cache_dir.exists(): + log.debug( + 'Skipping nodeenv installation as it already exists at %s', + cache_dir, + ) + else: + log.debug('Installing nodeenv to %s', cache_dir) + subprocess.run( + [ + sys.executable, + '-m', + 'nodeenv', + str(cache_dir), + *config.nodeenv_extra_args, + ], + check=True, + stdout=sys.stdout, + stderr=sys.stderr, + ) + + if not cache_dir.exists(): + raise RuntimeError( + 'Could not install nodeenv to the expected directory; See the output above for more details.' + ) + + # TODO: what hapens on cygwin? + if platform.name() == 'windows': + bin_dir = cache_dir / 'Scripts' + if target == 'node': + path = bin_dir / 'node.exe' + else: + path = bin_dir / f'{target}.cmd' + else: + path = cache_dir / 'bin' / target + + if target == 'npm': + return cls(path=path, resolver='nodeenv', target=target) + elif target == 'node': + return cls(path=path, resolver='nodeenv', target=target) + else: + raise UnknownTargetError(target=target) + + +class NodeJSPythonStrategy(Strategy): + target: Target + resolver: Literal['nodejs-bin'] + + def __init__(self, *, target: Target) -> None: + self.target = target + self.resolver = 'nodejs-bin' + + def __run__( + self, + *args: str, + check: bool = False, + cwd: Path | None = None, + stdout: File | None = None, + stderr: File | None = None, + env: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[bytes]: + if nodejs is None: + raise MissingNodejsBinError() + + func = None + if self.target == 'node': + func = nodejs.node.run + elif self.target == 'npm': + func = nodejs.npm.run + else: + raise UnknownTargetError(target=self.target) + + return cast( + 'subprocess.CompletedProcess[bytes]', + func( + args, + check=check, + cwd=cwd, + env=env, + stdout=stdout, + stderr=stderr, + ), + ) + + @property + def node_path(self) -> Path: + """Returns the path to the `node` binary""" + if nodejs is None: + raise MissingNodejsBinError() + + return Path(nodejs.node.path) + + @property + def target_bin(self) -> Path: + return Path(self.node_path).parent + + +Node = Union[NodeJSPythonStrategy, NodeBinaryStrategy] + + +def resolve(target: Target) -> Node: + if target not in {'node', 'npm'}: + raise UnknownTargetError(target=target) + + if config.use_nodejs_bin: + log.debug('Checking if nodejs-bin is installed') + if nodejs is not None: + log.debug('Using nodejs-bin with version: %s', nodejs.node_version) + return NodeJSPythonStrategy(target=target) + + return NodeBinaryStrategy.resolve(target) + + +def _update_path_env( + *, + env: Mapping[str, str] | None, + target_bin: Path, + sep: str = os.pathsep, +) -> dict[str, str]: + """Returns a modified version of `os.environ` with the `PATH` environment variable updated + to include the location of the downloaded Node binaries. + """ + if env is None: + env = dict(os.environ) + + log.debug('Attempting to preprend %s to the PATH', target_bin) + assert target_bin.exists(), 'Target `bin` directory does not exist' + + path = env.get('PATH', '') or os.environ.get('PATH', '') + if path: + # handle the case where the PATH already starts with the separator (this probably shouldn't happen) + if path.startswith(sep): + path = f'{target_bin.absolute()}{path}' + else: + path = f'{target_bin.absolute()}{sep}{path}' + else: + # handle the case where there is no PATH set (unlikely / impossible to actually happen?) + path = str(target_bin.absolute()) + + log.debug('Using PATH environment variable: %s', path) + return {**env, 'PATH': path} + + +def _get_global_binary(target: Target) -> Path | None: + """Returns the path to a globally installed binary. + + This also ensures that the binary is of the right version. + """ + log.debug('Checking for global target binary: %s', target) + + which = shutil.which(target) + if which is None: + log.debug('Global target binary: %s not found', target) + return None + + log.debug('Found global binary at: %s', which) + + path = Path(which) + if not path.exists(): + log.debug('Global binary does not exist at: %s', which) + return None + + if not _should_use_binary(target=target, path=path): + return None + + log.debug('Using global %s binary at %s', target, path) + return path + + +def _should_use_binary(target: Target, path: Path) -> bool: + """Call the binary at `path` with a `--version` flag to check if it matches our minimum version requirements. + + This only applies to the global node installation as: + + - the minimum version of `nodejs-bin` is higher than our requirement + - `nodeenv` defaults to the latest stable version of node + """ + if target == 'node': + min_version = MIN_NODE_VERSION + elif target == 'npm': + min_version = MIN_NPM_VERSION + else: + raise UnknownTargetError(target=target) + + version = _get_binary_version(target, path) + if version is None: + log.debug( + 'Could not resolve %s version, ignoring global %s installation', + target, + target, + ) + return False + + if version < min_version: + log.debug( + 'Global %s version (%s) is lower than the minimum required version (%s), ignoring', + target, + version, + min_version, + ) + return False + + return True + + +def _get_binary_version(target: Target, path: Path) -> tuple[int, ...] | None: + proc = subprocess.run( + [str(path), '--version'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + log.debug('%s version check exited with code %s', target, proc.returncode) + + output = proc.stdout.decode('utf-8').rstrip('\n') + log.debug('%s version check output: %s', target, output) + + match = VERSION_RE.search(output) + if not match: + return None + + version = tuple(int(value) for value in match.groups()) + log.debug('%s version check returning %s', target, version) + return version + + +class LazyBinaryProxy(LazyProxy[Node]): + target: Target + + def __init__(self, target: Target) -> None: + super().__init__() + self.target = target + + def __load__(self) -> Node: + return resolve(self.target) + + +npm = LazyBinaryProxy('npm').__as_proxied__() +node = LazyBinaryProxy('node').__as_proxied__() diff --git a/src/prisma/cli/commands/fetch.py b/src/prisma/cli/commands/fetch.py index 3fdb5db99..cd3fd64fb 100644 --- a/src/prisma/cli/commands/fetch.py +++ b/src/prisma/cli/commands/fetch.py @@ -1,5 +1,9 @@ +import shutil import click +from ... import config +from ...cli.prisma import ensure_cached + @click.command('fetch', short_help='Download all required binaries.') @click.option( @@ -9,12 +13,10 @@ ) def cli(force: bool) -> None: """Ensures all required binaries are available.""" - from ... import binaries - if force: - binaries.remove_all() + shutil.rmtree(config.binary_cache_dir) - directory = binaries.ensure_cached() + directory = ensure_cached().cache_dir click.echo( f'Downloaded binaries to {click.style(str(directory), fg="green")}' ) diff --git a/src/prisma/cli/commands/version.py b/src/prisma/cli/commands/version.py index 323d34ba0..0a45d69ff 100644 --- a/src/prisma/cli/commands/version.py +++ b/src/prisma/cli/commands/version.py @@ -38,7 +38,7 @@ def cli(output_json: bool) -> None: 'prisma': config.prisma_version, 'prisma client python': __version__, 'platform': binary_platform(), - 'engines': config.engine_version, + 'expected engine version': config.expected_engine_version, 'install path': str(Path(__file__).resolve().parent.parent.parent), 'installed extras': installed, } diff --git a/src/prisma/cli/prisma.py b/src/prisma/cli/prisma.py index bbb42ddea..79db43b85 100644 --- a/src/prisma/cli/prisma.py +++ b/src/prisma/cli/prisma.py @@ -1,13 +1,18 @@ +from __future__ import annotations + import os import sys +import json import logging import subprocess -from textwrap import indent -from typing import List, Optional, Dict +from pathlib import Path +from typing import Any, List, Optional, Dict, NamedTuple import click -from .. import binaries +from ._node import node, npm +from .. import config +from ..errors import PrismaError log: logging.Logger = logging.getLogger(__name__) @@ -18,14 +23,6 @@ def run( check: bool = False, env: Optional[Dict[str, str]] = None, ) -> int: - directory = binaries.ensure_cached() - path = directory.joinpath(binaries.PRISMA_CLI_NAME) - if not path.exists(): - raise RuntimeError( - f'The Prisma CLI is not downloaded, expected {path} to exist.' - ) - - log.debug('Using Prisma CLI at %s', path) log.debug('Running prisma command with args: %s', args) default_env = { @@ -34,47 +31,12 @@ def run( 'PRISMA_CLI_QUERY_ENGINE_TYPE': 'binary', } env = {**default_env, **env} if env is not None else default_env - # ensure the client uses our engine binaries - for engine in binaries.ENGINES: - env[engine.env] = str(engine.path.absolute()) - if args and args[0] == 'studio': - click.echo( - click.style( - 'ERROR: Prisma Studio does not work natively with Prisma Client Python', - fg='red', - ), - ) - click.echo( - '\nThere are two other possible ways to use Prisma Studio:\n' - ) - click.echo( - click.style('1. Download the Prisma Studio app\n', bold=True) - ) - click.echo( - indent( - 'Prisma Studio can be downloaded from: ' - + click.style( - 'https://github.com/prisma/studio/releases', underline=True - ), - ' ' * 3, - ) - ) - click.echo( - click.style('\n2. Use the Node based Prisma CLI:\n', bold=True) - ) - click.echo( - indent( - 'If you have Node / NPX installed you can launch Prisma Studio by running the command: ', - ' ' * 3, - ), - nl=False, - ) - click.echo(click.style('npx prisma studio', bold=True)) - return 1 - - process = subprocess.run( - [str(path.absolute()), *args], + # TODO: ensure graceful termination + entrypoint = ensure_cached().entrypoint + process = node.run( + str(entrypoint), + *args, env=env, check=check, stdout=sys.stdout, @@ -82,12 +44,66 @@ def run( ) if args and args[0] in {'--help', '-h'}: - prefix = ' ' - click.echo(click.style(prefix + 'Python Commands\n', bold=True)) + click.echo(click.style('Python Commands\n', bold=True)) click.echo( - prefix - + 'For Prisma Client Python commands see ' + ' ' + + 'For Prisma Client Python commands run ' + click.style('prisma py --help', bold=True) ) return process.returncode + + +class CLICache(NamedTuple): + cache_dir: Path + entrypoint: Path + + +DEFAULT_PACKAGE_JSON: dict[str, Any] = { + 'name': 'prisma-binaries', + 'version': '1.0.0', + 'private': True, + 'description': 'Cache directory created by Prisma Client Python to store Prisma Engines', + 'main': 'node_modules/prisma/build/index.js', + 'author': 'RobertCraigie', + 'license': 'Apache-2.0', +} + + +def ensure_cached() -> CLICache: + cache_dir = config.binary_cache_dir + entrypoint = cache_dir / 'node_modules' / 'prisma' / 'build' / 'index.js' + + if not cache_dir.exists(): + cache_dir.mkdir(parents=True) + + # We need to create a dummy `package.json` file so that `npm` doesn't try + # and search for it elsewhere. + # + # If it finds a different `package.json` file then the `prisma` package + # will be installed there instead of our cache directory. + package = cache_dir / 'package.json' + if not package.exists(): + package.write_text(json.dumps(DEFAULT_PACKAGE_JSON)) + + if not entrypoint.exists(): + click.echo('Installing Prisma CLI') + proc = npm.run( + 'install', + f'prisma@{config.prisma_version}', + cwd=config.binary_cache_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if proc.returncode != 0: + click.echo( + f'An error ocurred while installing the Prisma CLI; npm install log: {proc.stdout.decode("utf-8")}' + ) + proc.check_returncode() + + if not entrypoint.exists(): + raise PrismaError( + f'CLI installation appeared to complete but the expected entrypoint ({entrypoint}) could not be found.' + ) + + return CLICache(cache_dir=cache_dir, entrypoint=entrypoint) diff --git a/src/prisma/engine/errors.py b/src/prisma/engine/errors.py index d67a3505e..dc3f49b27 100644 --- a/src/prisma/engine/errors.py +++ b/src/prisma/engine/errors.py @@ -41,7 +41,9 @@ class MismatchedVersionsError(EngineError): def __init__(self, *, expected: str, got: str): super().__init__( - f'Expected query engine version `{expected}` but got `{got}`' + f'Expected query engine version `{expected}` but got `{got}`.\n' + + 'If this is intentional then please set the PRISMA_PY_DEBUG_GENERATOR environment ' + + 'variable to 1 and try again.' ) self.expected = expected self.got = got diff --git a/src/prisma/engine/utils.py b/src/prisma/engine/utils.py index f59033f39..535d1c5c2 100644 --- a/src/prisma/engine/utils.py +++ b/src/prisma/engine/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import sys import time @@ -12,11 +14,12 @@ from .. import config from ..http_abstract import AbstractResponse -from ..utils import time_since +from ..utils import DEBUG_GENERATOR, time_since from ..binaries import platform log: logging.Logger = logging.getLogger(__name__) + ERROR_MAPPING: Dict[str, Type[Exception]] = { 'P2002': prisma_errors.UniqueViolationError, 'P2003': prisma_errors.ForeignKeyViolationError, @@ -29,20 +32,51 @@ } -def ensure() -> Path: +def query_engine_name() -> str: + return f'prisma-query-engine-{platform.check_for_extension(platform.binary_platform())}' + + +def _resolve_from_binary_paths(binary_paths: dict[str, str]) -> Path | None: + if config.binary_platform is not None: + return Path(binary_paths[config.binary_platform]) + + paths = [Path(p) for p in binary_paths.values()] + + # fast path for when there are no `binaryTargets` defined + if len(paths) == 1: + return paths[0] + + for path in paths: + # we only want to resolve to binary's that we can run on the current machine. + # because of the `binaryTargets` option some of the binaries we are given may + # not be targeting the same architecture as the current machine + if path.exists() and _can_execute_binary(path): + return path + + # none of the given paths existed or they target a different architecture + return None + + +def _can_execute_binary(path: Path) -> bool: + proc = subprocess.run([str(path), '--version'], check=False) + log.debug( + 'Executable check for %s exited with code: %s', path, proc.returncode + ) + return proc.returncode == 0 + + +def ensure(binary_paths: dict[str, str]) -> Path: start_time = time.monotonic() file = None - force_version = True - binary_name = platform.check_for_extension(platform.binary_platform()) - - name = f'prisma-query-engine-{binary_name}' + force_version = not DEBUG_GENERATOR + name = query_engine_name() local_path = Path.cwd().joinpath(name) global_path = config.binary_cache_dir.joinpath(name) + file_from_paths = _resolve_from_binary_paths(binary_paths) log.debug('Expecting local query engine %s', local_path) log.debug('Expecting global query engine %s', global_path) - # TODO: this resolving should be moved to the binary class binary = os.environ.get('PRISMA_QUERY_ENGINE_BINARY') if binary: log.debug('PRISMA_QUERY_ENGINE_BINARY is defined, using %s', binary) @@ -58,16 +92,29 @@ def ensure() -> Path: elif local_path.exists(): file = local_path log.debug('Query engine found in the working directory') + elif file_from_paths is not None and file_from_paths.exists(): + file = file_from_paths + log.debug( + 'Query engine found from the Prisma CLI generated path: %s', + file_from_paths, + ) elif global_path.exists(): file = global_path log.debug('Query engine found in the global path') if not file: + if file_from_paths is not None: + expected = f'{local_path}, {global_path} or {file_from_paths} to exist but none' + else: + expected = f'{local_path} or {global_path} to exist but neither' + raise errors.BinaryNotFoundError( - f'Expected {local_path} or {global_path} but neither were found.\n' - 'Try running prisma py fetch' + f'Expected {expected} were found.\n' + + 'Try running prisma py fetch' ) + log.debug('Using Query Engine binary at %s', file) + start_version = time.monotonic() process = subprocess.run( [str(file.absolute()), '--version'], stdout=subprocess.PIPE, check=True @@ -81,14 +128,13 @@ def ensure() -> Path: ) log.debug('Using query engine version %s', version) - if force_version and version != config.engine_version: + if force_version and version != config.expected_engine_version: raise errors.MismatchedVersionsError( - expected=config.engine_version, got=version + expected=config.expected_engine_version, got=version ) log.debug('Using query engine at %s', file) log.debug('Ensuring query engine took: %s', time_since(start_time)) - return file diff --git a/src/prisma/generator/generator.py b/src/prisma/generator/generator.py index 2370eb9f1..1c1e3be56 100644 --- a/src/prisma/generator/generator.py +++ b/src/prisma/generator/generator.py @@ -227,6 +227,9 @@ def get_manifest(self) -> Manifest: return Manifest( name=f'Prisma Client Python (v{__version__})', default_output=BASE_PACKAGE_DIR, + requires_engines=[ + 'queryEngine', + ], ) def generate(self, data: PythonData) -> None: diff --git a/src/prisma/generator/jsonrpc.py b/src/prisma/generator/jsonrpc.py index 75f7dabad..3fdb60851 100644 --- a/src/prisma/generator/jsonrpc.py +++ b/src/prisma/generator/jsonrpc.py @@ -5,7 +5,7 @@ import logging from pathlib import Path from typing import Dict, List, Optional, Union, Type, Any -from typing_extensions import TypedDict +from typing_extensions import TypedDict, Literal from pydantic import Field @@ -52,6 +52,14 @@ class ErrorResponse(BaseModel): Response = Union[SuccessResponse, ErrorResponse] +EngineType = Literal[ + 'prismaFmt', + 'queryEngine', + 'libqueryEngine', + 'migrationEngine', + 'introspectionEngine', +] + class Manifest(BaseModel): """Generator metadata""" @@ -59,7 +67,7 @@ class Manifest(BaseModel): prettyName: str = Field(alias='name') defaultOutput: Union[str, Path] = Field(alias='default_output') denylist: Optional[List[str]] = None - requiresEngines: Optional[List[str]] = Field( + requiresEngines: Optional[List[EngineType]] = Field( alias='requires_engines', default=None ) requiresGenerators: Optional[List[str]] = Field( diff --git a/src/prisma/generator/models.py b/src/prisma/generator/models.py index 973894c17..a137839dd 100644 --- a/src/prisma/generator/models.py +++ b/src/prisma/generator/models.py @@ -304,6 +304,9 @@ class GenericData(GenericModel, Generic[ConfigT]): other_generators: List['Generator[_ModelAllowAll]'] = FieldInfo( alias='otherGenerators' ) + binary_paths: 'BinaryPaths' = FieldInfo( + alias='binaryPaths', default_factory=lambda: BinaryPaths() + ) @classmethod def parse_obj(cls, obj: Any) -> 'GenericData[ConfigT]': @@ -334,9 +337,9 @@ def to_params(self) -> Dict[str, Any]: def validate_version(cls, values: Dict[Any, Any]) -> Dict[Any, Any]: # TODO: test this version = values.get('version') - if not DEBUG_GENERATOR and version != config.engine_version: + if not DEBUG_GENERATOR and version != config.expected_engine_version: raise ValueError( - f'Prisma Client Python expected Prisma version: {config.engine_version} ' + f'Prisma Client Python expected Prisma version: {config.expected_engine_version} ' f'but got: {version}\n' ' If this is intentional, set the PRISMA_PY_DEBUG_GENERATOR environment ' 'variable to 1 and try again.\n' @@ -347,6 +350,46 @@ def validate_version(cls, values: Dict[Any, Any]) -> Dict[Any, Any]: return values +class BinaryPaths(BaseModel): + """This class represents the paths to engine binaries. + + Each property in this class is a mapping of platform name to absolute path, for example: + + ```py + # This is what will be set on an M1 chip if there are no other `binaryTargets` set + binary_paths.query_engine == { + 'darwin-arm64': '/Users/robert/.cache/prisma-python/binaries/3.13.0/efdf9b1183dddfd4258cd181a72125755215ab7b/node_modules/prisma/query-engine-darwin-arm64' + } + ``` + + This is only available if the generator explicitly requests them using the `requires_engines` manifest property. + """ + + query_engine: Dict[str, str] = FieldInfo( + default_factory=dict, + alias='queryEngine', + ) + introspection_engine: Dict[str, str] = FieldInfo( + default_factory=dict, + alias='introspectionEngine', + ) + migration_engine: Dict[str, str] = FieldInfo( + default_factory=dict, + alias='migrationEngine', + ) + libquery_engine: Dict[str, str] = FieldInfo( + default_factory=dict, + alias='libqueryEngine', + ) + prisma_format: Dict[str, str] = FieldInfo( + default_factory=dict, + alias='prismaFmt', + ) + + class Config(BaseModel.Config): + extra: Extra = Extra.ignore + + class Datasource(BaseModel): # TODO: provider enums name: str @@ -368,11 +411,12 @@ class Generator(GenericModel, Generic[ConfigT]): def warn_binary_targets( cls, targets: List['ValueFromEnvVar'] ) -> List['ValueFromEnvVar']: - if targets and any(target.value != 'native' for target in targets): + # Prisma by default sends one binary target which is the current platform. + if len(targets) > 1: click.echo( click.style( 'Warning: ' - 'The binaryTargets option is not currently supported by Prisma Client Python', + + 'The binaryTargets option is not officially supported by Prisma Client Python.', fg='yellow', ), file=sys.stdout, diff --git a/src/prisma/generator/templates/client.py.jinja b/src/prisma/generator/templates/client.py.jinja index 37ae02b9b..0058c040e 100644 --- a/src/prisma/generator/templates/client.py.jinja +++ b/src/prisma/generator/templates/client.py.jinja @@ -9,13 +9,14 @@ from .types import DatasourceOverride, HttpConfig from ._types import BaseModelT from .engine import AbstractEngine, QueryEngine from .builder import QueryBuilder -from .generator.models import EngineType, OptionalValueFromEnvVar +from .generator.models import EngineType, OptionalValueFromEnvVar, BinaryPaths from ._compat import removeprefix __all__ = ( 'ENGINE_TYPE', 'SCHEMA_PATH', + 'BINARY_PATHS', 'Batch', 'Prisma', 'Client', @@ -26,8 +27,8 @@ __all__ = ( SCHEMA_PATH = Path('{{ schema_path.as_posix() }}') PACKAGED_SCHEMA_PATH = Path(__file__).parent.joinpath('schema.prisma') - ENGINE_TYPE: EngineType = EngineType.{{ generator.config.engine_type }} +BINARY_PATHS = BinaryPaths.parse_obj({{ binary_paths.dict(by_alias=True) }}) RegisteredClient = Union['Prisma', Callable[[], 'Prisma']] _registered_client: Optional[RegisteredClient] = None diff --git a/src/prisma/generator/templates/engine/query.py.jinja b/src/prisma/generator/templates/engine/query.py.jinja index 8154e0e4b..0413aa050 100644 --- a/src/prisma/generator/templates/engine/query.py.jinja +++ b/src/prisma/generator/templates/engine/query.py.jinja @@ -14,6 +14,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine +from .. import config from ..utils import DEBUG from ..binaries import platform from ..utils import time_since, _env_bool @@ -70,6 +71,12 @@ class QueryEngine(HTTPEngine): if self.session and not self.session.closed: {{ maybe_await }}self.session.close() + def _ensure_file(self) -> Path: + # circular import + from ..client import BINARY_PATHS + + return utils.ensure(BINARY_PATHS.query_engine) + {{ maybe_async_def }}connect( self, timeout: int = 10, @@ -80,7 +87,7 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + self.file = file = self._ensure_file() try: {{ maybe_await }}self.spawn(file, timeout=timeout, datasources=datasources) diff --git a/src/prisma/utils.py b/src/prisma/utils.py index ca4ac7e4b..72e1e9b06 100644 --- a/src/prisma/utils.py +++ b/src/prisma/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import time import asyncio @@ -124,3 +126,12 @@ def assert_never(value: NoReturn) -> NoReturn: assert False, 'Unhandled type: {}'.format( type(value).__name__ ) # pragma: no cover + + +def make_optional(value: _T) -> _T | None: + """Helper function for type checkers to change the given type to include None. + + This is useful in cases where you do not have an explicit type for a symbol (e.g. modules) + but want to mark it as potentially None. + """ + return value diff --git a/tests/Dockerfile b/tests/Dockerfile index 3cf55da0a..fdc878b7a 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -58,3 +58,13 @@ RUN prisma -v # Very light-weight test that CLI generation works RUN prisma generate --schema ./tests/data/schema.prisma + +# Ensure all combinations of non-global Node resolvers work +RUN nox -s test -p 3.10 -- tests/test_node + +# Ensure database access using SQLite works +ENV SQLITE_URL="file:sqlite.db" + +# We don't run linters here because pyright-python is currently broken in certain docker images +# TODO: lint when fixed +RUN nox -s databases -- test --databases=sqlite --no-lint diff --git a/tests/integrations/sync/tests/test_binary.py b/tests/integrations/sync/tests/test_binary.py deleted file mode 100644 index 3e4d080b9..000000000 --- a/tests/integrations/sync/tests/test_binary.py +++ /dev/null @@ -1,14 +0,0 @@ -import random -from prisma.binaries import BINARIES - - -def test_download() -> None: - """Binary can be downloaded""" - binary = random.choice(BINARIES) - assert binary.path.exists() - binary.path.unlink() - assert not binary.path.exists() - - binary.download() - - assert binary.path.exists() diff --git a/tests/test_binaries.py b/tests/test_binaries.py deleted file mode 100644 index 244366fb8..000000000 --- a/tests/test_binaries.py +++ /dev/null @@ -1,31 +0,0 @@ -from pathlib import Path - -import pytest -from _pytest.logging import LogCaptureFixture - -from prisma.utils import temp_env_update -from prisma.binaries import BINARIES, ENGINES, Engine -from prisma.binaries.constants import PRISMA_CLI_NAME - - -def test_skips_cached_binary(caplog: LogCaptureFixture) -> None: - """Downloading an already existing binary does not actually do anything""" - # NOTE: this is not a great way to test this - binary = BINARIES[0] - binary.download() - assert 'is cached' in caplog.records[0].message - - -@pytest.mark.parametrize('engine', ENGINES) -def test_engine_resolves_env_override(engine: Engine) -> None: - """Env variables override the default path for an engine binary""" - with temp_env_update({engine.env: 'foo'}): - assert engine.path == Path('foo') - - -def test_cli_binary_resolves_env_override() -> None: - """Env variable overrides the default path for the CLI binary""" - binary = BINARIES[-1] - assert binary.name == PRISMA_CLI_NAME - with temp_env_update({binary.env: 'foo'}): - assert binary.path == Path('foo') diff --git a/tests/test_cli/test_cli.py b/tests/test_cli/test_cli.py index 52fb78271..0221f68cd 100644 --- a/tests/test_cli/test_cli.py +++ b/tests/test_cli/test_cli.py @@ -47,7 +47,7 @@ def test_outputs_custom_commands_info(runner: Runner, args: List[str]) -> None: result = runner.invoke(args) assert 'Python Commands' in result.output assert ( - 'For Prisma Client Python commands see prisma py --help' + 'For Prisma Client Python commands run prisma py --help' in result.output ) @@ -97,19 +97,3 @@ def cli(argument: str) -> None: assert 'bob' in result.output assert 'alice' in result.output assert 'invalid' in result.output - - -def test_prisma_studio_not_supported_error(runner: Runner) -> None: - """Running `prisma studio` from the Python CLI is not supported: - - https://github.com/prisma/prisma/issues/10917 - - Ensure we provide an easy to understand error message detailing - potential solutions - """ - result = runner.invoke(['studio']) - assert result.exit_code == 1 - assert ( - 'ERROR: Prisma Studio does not work natively with Prisma Client Python' - in result.output - ) diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index 67ccaa358..5b065db6f 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -1,14 +1,10 @@ -import random -import shutil +from pathlib import Path from click.testing import Result -from prisma import binaries, config -from tests.utils import Runner, skipif_windows - - -# TODO: this could probably mess up other tests if one of these -# tests fails mid run, as the global binaries are deleted +from prisma import config +from prisma._config import Config +from ..utils import Runner, set_config def assert_success(result: Result) -> None: @@ -17,54 +13,13 @@ def assert_success(result: Result) -> None: f'Downloaded binaries to {config.binary_cache_dir}\n' ) - for binary in binaries.BINARIES: - assert binary.path.exists() - def test_fetch(runner: Runner) -> None: """Basic usage, binaries are already cached""" assert_success(runner.invoke(['py', 'fetch'])) -# it seems like we can't use `.unlink()` on binary paths on windows due to permissions errors - - -@skipif_windows -def test_fetch_one_binary_missing(runner: Runner) -> None: - """Downloads a binary if it is missing""" - binary = random.choice(binaries.BINARIES) - assert binary.path.exists() - binary.path.unlink() - assert not binary.path.exists() - - assert_success(runner.invoke(['py', 'fetch'])) - - -@skipif_windows -def test_fetch_force(runner: Runner) -> None: - """Passing --force re-downloads an already existing binary""" - binary = random.choice(binaries.BINARIES) - assert binary.path.exists() - old_stat = binary.path.stat() - - assert_success(runner.invoke(['py', 'fetch', '--force'])) - - new_stat = binary.path.stat() - - # modified time - assert old_stat.st_mtime_ns != new_stat.st_mtime_ns - - # ensure downloaded the same as before - assert old_stat.st_size == new_stat.st_size - - -@skipif_windows -def test_fetch_force_no_dir(runner: Runner) -> None: - """Passing --force when the base directory does not exist""" - binaries.remove_all() - shutil.rmtree(str(config.binary_cache_dir)) - - binary = binaries.BINARIES[0] - assert not binary.path.exists() - - assert_success(runner.invoke(['py', 'fetch', '--force'])) +def test_fetch_not_cached(runner: Runner, tmp_path: Path) -> None: + """Basic usage, binaries are not cached""" + with set_config(Config.parse(binary_cache_dir=tmp_path)): + assert_success(runner.invoke(['py', 'fetch'])) diff --git a/tests/test_cli/test_prisma.py b/tests/test_cli/test_prisma.py new file mode 100644 index 000000000..daa8d7469 --- /dev/null +++ b/tests/test_cli/test_prisma.py @@ -0,0 +1,21 @@ +from pathlib import Path + +from prisma.cli import prisma +from prisma._config import Config + +from ..utils import set_config + + +def test_package_json_in_parent_dir(tmp_path: Path) -> None: + """The CLI can be installed successfully when there is a `package.json` file + in a parent directory. + """ + tmp_path.joinpath('package.json').write_text('{"name": "prisma-binaries"}') + cache_dir = tmp_path / 'foo' / 'bar' + + with set_config( + Config.parse( + binary_cache_dir=cache_dir, + ) + ): + assert prisma.run(['-v']) == 0 diff --git a/tests/test_cli/test_version.py b/tests/test_cli/test_version.py index 1029c2fbe..dcc5e6153 100644 --- a/tests/test_cli/test_version.py +++ b/tests/test_cli/test_version.py @@ -10,12 +10,12 @@ PLACEHOLDER = re.compile(r'.*') SEMANTIC_VERSION = re.compile(r'(\d?\d\.){2}\d?\da?') PATTERN = re.compile( - f'prisma : (?P{SEMANTIC_VERSION.pattern})\n' - f'prisma client python : (?P{SEMANTIC_VERSION.pattern})\n' - f'platform : (?P{PLACEHOLDER.pattern})\n' - f'engines : (?P{HASH.pattern})\n' - f'install path : (?P{PLACEHOLDER.pattern})\n' - f'installed extras : (?P{PLACEHOLDER.pattern})' + f'prisma : (?P{SEMANTIC_VERSION.pattern})\n' + f'prisma client python : (?P{SEMANTIC_VERSION.pattern})\n' + f'platform : (?P{PLACEHOLDER.pattern})\n' + f'expected engine version : (?P{HASH.pattern})\n' + f'install path : (?P{PLACEHOLDER.pattern})\n' + f'installed extras : (?P{PLACEHOLDER.pattern})' ) @@ -35,7 +35,7 @@ def test_version_json(runner: Runner) -> None: assert SEMANTIC_VERSION.match(data['prisma']) assert SEMANTIC_VERSION.match(data['prisma-client-python']) assert PLACEHOLDER.match(data['platform']) - assert HASH.match(data['engines']) + assert HASH.match(data['expected-engine-version']) assert isinstance(data['installed-extras'], list) diff --git a/tests/test_config.py b/tests/test_config.py index 588bbe4b4..c16e32f52 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,7 +17,7 @@ def test_lazy_proxy(mocker: MockerFixture) -> None: mocked.assert_called_once() for _ in range(10): - print(proxy.prisma_url) + print(proxy.expected_engine_version) mocked.assert_called_once() @@ -39,7 +39,6 @@ def test_loading(testdir: Testdir) -> None: """ [tool.prisma] prisma_version = '0.1.2.3' - engine_version = 'foo' """ ), ) @@ -47,7 +46,7 @@ def test_loading(testdir: Testdir) -> None: assert config.prisma_version == '0.1.2.3' # updated options are used in computed options - assert 'foo' in str(config.binary_cache_dir) + assert '0.1.2.3' in str(config.binary_cache_dir) def test_allows_extra_keys(testdir: Testdir) -> None: diff --git a/tests/test_engine.py b/tests/test_engine.py index 6f3e12282..bef74144b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -7,10 +7,9 @@ from _pytest.monkeypatch import MonkeyPatch from pytest_subprocess import FakeProcess -from prisma import Prisma, config +from prisma import Prisma, config, BINARY_PATHS from prisma.utils import temp_env_update from prisma.binaries import platform -from prisma.binaries import BINARIES from prisma.engine import errors, utils from prisma.engine.query import QueryEngine from prisma._compat import get_running_loop @@ -18,11 +17,6 @@ from .utils import Testdir -QUERY_ENGINE = next( # pragma: no branch - b for b in BINARIES if b.name == 'query-engine' -) - - @contextlib.contextmanager def no_event_loop() -> Iterator[None]: try: @@ -70,25 +64,46 @@ def mock_exists(path: Path) -> bool: monkeypatch.setattr(Path, 'exists', mock_exists, raising=True) with pytest.raises(errors.BinaryNotFoundError) as exc: - utils.ensure() + utils.ensure(BINARY_PATHS.query_engine) + + assert exc.match( + r'Expected .*, .* or .* to exist but none were found\.\nTry running prisma py fetch' + ) + + +def test_engine_binary_does_not_exist_no_binary_paths( + monkeypatch: MonkeyPatch, +) -> None: + """No query engine binary found raises an error""" + + def mock_exists(path: Path) -> bool: + return False + + monkeypatch.setattr(Path, 'exists', mock_exists, raising=True) + + with pytest.raises(errors.BinaryNotFoundError) as exc: + utils.ensure({}) assert exc.match( - r'Expected .* or .* but neither were found\.\nTry running prisma py fetch' + r'Expected .* or .* to exist but neither were found\.\nTry running prisma py fetch' ) def test_mismatched_version_error(fake_process: FakeProcess) -> None: """Mismatched query engine versions raises an error""" fake_process.register_subprocess( - [str(QUERY_ENGINE.path), '--version'], + [ + str(utils._resolve_from_binary_paths(BINARY_PATHS.query_engine)), + '--version', + ], stdout='query-engine unexpected-hash', ) with pytest.raises(errors.MismatchedVersionsError) as exc: - utils.ensure() + utils.ensure(BINARY_PATHS.query_engine) assert exc.match( - f'Expected query engine version `{config.engine_version}` but got `unexpected-hash`' + f'Expected query engine version `{config.expected_engine_version}` but got `unexpected-hash`' ) @@ -106,13 +121,13 @@ def test_ensure_local_path( stdout='query-engine a-different-hash', ) with pytest.raises(errors.MismatchedVersionsError): - path = utils.ensure() + path = utils.ensure(BINARY_PATHS.query_engine) fake_process.register_subprocess( [str(fake_engine), '--version'], - stdout=f'query-engine {config.engine_version}', + stdout=f'query-engine {config.expected_engine_version}', ) - path = utils.ensure() + path = utils.ensure(BINARY_PATHS.query_engine) assert path == fake_engine @@ -129,7 +144,7 @@ def test_ensure_env_override( ) with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': str(fake_engine)}): - path = utils.ensure() + path = utils.ensure(BINARY_PATHS.query_engine) assert path == fake_engine @@ -138,7 +153,7 @@ def test_ensure_env_override_does_not_exist() -> None: """Query engine path in environment variable not found raises an error""" with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': 'foo'}): with pytest.raises(errors.BinaryNotFoundError) as exc: - utils.ensure() + utils.ensure(BINARY_PATHS.query_engine) assert exc.match( r'PRISMA_QUERY_ENGINE_BINARY was provided, but no query engine was found at foo' diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[client.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[client.py].raw index 2b2caf808..2ca62efd4 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[client.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[client.py].raw @@ -43,13 +43,14 @@ from .types import DatasourceOverride, HttpConfig from ._types import BaseModelT from .engine import AbstractEngine, QueryEngine from .builder import QueryBuilder -from .generator.models import EngineType, OptionalValueFromEnvVar +from .generator.models import EngineType, OptionalValueFromEnvVar, BinaryPaths from ._compat import removeprefix __all__ = ( 'ENGINE_TYPE', 'SCHEMA_PATH', + 'BINARY_PATHS', 'Batch', 'Prisma', 'Client', @@ -60,8 +61,8 @@ __all__ = ( SCHEMA_PATH = Path('') PACKAGED_SCHEMA_PATH = Path(__file__).parent.joinpath('schema.prisma') - ENGINE_TYPE: EngineType = EngineType.binary +BINARY_PATHS = '' RegisteredClient = Union['Prisma', Callable[[], 'Prisma']] _registered_client: Optional[RegisteredClient] = None diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw index 37dbde74b..4c854ad1f 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw @@ -48,6 +48,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine +from .. import config from ..utils import DEBUG from ..binaries import platform from ..utils import time_since, _env_bool @@ -101,6 +102,12 @@ class QueryEngine(HTTPEngine): if self.session and not self.session.closed: await self.session.close() + def _ensure_file(self) -> Path: + # circular import + from ..client import BINARY_PATHS + + return utils.ensure(BINARY_PATHS.query_engine) + async def connect( self, timeout: int = 10, @@ -111,7 +118,7 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + self.file = file = self._ensure_file() try: await self.spawn(file, timeout=timeout, datasources=datasources) diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[client.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[client.py].raw index 039dc39ce..8ebdc77c2 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[client.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[client.py].raw @@ -43,13 +43,14 @@ from .types import DatasourceOverride, HttpConfig from ._types import BaseModelT from .engine import AbstractEngine, QueryEngine from .builder import QueryBuilder -from .generator.models import EngineType, OptionalValueFromEnvVar +from .generator.models import EngineType, OptionalValueFromEnvVar, BinaryPaths from ._compat import removeprefix __all__ = ( 'ENGINE_TYPE', 'SCHEMA_PATH', + 'BINARY_PATHS', 'Batch', 'Prisma', 'Client', @@ -60,8 +61,8 @@ __all__ = ( SCHEMA_PATH = Path('') PACKAGED_SCHEMA_PATH = Path(__file__).parent.joinpath('schema.prisma') - ENGINE_TYPE: EngineType = EngineType.binary +BINARY_PATHS = '' RegisteredClient = Union['Prisma', Callable[[], 'Prisma']] _registered_client: Optional[RegisteredClient] = None diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw index 44d641f4d..c2389f0c4 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw @@ -48,6 +48,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine +from .. import config from ..utils import DEBUG from ..binaries import platform from ..utils import time_since, _env_bool @@ -102,6 +103,12 @@ class QueryEngine(HTTPEngine): if self.session and not self.session.closed: self.session.close() + def _ensure_file(self) -> Path: + # circular import + from ..client import BINARY_PATHS + + return utils.ensure(BINARY_PATHS.query_engine) + def connect( self, timeout: int = 10, @@ -112,7 +119,7 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + self.file = file = self._ensure_file() try: self.spawn(file, timeout=timeout, datasources=datasources) diff --git a/tests/test_generation/exhaustive/test_exhaustive.py b/tests/test_generation/exhaustive/test_exhaustive.py index 79f45cd17..083d9bc8c 100644 --- a/tests/test_generation/exhaustive/test_exhaustive.py +++ b/tests/test_generation/exhaustive/test_exhaustive.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import sys import subprocess from pathlib import Path @@ -79,9 +80,10 @@ def get_files_from_templates(directory: Path) -> List[str]: ASYNC_ROOTDIR = ROOTDIR / '__prisma_async_output__' / 'prisma' FILES = get_files_from_templates(BASE_PACKAGE_DIR / 'generator' / 'templates') THIS_DIR = Path(__file__).parent +BINARY_PATH_RE = re.compile(r'BINARY_PATHS = (.*)') -def schema_path_matcher( +def path_replacer( schema_path: Path, ) -> Callable[[object, object], Optional[object]]: def pathlib_matcher(data: object, path: object) -> Optional[object]: @@ -90,10 +92,14 @@ def pathlib_matcher(data: object, path: object) -> Optional[object]: f'schema_path_matcher expected data to be a `str` but received {type(data)} instead.' ) - return data.replace( + data = data.replace( f"Path('{schema_path.absolute().as_posix()}')", "Path('')", ) + data = BINARY_PATH_RE.sub( + "BINARY_PATHS = ''", data + ) + return data return pathlib_matcher @@ -106,7 +112,7 @@ def pathlib_matcher(data: object, path: object) -> Optional[object]: def test_sync(snapshot: SnapshotAssertion, file: str) -> None: """Ensure synchronous client files match""" assert SYNC_ROOTDIR.joinpath(file).absolute().read_text() == snapshot( - matcher=schema_path_matcher(THIS_DIR / 'sync.schema.prisma') # type: ignore + matcher=path_replacer(THIS_DIR / 'sync.schema.prisma') # type: ignore ) @@ -115,7 +121,7 @@ def test_sync(snapshot: SnapshotAssertion, file: str) -> None: def test_async(snapshot: SnapshotAssertion, file: str) -> None: """Ensure asynchronous client files match""" assert ASYNC_ROOTDIR.joinpath(file).absolute().read_text() == snapshot( - matcher=schema_path_matcher(THIS_DIR / 'async.schema.prisma') # type: ignore + matcher=path_replacer(THIS_DIR / 'async.schema.prisma') # type: ignore ) diff --git a/tests/test_generation/test_validation.py b/tests/test_generation/test_validation.py index 94f54c575..6e29b76c3 100644 --- a/tests/test_generation/test_validation.py +++ b/tests/test_generation/test_validation.py @@ -172,7 +172,7 @@ def test_binary_targets_warning(testdir: Testdir) -> None: assert_no_generator_output(stdout) assert ( 'Warning: The binaryTargets option ' - 'is not currently supported by Prisma Client Python' in stdout + 'is not officially supported by Prisma Client Python' in stdout ) diff --git a/tests/test_node/__init__.py b/tests/test_node/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_node/test.js b/tests/test_node/test.js new file mode 100644 index 000000000..2c3e4a37b --- /dev/null +++ b/tests/test_node/test.js @@ -0,0 +1,2 @@ +// Sample JS file to test node usage +console.log('Hello world!') diff --git a/tests/test_node/test_node.py b/tests/test_node/test_node.py new file mode 100644 index 000000000..f6bf3deea --- /dev/null +++ b/tests/test_node/test_node.py @@ -0,0 +1,259 @@ +import os +import sys +import shutil +import subprocess +from typing import cast +from pathlib import Path + +import pytest +from pytest_subprocess import FakeProcess +from prisma.cli import _node as node +from prisma.cli._node import Target +from prisma._config import Config +from prisma._compat import nodejs + +from ..utils import set_config + + +THIS_DIR = Path(__file__).parent + +parametrize_target = pytest.mark.parametrize('target', ['node', 'npm']) + + +def _assert_can_run_js(strategy: node.Node) -> None: + proc = strategy.run( + str(THIS_DIR.joinpath('test.js')), + stdout=subprocess.PIPE, + ) + output = proc.stdout.decode('utf-8') + assert output == 'Hello world!\n' + + +def _assert_can_run_npm(strategy: node.Node) -> None: + assert strategy.target == 'npm' + + proc = strategy.run('help', stdout=subprocess.PIPE) + output = proc.stdout.decode('utf-8') + + assert 'npm' in output + + +def assert_strategy(strategy: node.Node) -> None: + if strategy.target == 'node': + _assert_can_run_js(strategy) + elif strategy.target == 'npm': + _assert_can_run_npm(strategy) + else: # pragma: no cover + raise ValueError( + f'No tests implemented for strategy target: {strategy.target}' + ) + + +def test_resolve_bad_target() -> None: + """resolve() raises a helpful error message when given an unknown target""" + with pytest.raises( + node.UnknownTargetError, + match='Unknown target: foo; Valid choices are: node, npm', + ): + node.resolve(cast(node.Target, 'foo')) + + +@parametrize_target +@pytest.mark.skipif(nodejs is None, reason='nodejs-bin is not installed') +def test_nodejs_bin(target: Target) -> None: + """When `nodejs-bin` is installed, it is resolved to and can be successfully used""" + with set_config( + Config.parse( + use_nodejs_bin=True, + use_global_node=False, + ) + ): + strategy = node.resolve(target) + assert strategy.resolver == 'nodejs-bin' + assert_strategy(strategy) + + +@parametrize_target +@pytest.mark.skipif( + shutil.which('node') is None, + reason='Node is not installed globally', +) +def test_resolves_binary_node(target: Target) -> None: + """When `node` is installed globally, it is resolved to and can be successfully used""" + with set_config( + Config.parse( + use_nodejs_bin=False, + use_global_node=True, + ) + ): + strategy = node.resolve(target) + assert strategy.resolver == 'global' + assert_strategy(strategy) + + with set_config( + Config.parse( + use_nodejs_bin=False, + use_global_node=False, + ) + ): + strategy = node.resolve(target) + assert strategy.resolver == 'nodeenv' + assert_strategy(strategy) + + +@parametrize_target +def test_nodeenv(target: Target) -> None: + """When `nodejs-bin` and global `node` is not installed / configured to use, `nodeenv` is resolved to and can be successfully used""" + with set_config( + Config.parse( + use_nodejs_bin=False, + use_global_node=False, + ) + ): + strategy = node.resolve(target) + assert strategy.resolver == 'nodeenv' + assert_strategy(strategy) + + +@parametrize_target +def test_nodeenv_extra_args( + target: Target, + tmp_path: Path, + fake_process: FakeProcess, +) -> None: + """The config option `nodeenv_extra_args` is respected""" + cache_dir = tmp_path / 'nodeenv' + + fake_process.register_subprocess( + [sys.executable, '-m', 'nodeenv', str(cache_dir), '--my-extra-flag'], + returncode=403, + ) + + with set_config( + Config.parse( + use_nodejs_bin=False, + use_global_node=False, + nodeenv_extra_args=['--my-extra-flag'], + nodeenv_cache_dir=cache_dir, + ) + ): + with pytest.raises(subprocess.CalledProcessError) as exc: + node.resolve(target) + + assert exc.value.returncode == 403 + + +def test_update_path_env() -> None: + """The _update_path_env() function correctly appends the target binary path to the PATH environment variable""" + target = THIS_DIR / 'bin' + if not target.exists(): # pragma: no branch + target.mkdir() + + sep = os.pathsep + + # known PATH separators - please update if need be + assert sep in {':', ';'} + + # no env + env = node._update_path_env(env=None, target_bin=target) + assert env['PATH'].startswith(f'{target.absolute()}{sep}') + + # env without PATH + env = node._update_path_env( + env={'FOO': 'bar'}, + target_bin=target, + ) + assert env['PATH'].startswith(f'{target.absolute()}{sep}') + + # env with empty PATH + env = node._update_path_env( + env={'PATH': ''}, + target_bin=target, + ) + assert env['PATH'].startswith(f'{target.absolute()}{sep}') + + # env with set PATH without the separator postfix + env = node._update_path_env( + env={'PATH': '/foo'}, + target_bin=target, + ) + assert env['PATH'] == f'{target.absolute()}{sep}/foo' + + # env with set PATH with the separator as a prefix + env = node._update_path_env( + env={'PATH': f'{sep}/foo'}, + target_bin=target, + ) + assert env['PATH'] == f'{target.absolute()}{sep}/foo' + + # returned env included non PATH environment variables + env = node._update_path_env( + env={'PATH': '/foo', 'FOO': 'bar'}, + target_bin=target, + ) + assert env['FOO'] == 'bar' + assert env['PATH'] == f'{target.absolute()}{sep}/foo' + + # accepts a custom path separator + env = node._update_path_env( + env={'PATH': '/foo'}, + target_bin=target, + sep='---', + ) + assert env['PATH'] == f'{target.absolute()}---/foo' + + +@parametrize_target +@pytest.mark.skipif( + shutil.which('node') is None, + reason='Node is not installed globally', +) +def test_node_version(target: Target, fake_process: FakeProcess) -> None: + """The node version can be detected properly and correctly constrained to our minimum required version""" + + def _register_process(stdout: str) -> None: + # register the same process twice as we will call it twice + # in our assertions and fake_process consumes called processes + for _ in range(2): + fake_process.register_subprocess( + [str(path), '--version'], stdout=stdout + ) + + which = shutil.which(target) + assert which is not None + + path = Path(which) + + _register_process('v1.3.4') + version = node._get_binary_version(target, path) + assert version == (1, 3) + assert node._should_use_binary(target, path) is False + + _register_process('v1.32433') + version = node._get_binary_version(target, path) + assert version == (1, 32433) + assert node._should_use_binary(target, path) is False + + _register_process('v16.15.a4') + version = node._get_binary_version(target, path) + assert version == (16, 15) + assert node._should_use_binary(target, path) is True + + _register_process('v14.17.1') + version = node._get_binary_version(target, path) + assert version == (14, 17) + assert node._should_use_binary(target, path) is True + + _register_process('14.17.1') + version = node._get_binary_version(target, path) + assert version == (14, 17) + assert node._should_use_binary(target, path) is True + + +def test_should_use_binary_unknown_target() -> None: + """The UnknownTargetError() is raised by the _should_use_binary() function given an invalid target""" + with pytest.raises(node.UnknownTargetError): + node._should_use_binary( + target='foo', # type: ignore + path=Path.cwd(), + ) diff --git a/tests/utils.py b/tests/utils.py index 4a6c94634..af96b29cc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,7 +25,9 @@ import pytest from click.testing import CliRunner, Result +from prisma import _config from prisma.cli import main +from prisma._proxy import LazyProxy from prisma._types import FuncType from prisma.binaries import platform from prisma.generator.utils import copy_tree @@ -284,6 +286,18 @@ def get_source_from_function(function: FuncType, **env: Any) -> str: return IMPORT_RELOADER + '\n'.join(lines) +@contextlib.contextmanager +def set_config(config: _config.Config) -> Iterator[_config.Config]: + proxy = cast(LazyProxy[_config.Config], _config.config) + old = proxy.__get_proxied__() + + try: + proxy.__set_proxied__(config) + yield config + finally: + proxy.__set_proxied__(old) + + def patch_method( patcher: 'MonkeyPatch', obj: object, diff --git a/tests/windows.Dockerfile b/tests/windows.Dockerfile index f3508cd72..b90de6df3 100644 --- a/tests/windows.Dockerfile +++ b/tests/windows.Dockerfile @@ -21,3 +21,13 @@ RUN prisma -v # Very light-weight test that CLI generation works RUN prisma generate --schema ./tests/data/schema.prisma + +# Ensure all combinations of non-global Node resolvers work +RUN nox -s test -p 3.10 -- tests/test_node + +# Ensure database access using SQLite works +ENV SQLITE_URL="file:sqlite.db" + +# We don't run linters here because pyright-python is currently broken in certain docker images +# TODO: lint when fixed +RUN nox -s databases -- test --databases=sqlite --no-lint