refactor(cli): remove pkg in favour of automatically downloading Node (…

…#454)

## Change Summary

This PR completely refactors how the Prisma CLI is downloaded /
installed / called. We now download Node itself at runtime and use that
to install the Prisma CLI and then run it directly with Node as well.

This has some significant advantages:

- We are now no longer tied to releases of the packaged CLI
- These were being released by Luca, who created the Go Client and is no
longer at Prisma, on his own free time.
- Only major versions were released which means the CLI couldn't be
easily tested with the latest changes on the
https://github.com/prisma/prisma repository
- Prisma Studio can now be launched from the CLI
- The TypeScript Client can now be generated from our CLI wrapper
- We now longer have to manually download the engine binaries ourselves,
that's handled transparently for us
- A packaged version of Node no longer has to be installed for each new
Prisma version
- We now have support for ARM

However, this does not come without some concerns:

- Size increase
- We use https://github.com/ekalinin/nodeenv to download Node at runtime
if it isn't already installed. This downloads and creates extra files
that are not strictly necessary for our use case. This results in an
increase from ~140MB -> ~300MB. -
- However this size increase can be reduced by installing
[nodejs-bin](https://pypi.org/project/nodejs-bin/) which you can do by
providing the `node` extra, e.g. `pip install prisma[node]`. This brings
the total download size to be very similar to the packaged CLI.
- This concern also doesn't apply in cases where Node is already
present. It actually will greatly improve the experience in this case.


## How does it work?

We now resolve a Node binary using this flow:

- Check if [nodejs-bin](https://pypi.org/project/nodejs-bin/) is
installed
- Check if Node is installed globally
- Downloads Node using https://github.com/ekalinin/nodeenv to a
configurable location which defaults to `~/.cache/prisma-nodeenv`

The first two steps in this flow can be skipped if you so desire through
your `pyproject.toml` file or using environment variables. For example:

```toml
[tool.prisma]
# skip global node check
use_global_node = false

# skip nodejs-bin check
use_nodejs_bin = false

# change nodeenv installation directory
nodeenv_cache_dir = '~/.foo/nodeenv'
```

Or using the environment variables, `PRISMA_USE_GLOBAL_NODE`,
`PRISMA_USE_NODEJS_BIN` and `PRISMA_NODEENV_CACHE_DIR` respectively.

The Prisma CLI is then installed directly from
[npm](https://www.npmjs.com/package/prisma) and ran directly using the
resolved Node binary.

.dockerignore

@@ -4,4 +4,9 @@
 !src/
 !tests
 !requirements/
+!databases/
+!lib/
+!noxfile.py
 !pipelines/
+!pytest.ini
+!MANIFEST.in

.github/workflows/test.yml

@@ -21,6 +21,7 @@ jobs:
     name: test
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: [3.7, 3.8, 3.9, "3.10", "3.11.0-rc.1"]
@@ -303,21 +304,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        docker-platform: [linux/amd64]
-        # TODO: Uncomment this to add testing support for arm64 and delete
-        # the above
-        # docker-platform: ["linux/amd64", "linux/arm64"]
-        # TODO: Uncomment this later, Go-based CLI does not run on Alpine
-        # https://github.com/prisma/prisma-client-go/issues/357
-        # python-os-distro: [slim-bullseye, alpine]
-        python-os-distro: [slim-bullseye]
+        docker-platform: [linux/amd64, linux/arm64]
+        python-os-distro: [slim-bullseye, alpine]
+        exclude:
+          # Dockerfile is currently broken for alpine / arm64
+          # https://github.com/RobertCraigie/prisma-client-py/issues/581
+          - docker-platform: linux/arm64
+            python-os-distro: alpine
     steps:
       - uses: actions/checkout@v3
+
       # https://github.com/docker/build-push-action/
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+
       - name: Docker Build
         uses: docker/build-push-action@v3
         # https://github.com/docker/build-push-action/#inputs

databases/main.py

@@ -21,7 +21,11 @@
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
 
 from lib.utils import flatten, escape_path
-from pipelines.utils import setup_coverage, get_pkg_location
+from pipelines.utils import (
+    setup_coverage,
+    get_pkg_location,
+    maybe_install_nodejs_bin,
+)
 from prisma._compat import cached_property
 
 from .utils import DatabaseConfig
@@ -68,6 +72,8 @@ def test(
     with session.chdir(DATABASES_DIR):
         # setup env
         session.install('-r', 'requirements.txt')
+        maybe_install_nodejs_bin(session)
+
         if inplace:  # pragma: no cover
             # useful for updating the generated code so that Pylance picks it up
             session.install('-U', '-e', '..')

databases/requirements.txt

@@ -3,6 +3,7 @@ click==8.1.3
 coverage==6.5.0
 syrupy==3.0.5
 dirty-equals==0.5.0
+distro
 
 -r ../pipelines/requirements/deps/pyright.txt
 -r ../pipelines/requirements/deps/pytest.txt

docs/reference/config.md

@@ -199,54 +199,87 @@ Or through environment variables, e.g. `PRISMA_BINARY_CACHE_DIR`. In the case th
 
 ### Binary Cache Directory
 
-This option controls where the Prisma Engine and Prisma CLI binaries should be downloaded to. This defaults to a temporary directory that includes the current Prisma Engine version.
+This option controls where the Prisma Engine and Prisma CLI binaries should be downloaded to. This defaults to a cache directory that includes the current Prisma Engine version.
+
+| Option             | Environment Variable       | Default                                                                   |
+| ------------------ | -------------------------- | ------------------------------------------------------------------------- |
+| `binary_cache_dir` | `PRISMA_BINARY_CACHE_DIR`  | `/{home}/.cache/prisma-python/binaries/{prisma_version}/{engine_version}` |
+
+### Home Directory
+
+This option can be used to change the base directory of the `binary_cache_dir` option without having to worry about versioning the Prisma binaries. This is useful if you need to download the binaries to a local directory.
+
+| Option     | Environment Variable  | Default |
+| --------   | --------------------- | ------- |
+| `home_dir` | `PRISMA_HOME_DIR`     | `~`     |
 
-| Option             | Environment Variable       | Default                                               |
-| ------------------ | -------------------------- | ----------------------------------------------------- |
-| `binary_cache_dir` | `PRISMA_BINARY_CACHE_DIR`  | `/{tmp}/prisma/binaries/engines/{engine_version}` |
 
 ### Prisma Version
 
-This option controls the version of the Prisma CLI to use. It should be noted that this is intended to be internal and only the pinned prisma version is guaranteed to be supported.
+This option controls the version of Prisma to use. It should be noted that this is intended to be internal and only the pinned Prisma version is guaranteed to be supported.
 
 | Option           | Environment Variable  | Default  |
 | ---------------- | --------------------- | -------- |
 | `prisma_version` | `PRISMA_VERSION`      | `3.13.0` |
 
-### Engine Version
+### Expected Engine Version
 
-This option controls the version of the [Prisma Engines](https://github.com/prisma/prisma-engines) to use, like `prisma_version` this is intended to be internal and only the pinned engine version is guaranteed to be supported.
+This is an internal option that is here as a safeguard for the `prisma_version` option. If you modify the `prisma_version` option then you must also update this option to use the corresponding engine version. You can find a list of engine versions [here](https://github.com/prisma/prisma-engines).
 
-| Option           | Environment Variable    | Default                                    |
-| ---------------- | ----------------------- | ------------------------------------------ |
-| `engine_version` | `PRISMA_ENGINE_VERSION` | `efdf9b1183dddfd4258cd181a72125755215ab7b` |
+| Option                    | Environment Variable             | Default                                    |
+| ------------------------- | -------------------------------- | ------------------------------------------ |
+| `expected_engine_version` | `PRISMA_EXPECTED_ENGINE_VERSION` | `efdf9b1183dddfd4258cd181a72125755215ab7b` |
 
-### Prisma URL
 
-This option controls where the Prisma CLI binaries should be downloaded from. If set, this must be a string that takes two format arguments, `version` and `platform`, for example:
+### Binary Platform
+
+This option is useful if you need to make use of the [binaryTargets](https://www.prisma.io/docs/concepts/components/prisma-schema/generators#binary-targets) schema option to build your application on one platform and deploy it on another.
+
+This allows you to set the current platform dynamically as Prisma Client Python does not have official support for `binaryTargets` and although we do have some safe guards in place to attempt to use the correct binary, it has not been thoroughly tested.
+
+A list of valid options can be found [here](https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference#binarytargets-options).
 
-```
-https://example.com/prisma-cli-{version}-{platform}.gz
-```
 
-| Option       | Environment Variable | Default                                                                                 |
-| -------------| -------------------- | --------------------------------------------------------------------------------------- |
-| `prisma_url` | `PRISMA_CLI_URL`     | `https://prisma-photongo.s3-eu-west-1.amazonaws.com/prisma-cli-{version}-{platform}.gz` |
+| Option            | Environment Variable     |
+| ----------------- | ------------------------ |
+| `binary_platform` | `PRISMA_BINARY_PLATFORM` |
 
-### Engine URL
+### Use Global Node
 
-This option controls where the [Prisma Engine](https://github.com/prisma/prisma-engines) binaries should be downloaded from. If set, this must be a string that takes three positional format arguments, for example:
+This option configures whether or not Prisma Client Python will attempt to use the globally installed version of [Node](https://nodejs.org/en/), if it is available, to run the Prisma CLI.
+
+| Option            | Environment Variable     | Default |
+| ----------------- | ------------------------ | ------- |
+| `use_global_node` | `PRISMA_USE_GLOBAL_NODE` | `True`  |
+
+### Use nodejs-bin
+
+This option configures whether or not Prisma Client Python will attempt to use the installed version of the [nodejs-bin](https://pypi.org/project/nodejs-bin/) package, if it is available, to run the Prisma CLI.
+
+| Option           | Environment Variable    | Default |
+| ---------------- | ----------------------- | ------- |
+| `use_nodejs_bin` | `PRISMA_USE_NODEJS_BIN` | `True`  |
+
+### Extra Nodeenv Arguments
+
+This option allows you to pass additional arguments to [nodeenv](https://github.com/ekalinin/nodeenv) which is the package we use to automatically download a Node binary to run the CLI with.
+
+Arguments are passed after the path, for example:
 
 ```
-https://example.com/prisma-binaries-mirror/{0}/{1}/{2}.gz
+python -m nodeenv <path> <extra args>
 ```
 
-Where:
+| Option               | Environment Variable        |
+| -------------------- | --------------------------- |
+| `nodeenv_extra_args` | `PRISMA_NODEENV_EXTRA_ARGS` |
+
+### Nodeenv Cache Directory
+
+This option configures where Prisma Client Python will store the Node binary that is installed by [nodeenv](https://github.com/ekalinin/nodeenv).
 
-- `0` corresponds to the [engine version](#engine-version)
-- `1` corresponds to the current binary platform
-- `2` corresponds to the name of the engine being downloaded.
+Note that this does not make use of the [Home Directory](#home-directory) option and instead uses the actual user home directory.
 
-| Option       | Environment Variable | Default                                                 |
-| -------------| -------------------- | ------------------------------------------------------- |
-| `engine_url` | `PRISMA_ENGINE_URL`  | `https://binaries.prisma.sh/all_commits/{0}/{1}/{2}.gz` |
+| Option              | Environment Variable       | Default                           |
+| ------------------- | -------------------------- | --------------------------------- |
+| `nodeenv_cache_dir` | `PRISMA_NODEENV_CACHE_DIR` | `~/.cache/prisma-python/nodeenv/` |

lib/testing/shared_conftest/__init__.py

@@ -1,6 +1,7 @@
 # NOTE: a lot of these are not intended to be imported and referenced directly
 # and are instead intended to be `*` imported in a `conftest.py` file
 from ._shared_conftest import (
+    setup_env as setup_env,
     event_loop as event_loop,
     client_fixture as client_fixture,
     patch_prisma_fixture as patch_prisma_fixture,

lib/testing/shared_conftest/_shared_conftest.py

@@ -1,6 +1,7 @@
 import asyncio
 import inspect
 from typing import TYPE_CHECKING, Iterator
+from pathlib import Path
 
 import pytest
 
@@ -14,6 +15,10 @@
 
 if TYPE_CHECKING:
     from _pytest.fixtures import FixtureRequest
+    from _pytest.monkeypatch import MonkeyPatch
+
+
+HOME_DIR = Path.home()
 
 
 @async_fixture(name='client', scope='session')
@@ -31,6 +36,14 @@ def event_loop() -> asyncio.AbstractEventLoop:
     return get_or_create_event_loop()
 
 
+@pytest.fixture(autouse=True)
+def setup_env(monkeypatch: 'MonkeyPatch') -> None:
+    # Set a custom home directory to use for caching binaries so that
+    # when we make use of pytest's temporary directory functionality the binaries
+    # don't have to be downloaded again.
+    monkeypatch.setenv('PRISMA_HOME_DIR', str(HOME_DIR))
+
+
 @pytest.fixture(name='patch_prisma', autouse=True)
 def patch_prisma_fixture(request: 'FixtureRequest') -> Iterator[None]:
     if request_has_client(request):

pipelines/coverage.nox.py

@@ -4,7 +4,6 @@
 from pathlib import Path
 
 import nox
-from git.repo import Repo
 
 from lib.utils import maybe_decode
 from pipelines.utils import setup_env, CACHE_DIR, TMP_DIR
@@ -18,6 +17,10 @@
 
 @nox.session(name='push-coverage')
 def push_coverage(session: nox.Session) -> None:
+    # We have to import `git` here as it will cause an error on machines that don't have
+    # git installed. This happens in our docker tests.
+    from git.repo import Repo
+
     session.env['COVERAGE_FILE'] = str(CACHE_DIR / '.coverage')
     session.install(
         '-r',

pipelines/requirements/dev.txt

@@ -6,3 +6,4 @@ twine==4.0.1
 typer==0.7.0
 rtoml==0.9.0
 GitPython
+distro

pipelines/requirements/lint.txt

@@ -1,4 +1,5 @@
 -r test.txt
+-r node.txt
 -r deps/pyright.txt
 interrogate==1.5.0
 blue==0.9.1

pipelines/requirements/mypy.txt

@@ -1,3 +1,4 @@
 -r test.txt
+-r node.txt
 mypy==0.950
 types-mock

pipelines/requirements/node.txt

@@ -0,0 +1 @@
+nodejs-bin==16.15.1a4

pipelines/test.nox.py

@@ -1,6 +1,6 @@
 import nox
 
-from pipelines.utils import setup_env
+from pipelines.utils import setup_env, maybe_install_nodejs_bin
 from pipelines.utils.prisma import generate
 
 
@@ -9,6 +9,7 @@ def test(session: nox.Session) -> None:
     setup_env(session)
     session.install('-r', 'pipelines/requirements/test.txt')
     session.install('.')
+    maybe_install_nodejs_bin(session)
 
     generate(session)
 

pipelines/utils/utils.py

@@ -2,10 +2,12 @@
 from pathlib import Path
 
 import nox
+import distro
 
 
 CACHE_DIR = Path.cwd() / '.cache'
 TMP_DIR = Path(tempfile.gettempdir())
+PIPELINES_DIR = Path(__file__).parent.parent
 
 
 def get_pkg_location(session: nox.Session, pkg: str) -> str:
@@ -44,3 +46,13 @@ def setup_env(session: nox.Session) -> None:
             ]
         ]
     )
+
+
+def maybe_install_nodejs_bin(session: nox.Session) -> bool:
+    # nodejs-bin is not available on alpine yet, we need to wait until this fix is released:
+    # https://github.com/samwillis/nodejs-pypi/issues/11
+    if distro.id() == 'alpine':
+        return False
+
+    session.install('-r', str(PIPELINES_DIR / 'requirements/node.txt'))
+    return True

requirements/base.txt

@@ -5,4 +5,5 @@ click>=7.1.2
 python-dotenv>=0.12.0
 typing-extensions>=3.7
 tomlkit
+nodeenv
 cached-property; python_version < '3.8'

requirements/node.txt

@@ -0,0 +1 @@
+nodejs-bin

scripts/docs.py

@@ -46,21 +46,13 @@ def main() -> None:
         r'\1' + '`' + config.prisma_version + '`',
         content,
     )
+
     content = re.sub(
-        r'(`PRISMA_ENGINE_VERSION` \| )`(.*)`',
-        r'\1' + '`' + config.engine_version + '`',
-        content,
-    )
-    content = re.sub(
-        r'(`PRISMA_CLI_URL`     \| )`(.*)`',
-        r'\1' + '`' + config.prisma_url + '`',
-        content,
-    )
-    content = re.sub(
-        r'(`PRISMA_ENGINE_URL`  \| )`(.*)`',
-        r'\1' + '`' + config.engine_url + '`',
+        r'(`PRISMA_EXPECTED_ENGINE_VERSION` \| )`(.*)`',
+        r'\1' + '`' + config.expected_engine_version + '`',
         content,
     )
+
     config_doc.write_text(content)
 
 

setup.py

@@ -28,6 +28,10 @@ def requirements(name: str) -> List[str]:
     raise RuntimeError('version is not set')
 
 
+extras = {
+    'node': requirements('node.txt'),
+}
+
 setup(
     name='prisma',
     version=version,
@@ -50,8 +54,10 @@ def requirements(name: str) -> List[str]:
     include_package_data=True,
     zip_safe=False,
     extras_require={
-        # we define `all` even though it's empty so that we can add to it in the future
-        'all': [],
+        **extras,
+        'all': [
+            req for requirements in extras.values() for req in requirements
+        ],
     },
     entry_points={
         'console_scripts': [