From 325091c02f928f12b302bdeb05a8edcccc0c12db Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:06:10 +0100 Subject: [PATCH 01/55] feat(fetch_binaries): add `distro` requirement --- requirements/base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/base.txt b/requirements/base.txt index d165e032e..0820c8e7b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,5 +4,6 @@ pydantic>=1.8.0 click>=7.1.2 python-dotenv>=0.12.0 typing-extensions>=3.7 +distro>=1.6 contextvars; python_version < '3.7' cached-property; python_version < '3.8' From 6d5f61e14704982ad3f2f91f813e6322945a2274 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:43:21 +0100 Subject: [PATCH 02/55] feat(fetch_binaris): update `platform` to comply with original impl --- src/prisma/binaries/platform.py | 137 +++++++++++++++++++------------- 1 file changed, 83 insertions(+), 54 deletions(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index e6b041019..fbae09f36 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -1,64 +1,19 @@ import re -import sys import subprocess -import platform as _platform +import platform +import distro from functools import lru_cache -from typing import Tuple +from typing import Optional, TypedDict -def name() -> str: - return _platform.system().lower() - - -def check_for_extension(file: str) -> str: - if name() == 'windows' and '.exe' not in file: - if '.gz' in file: - return file.replace('.gz', '.exe.gz') - return file + '.exe' - return file - - -def linux_distro() -> str: - # NOTE: this has only been tested on ubuntu - distro_id, distro_id_like = _get_linux_distro_details() - if distro_id == 'alpine': - return 'alpine' - - if any(distro in distro_id_like for distro in ['centos', 'fedora', 'rhel']): - return 'rhel' - - # default to debian - return 'debian' - - -def _get_linux_distro_details() -> Tuple[str, str]: - process = subprocess.run( - ['cat', '/etc/os-release'], stdout=subprocess.PIPE, check=True - ) - output = str(process.stdout, sys.getdefaultencoding()) - - match = re.search(r'ID="?([^"\n]*)"?', output) - distro_id = match.group(1) if match else '' # type: str - - match = re.search(r'ID_LIKE="?([^"\n]*)"?', output) - distro_id_like = match.group(1) if match else '' # type: str - return distro_id, distro_id_like - - -@lru_cache(maxsize=None) -def binary_platform() -> str: - platform = name() - if platform != 'linux': - return platform - - distro = linux_distro() - if distro == 'alpine': - return 'linux-musl' - - ssl = get_openssl() - return f'{distro}-openssl-{ssl}' +class OsSettings(TypedDict): + system: str + machine: str + libssl: str + distro: Optional[str] +@lru_cache() def get_openssl() -> str: process = subprocess.run( ['openssl', 'version', '-v'], stdout=subprocess.PIPE, check=True @@ -66,6 +21,7 @@ def get_openssl() -> str: return parse_openssl_version(str(process.stdout, sys.getdefaultencoding())) +@lru_cache() def parse_openssl_version(string: str) -> str: match = re.match(r'^OpenSSL\s(\d+\.\d+)\.\d+', string) if match is None: @@ -73,3 +29,76 @@ def parse_openssl_version(string: str) -> str: return '1.1.x' return match.group(1) + '.x' + + +def resolve_known_distro(distro_id: str, distro_like: str) -> Optional[str]: + if distro_id == "alpine": + return "musl" + elif distro_id == "raspbian": + return "arm" + elif distro_id == "nixos": + return "nixos" + elif ( + distro_id == "fedora" + or "fedora" in distro_like + or "rhel" in distro_like + or "centros" in distro_like + ): + return "rhel" + elif ( + distro_id == "ubuntu" + or distro_id == "debian" + or "ubuntu" in distro_like + or "debian" in distro_like + ): + return "debian" + return None + + +def get_os_settings() -> OsSettings: + system = platform.system().lower() + machine = platform.machine().lower() + openssl_version = get_openssl() + distro_id = distro.id() + distro_like = distro.like() + distr = resolve_known_distro(distro_id, distro_like) + return OsSettings( + system=system, machine=machine, libssl=openssl_version, distro=distr + ) + + +def resolve_platform(os: OsSettings) -> str: + system, machine, libssl, distro = ( + os['system'], + os['machine'], + os['libssl'], + os['distro'], + ) + + if system == "darwin" and machine == "aarch64": + return "darwin-arm64" + elif system == "darwin": + return "darwin" + elif system == "windows": + return "windows" + elif system == "freebsd": + return "freebsd" + elif system == "openbsd": + return "openbsd" + elif system == "netbsd": + return "netbsd" + elif system == "linux" and machine == "aarch64" + return f"linux-arm64-openssl-{libssl}" + elif system == "linux" and machine == "arm": + return f"linux-arm-openssl-{libssl}" + elif system == "linux" and distro == "musl": + return "linux-musl" + elif system == "linux" and distro == "nixos": + return "linux-nixos" + elif distro: + return f"{distro}-openssl-{libssl}" + return "debian-openssl-1.1.x" # default fallback + + +def get_platform() -> str: + return resolve_platform(get_os_settings()) From 44534d9c88970a56597e75655d9835cc88cee539 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:46:02 +0100 Subject: [PATCH 03/55] feat(fetch_binaries): forgot to import sys in platform.py --- src/prisma/binaries/platform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index fbae09f36..c92ef3ee2 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -1,6 +1,7 @@ import re import subprocess import platform +import sys import distro from functools import lru_cache from typing import Optional, TypedDict From 95bcd6745b5de8efdcf124b9870df4067196e3f7 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Wed, 19 Jan 2022 23:46:22 +0100 Subject: [PATCH 04/55] feat(fetch_binaries): missing `:` --- src/prisma/binaries/platform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index c92ef3ee2..550221f2e 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -88,7 +88,7 @@ def resolve_platform(os: OsSettings) -> str: return "openbsd" elif system == "netbsd": return "netbsd" - elif system == "linux" and machine == "aarch64" + elif system == "linux" and machine == "aarch64": return f"linux-arm64-openssl-{libssl}" elif system == "linux" and machine == "arm": return f"linux-arm-openssl-{libssl}" @@ -98,7 +98,7 @@ def resolve_platform(os: OsSettings) -> str: return "linux-nixos" elif distro: return f"{distro}-openssl-{libssl}" - return "debian-openssl-1.1.x" # default fallback + return "debian-openssl-1.1.x" # default fallback def get_platform() -> str: From d24d83b09982df258abca25575603d5cd956e9c8 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 01:48:16 +0100 Subject: [PATCH 05/55] feat(fetch_binaries): working prototype needs cleaning up and tests --- src/prisma/binaries/__init__.py | 3 +- src/prisma/binaries/binaries.py | 144 ++++++++++++++---- src/prisma/binaries/binary.py | 46 ------ src/prisma/binaries/constants.py | 48 ------ src/prisma/binaries/{utils.py => download.py} | 6 +- src/prisma/binaries/engine.py | 37 ----- src/prisma/cli/prisma.py | 6 +- src/prisma/generator/models.py | 2 +- 8 files changed, 124 insertions(+), 168 deletions(-) delete mode 100644 src/prisma/binaries/binary.py delete mode 100644 src/prisma/binaries/constants.py rename src/prisma/binaries/{utils.py => download.py} (84%) delete mode 100644 src/prisma/binaries/engine.py diff --git a/src/prisma/binaries/__init__.py b/src/prisma/binaries/__init__.py index af30b1f84..cb0849161 100644 --- a/src/prisma/binaries/__init__.py +++ b/src/prisma/binaries/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from .engine import * + from .binaries import * -from .constants import * diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index d677a85d2..2971e7f5b 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -6,60 +6,144 @@ import click -from .binary import Binary -from .engine import Engine -from .constants import GLOBAL_TEMP_DIR, PRISMA_CLI_NAME - -__all__ = ( - 'ENGINES', - 'BINARIES', - 'ensure_cached', - 'remove_all', +import os +import tempfile +from pathlib import Path +from typing import List + +from prisma.binaries.download import download + +from . import platform + + +# PLATFORMS: List[str] = [ +# 'darwin', +# 'darwin-arm64', +# 'debian-openssl-1.0.x', +# 'debian-openssl-1.1.x', +# 'rhel-openssl-1.0.x', +# 'rhel-openssl-1.1.x', +# 'linux-arm64-openssl-1.1.x', +# 'linux-arm64-openssl-1.0.x', +# 'linux-arm-openssl-1.1.x', +# 'linux-arm-openssl-1.0.x', +# 'linux-musl', +# 'linux-nixos', +# 'windows', +# 'freebsd11', +# 'freebsd12', +# 'openbsd', +# 'netbsd', +# 'arm', +# ] + +PLATFORM = platform.get_platform() + +from pydantic import BaseSettings, Field + + +# TODO: if this version changes but the engine version +# doesn't change then the CLI is incorrectly cached +# hardcoded CLI version version +PRISMA_VERSION = '3.7.0' + +# versions can be found under https://github.com/prisma/prisma-engine/commits/main +ENGINE_VERSION = os.environ.get( + 'PRISMA_ENGINE_VERSION', '8746e055198f517658c08a0c426c7eec87f5a85f' +) +GLOBAL_TEMP_DIR = ( + Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) +PLATFORM_EXE_EXTENSION = ".exe" if PLATFORM == "windows" else "" + + +def default_in_temp(name: str): + return lambda: (GLOBAL_TEMP_DIR / name).with_suffix(PLATFORM_EXE_EXTENSION) + + +class PrismaSettings(BaseSettings): + PRISMA_CLI_MIRROR: str = "https://prisma-photongo.s3-eu-west-1.amazonaws.com" + PRISMA_ENGINES_MIRROR: str = "https://binaries.prisma.sh" + PRISMA_QUERY_ENGINE_BINARY: Path = Field( + default_factory=default_in_temp("query-engine") + ) + PRISMA_MIGRATION_ENGINE_BINARY: Path = Field( + default_factory=default_in_temp("migration-engine") + ) + PRISMA_INTROSPECTION_ENGINE_BINARY: Path = Field( + default_factory=default_in_temp("introspection-engine") + ) + PRISMA_FMT_BINARY: Path = Field(default_factory=default_in_temp("prisma-fmt")) + PRISMA_CLI_BINARY_TARGETS: List[str] = Field(default_factory=list) + + +settings = PrismaSettings() + +# CLI binaries are stored here +PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{PLATFORM}{PLATFORM_EXE_EXTENSION}" +PRISMA_CLI_PATH = GLOBAL_TEMP_DIR / PRISMA_CLI_NAME +PRISMA_CLI_URL = f"{settings.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" + + +def engine_url_for(name: str) -> str: + return f"{settings.PRISMA_ENGINES_MIRROR}/all_commits/{ENGINE_VERSION}/{PLATFORM}/{name}{PLATFORM_EXE_EXTENSION}.gz" + log: logging.Logger = logging.getLogger(__name__) -ENGINES = [ - Engine(name='query-engine', env='PRISMA_QUERY_ENGINE_BINARY'), - Engine(name='migration-engine', env='PRISMA_MIGRATION_ENGINE_BINARY'), - Engine(name='introspection-engine', env='PRISMA_INTROSPECTION_ENGINE_BINARY'), - Engine(name='prisma-fmt', env='PRISMA_FMT_BINARY'), + +class Binary: + name: str + url: str + path: Path + + def __init__(self, name: str, path: Path, *, url: Optional[str] = None): + self.name = name + self.path = path + self.url = engine_url_for(name) if url is None else url + + +ENGINES: List[Binary] = [ + Binary(name='query-engine', path=settings.PRISMA_QUERY_ENGINE_BINARY), + Binary(name='migration-engine', path=settings.PRISMA_MIGRATION_ENGINE_BINARY), + Binary( + name='introspection-engine', path=settings.PRISMA_INTROSPECTION_ENGINE_BINARY + ), + Binary(name='prisma-fmt', path=settings.PRISMA_FMT_BINARY), ] BINARIES: List[Binary] = [ *ENGINES, - Binary(name=PRISMA_CLI_NAME, env='PRISMA_CLI_BINARY'), + Binary(name="prisma-cli", path=PRISMA_CLI_PATH, url=PRISMA_CLI_URL), ] def ensure_cached() -> Path: - binaries: List[Binary] = [] + to_download: List[Binary] = [] for binary in BINARIES: - path = binary.path - if path.exists(): - log.debug('%s cached at %s', binary.name, path) - else: - log.debug('%s not cached at %s', binary.name, path) - binaries.append(binary) - - if not binaries: - log.debug('All binaries are cached') + if binary.path.exists(): + log.debug(f"{binary.name} is cached, skipping download") + continue + log.debug(f"{binary.name} is not cached, will download") + to_download.append(binary) + + if len(to_download) == 0: + log.debug("All binaries are cached, skipping download") return GLOBAL_TEMP_DIR def show_item(item: Optional[Binary]) -> str: - if item is not None: - return binary.name - return '' + return "" if item is None else item.name with click.progressbar( - binaries, + to_download, label='Downloading binaries', fill_char=click.style('#', fg='yellow'), item_show_func=show_item, ) as iterator: for binary in iterator: - binary.download() + print(f"Downloading {binary.url}") + download(binary.url, binary.path) return GLOBAL_TEMP_DIR diff --git a/src/prisma/binaries/binary.py b/src/prisma/binaries/binary.py deleted file mode 100644 index a09f500da..000000000 --- a/src/prisma/binaries/binary.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import logging -from pathlib import Path -from pydantic import BaseModel - -from . import platform -from .utils import download -from .constants import GLOBAL_TEMP_DIR, PRISMA_URL, PRISMA_VERSION - - -__all__ = ('Binary',) - -log: logging.Logger = logging.getLogger(__name__) - - -class Binary(BaseModel): - name: str - env: str - - def download(self) -> None: - # TODO: respect schema binary options - url = self.url - dest = self.path - - if dest.exists(): - log.debug('%s is cached, skipping download', self.name) - return - - log.debug('Downloading from %s to %s', url, dest) - download(url, str(dest.absolute())) - log.debug('Downloaded %s to %s', self.name, dest.absolute()) - - @property - def url(self) -> str: - return platform.check_for_extension(PRISMA_URL).format( - version=PRISMA_VERSION, platform=platform.name() - ) - - @property - def path(self) -> Path: - env = os.environ.get(self.env) - if env is not None: - log.debug('Using environment variable location: %s for %s', env, self.name) - return Path(env) - - return GLOBAL_TEMP_DIR.joinpath(platform.check_for_extension(self.name)) diff --git a/src/prisma/binaries/constants.py b/src/prisma/binaries/constants.py deleted file mode 100644 index 95cdb1b71..000000000 --- a/src/prisma/binaries/constants.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import tempfile -from pathlib import Path - -from . import platform - - -__all__ = ( - 'PRISMA_URL', - 'PRISMA_VERSION', - 'ENGINE_URL', - 'ENGINE_VERSION', - 'GLOBAL_TEMP_DIR', - 'PRISMA_CLI_NAME', -) - - -# TODO: if this version changes but the engine version -# doesn't change then the CLI is incorrectly cached -# hardcoded CLI version version -PRISMA_VERSION = '3.7.0' - -# CLI binaries are stored here -PRISMA_URL = os.environ.get( - 'PRISMA_CLI_URL', - 'https://prisma-photongo.s3-eu-west-1.amazonaws.com/prisma-cli-{version}-{platform}.gz', -) - -# engine binaries are stored here -ENGINE_URL = os.environ.get( - 'PRISMA_ENGINE_URL', 'https://binaries.prisma.sh/all_commits/{0}/{1}/{2}.gz' -) - -# versions can be found under https://github.com/prisma/prisma-engine/commits/master -ENGINE_VERSION = os.environ.get( - 'PRISMA_ENGINE_VERSION', '8746e055198f517658c08a0c426c7eec87f5a85f' -) - -# where the binaries live -GLOBAL_TEMP_DIR = ( - Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION -) - -# local file path for the prisma CLI -if platform.name() == 'windows': # pyright: reportConstantRedefinition=false - PRISMA_CLI_NAME = f'prisma-cli-{platform.name()}.exe' -else: - PRISMA_CLI_NAME = f'prisma-cli-{platform.name()}' diff --git a/src/prisma/binaries/utils.py b/src/prisma/binaries/download.py similarity index 84% rename from src/prisma/binaries/utils.py rename to src/prisma/binaries/download.py index abfd7cd25..49521ae92 100644 --- a/src/prisma/binaries/utils.py +++ b/src/prisma/binaries/download.py @@ -7,11 +7,11 @@ from ..utils import maybe_async_run -def download(url: str, to: str) -> None: +def download(url: str, to: Path) -> None: Path(to).parent.mkdir(parents=True, exist_ok=True) - tmp = to + '.tmp' - tar = to + '.gz.tmp' + tmp = to.with_suffix(".tmp") + tar = to.with_suffix(".gz.tmp") maybe_async_run(client.download, url, tar) # decompress to a tmp file before replacing the original diff --git a/src/prisma/binaries/engine.py b/src/prisma/binaries/engine.py deleted file mode 100644 index 96aebcee0..000000000 --- a/src/prisma/binaries/engine.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import logging -from pathlib import Path - -from . import platform -from .binary import Binary -from .constants import ENGINE_URL, ENGINE_VERSION, GLOBAL_TEMP_DIR - - -__all__ = ('Engine',) - -log: logging.Logger = logging.getLogger(__name__) - - -class Engine(Binary): - name: str - env: str - - @property - def url(self) -> str: - return platform.check_for_extension( - ENGINE_URL.format(ENGINE_VERSION, platform.binary_platform(), self.name) - ) - - @property - def path(self) -> Path: - env = os.environ.get(self.env) - if env is not None: - log.debug('Using environment variable location: %s for %s', env, self.name) - return Path(env) - - binary_name = platform.binary_platform() - return Path( - platform.check_for_extension( - str(GLOBAL_TEMP_DIR.joinpath(f'prisma-{self.name}-{binary_name}')) - ) - ) diff --git a/src/prisma/cli/prisma.py b/src/prisma/cli/prisma.py index 775da5beb..3f2cee8e5 100644 --- a/src/prisma/cli/prisma.py +++ b/src/prisma/cli/prisma.py @@ -34,8 +34,12 @@ def run( } env = {**default_env, **env} if env is not None else default_env # ensure the client uses our engine binaries + # TODO: this is a hack, probably there's a better way to do this + engine_env_dict = binaries.settings.dict() for engine in binaries.ENGINES: - env[engine.env] = str(engine.path.absolute()) + for engine_env, engine_path in engine_env_dict.items(): + if engine_path == engine.path: + env[engine_env] = str(engine_path.absolute()) process = subprocess.run( [str(path.absolute()), *args], diff --git a/src/prisma/generator/models.py b/src/prisma/generator/models.py index d5b1b0dd1..2cc328557 100644 --- a/src/prisma/generator/models.py +++ b/src/prisma/generator/models.py @@ -46,7 +46,7 @@ from .._compat import validator, root_validator, cached_property from .._constants import QUERY_BUILDER_ALIASES from ..errors import UnsupportedListTypeError -from ..binaries.constants import ENGINE_VERSION, PRISMA_VERSION +from ..binaries import ENGINE_VERSION, PRISMA_VERSION __all__ = ( From 66935b4cc5b97098f87416a1822ffcf33b2b6ea6 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Wed, 19 Jan 2022 21:54:18 -0500 Subject: [PATCH 06/55] chore: some minor cleanup and try fix linux CLI downloading --- src/prisma/binaries/binaries.py | 19 +++++++------------ src/prisma/binaries/download.py | 4 ++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 2971e7f5b..22e32b214 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -1,20 +1,17 @@ # -*- coding: utf-8 -*- +import os import logging +import tempfile +from platform import system from pathlib import Path from typing import Optional, List import click - - -import os -import tempfile -from pathlib import Path -from typing import List - -from prisma.binaries.download import download +from pydantic import BaseSettings, Field from . import platform +from .download import download # PLATFORMS: List[str] = [ @@ -40,8 +37,6 @@ PLATFORM = platform.get_platform() -from pydantic import BaseSettings, Field - # TODO: if this version changes but the engine version # doesn't change then the CLI is incorrectly cached @@ -56,6 +51,7 @@ Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) PLATFORM_EXE_EXTENSION = ".exe" if PLATFORM == "windows" else "" +CLI_PLATFORM = system().lower() def default_in_temp(name: str): @@ -80,8 +76,7 @@ class PrismaSettings(BaseSettings): settings = PrismaSettings() -# CLI binaries are stored here -PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{PLATFORM}{PLATFORM_EXE_EXTENSION}" +PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" PRISMA_CLI_PATH = GLOBAL_TEMP_DIR / PRISMA_CLI_NAME PRISMA_CLI_URL = f"{settings.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" diff --git a/src/prisma/binaries/download.py b/src/prisma/binaries/download.py index 49521ae92..49f08b657 100644 --- a/src/prisma/binaries/download.py +++ b/src/prisma/binaries/download.py @@ -10,8 +10,8 @@ def download(url: str, to: Path) -> None: Path(to).parent.mkdir(parents=True, exist_ok=True) - tmp = to.with_suffix(".tmp") - tar = to.with_suffix(".gz.tmp") + tmp = to.with_suffix('.tmp') + tar = to.with_suffix('.gz.tmp') maybe_async_run(client.download, url, tar) # decompress to a tmp file before replacing the original From 79fbf5cdf0b8cc291d80ebcc632cad92e7daa7ac Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Wed, 19 Jan 2022 21:56:45 -0500 Subject: [PATCH 07/55] chore: fix python3.7 --- src/prisma/binaries/platform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 550221f2e..e0cd0b5f3 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -4,7 +4,9 @@ import sys import distro from functools import lru_cache -from typing import Optional, TypedDict +from typing import Optional + +from .._types import TypedDict class OsSettings(TypedDict): From d9b5fa261d5b2c2784040412d8a7a7918d229057 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 14:48:31 +0100 Subject: [PATCH 08/55] feat: change OsSettings from TypedDict to basic class should simplify syntax and usage --- src/prisma/binaries/binaries.py | 8 ++++---- src/prisma/binaries/platform.py | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 22e32b214..b72bc4006 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -3,7 +3,6 @@ import os import logging import tempfile -from platform import system from pathlib import Path from typing import Optional, List @@ -35,8 +34,10 @@ # 'arm', # ] -PLATFORM = platform.get_platform() - +# Get system information +OS_SETTINGS = platform.get_os_settings() +PLATFORM = platform.resolve_platform(OS_SETTINGS) +CLI_PLATFORM = OS_SETTINGS.system # TODO: if this version changes but the engine version # doesn't change then the CLI is incorrectly cached @@ -51,7 +52,6 @@ Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) PLATFORM_EXE_EXTENSION = ".exe" if PLATFORM == "windows" else "" -CLI_PLATFORM = system().lower() def default_in_temp(name: str): diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index e0cd0b5f3..2435c60f3 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -6,15 +6,21 @@ from functools import lru_cache from typing import Optional -from .._types import TypedDict - -class OsSettings(TypedDict): +class OsSettings: system: str machine: str libssl: str distro: Optional[str] + def __init__( + self, system: str, machine: str, libssl: str, distro: Optional[str] + ) -> None: + self.system = system + self.machine = machine + self.libssl = libssl + self.distro = distro + @lru_cache() def get_openssl() -> str: @@ -72,10 +78,10 @@ def get_os_settings() -> OsSettings: def resolve_platform(os: OsSettings) -> str: system, machine, libssl, distro = ( - os['system'], - os['machine'], - os['libssl'], - os['distro'], + os.system, + os.machine, + os.libssl, + os.distro, ) if system == "darwin" and machine == "aarch64": @@ -101,7 +107,3 @@ def resolve_platform(os: OsSettings) -> str: elif distro: return f"{distro}-openssl-{libssl}" return "debian-openssl-1.1.x" # default fallback - - -def get_platform() -> str: - return resolve_platform(get_os_settings()) From f510d985e05edfebaf435531e04ec4d132f5f341 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 16:59:43 +0100 Subject: [PATCH 09/55] chore: clean up tests/test_binaries.py from old mechanics TODO: add new tests --- tests/test_binaries.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 244366fb8..07ee54b47 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -4,28 +4,3 @@ from _pytest.logging import LogCaptureFixture from prisma.utils import temp_env_update -from prisma.binaries import BINARIES, ENGINES, Engine -from prisma.binaries.constants import PRISMA_CLI_NAME - - -def test_skips_cached_binary(caplog: LogCaptureFixture) -> None: - """Downloading an already existing binary does not actually do anything""" - # NOTE: this is not a great way to test this - binary = BINARIES[0] - binary.download() - assert 'is cached' in caplog.records[0].message - - -@pytest.mark.parametrize('engine', ENGINES) -def test_engine_resolves_env_override(engine: Engine) -> None: - """Env variables override the default path for an engine binary""" - with temp_env_update({engine.env: 'foo'}): - assert engine.path == Path('foo') - - -def test_cli_binary_resolves_env_override() -> None: - """Env variable overrides the default path for the CLI binary""" - binary = BINARIES[-1] - assert binary.name == PRISMA_CLI_NAME - with temp_env_update({binary.env: 'foo'}): - assert binary.path == Path('foo') From 789b52d51d404722930278b9f8196ef6ceaad382 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:46:42 +0100 Subject: [PATCH 10/55] chore: remove unnecessary Path call --- src/prisma/binaries/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prisma/binaries/download.py b/src/prisma/binaries/download.py index 49f08b657..399ed6cb2 100644 --- a/src/prisma/binaries/download.py +++ b/src/prisma/binaries/download.py @@ -8,7 +8,7 @@ def download(url: str, to: Path) -> None: - Path(to).parent.mkdir(parents=True, exist_ok=True) + to.parent.mkdir(parents=True, exist_ok=True) tmp = to.with_suffix('.tmp') tar = to.with_suffix('.gz.tmp') From ff838c0e4266d3c804204fa6915f3ef3e8566a88 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:47:05 +0100 Subject: [PATCH 11/55] chore: comment out legacy ensure() function TODO: remove it? --- src/prisma/engine/utils.py | 120 ++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 62 deletions(-) diff --git a/src/prisma/engine/utils.py b/src/prisma/engine/utils.py index c6944779a..3a3fb15db 100644 --- a/src/prisma/engine/utils.py +++ b/src/prisma/engine/utils.py @@ -1,18 +1,12 @@ -import os -import sys -import time import socket import logging -import subprocess -from pathlib import Path + from typing import NoReturn, Dict, Type, Any from . import errors from .. import errors as prisma_errors from ..http import Response -from ..utils import time_since -from ..binaries import GLOBAL_TEMP_DIR, ENGINE_VERSION, platform log: logging.Logger = logging.getLogger(__name__) @@ -25,65 +19,67 @@ } -def ensure() -> Path: - start_time = time.monotonic() - file = None - force_version = True - binary_name = platform.check_for_extension(platform.binary_platform()) - - name = f'prisma-query-engine-{binary_name}' - local_path = Path.cwd().joinpath(name) - global_path = GLOBAL_TEMP_DIR.joinpath(name) - - log.debug('Expecting local query engine %s', local_path) - log.debug('Expecting global query engine %s', global_path) - - # TODO: this resolving should be moved to the binary class - binary = os.environ.get('PRISMA_QUERY_ENGINE_BINARY') - if binary: - log.debug('PRISMA_QUERY_ENGINE_BINARY is defined, using %s', binary) - - if not Path(binary).exists(): - raise errors.BinaryNotFoundError( - 'PRISMA_QUERY_ENGINE_BINARY was provided, ' - f'but no query engine was found at {binary}' - ) - - file = Path(binary) - force_version = False - elif local_path.exists(): - file = local_path - log.debug('Query engine found in the working directory') - elif global_path.exists(): - file = global_path - log.debug('Query engine found in the global path') - - if not file: - raise errors.BinaryNotFoundError( - f'Expected {local_path} or {global_path} but neither were found.\n' - 'Try running prisma py fetch' - ) - - start_version = time.monotonic() - process = subprocess.run( - [file.absolute(), '--version'], stdout=subprocess.PIPE, check=True - ) - log.debug('Version check took %s', time_since(start_version)) +# def ensure() -> Path: +# start_time = time.monotonic() +# file = None +# force_version = True - version = ( - str(process.stdout, sys.getdefaultencoding()) - .replace('query-engine', '') - .strip() - ) - log.debug('Using query engine version %s', version) +# binary_name = platform.check_for_extension(platform.binary_platform()) + +# name = f'prisma-query-engine-{binary_name}' + +# local_path = Path.cwd().joinpath(name) +# global_path = GLOBAL_TEMP_DIR.joinpath(name) + +# log.debug('Expecting local query engine %s', local_path) +# log.debug('Expecting global query engine %s', global_path) + +# # TODO: this resolving should be moved to the binary class +# binary = os.environ.get('PRISMA_QUERY_ENGINE_BINARY') +# if binary: +# log.debug('PRISMA_QUERY_ENGINE_BINARY is defined, using %s', binary) + +# if not Path(binary).exists(): +# raise errors.BinaryNotFoundError( +# 'PRISMA_QUERY_ENGINE_BINARY was provided, ' +# f'but no query engine was found at {binary}' +# ) + +# file = Path(binary) +# force_version = False +# elif local_path.exists(): +# file = local_path +# log.debug('Query engine found in the working directory') +# elif global_path.exists(): +# file = global_path +# log.debug('Query engine found in the global path') + +# if not file: +# raise errors.BinaryNotFoundError( +# f'Expected {local_path} or {global_path} but neither were found.\n' +# 'Try running prisma py fetch' +# ) + +# start_version = time.monotonic() +# process = subprocess.run( +# [file.absolute(), '--version'], stdout=subprocess.PIPE, check=True +# ) +# log.debug('Version check took %s', time_since(start_version)) + +# version = ( +# str(process.stdout, sys.getdefaultencoding()) +# .replace('query-engine', '') +# .strip() +# ) +# log.debug('Using query engine version %s', version) - if force_version and version != ENGINE_VERSION: - raise errors.MismatchedVersionsError(expected=ENGINE_VERSION, got=version) +# if force_version and version != ENGINE_VERSION: +# raise errors.MismatchedVersionsError(expected=ENGINE_VERSION, got=version) - log.debug('Using query engine at %s', file) - log.debug('Ensuring query engine took: %s', time_since(start_time)) +# log.debug('Using query engine at %s', file) +# log.debug('Ensuring query engine took: %s', time_since(start_time)) - return file +# return file def get_open_port() -> int: From 5dea8cfb1ac5ae6963a826762d4508fcfb812766 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:48:09 +0100 Subject: [PATCH 12/55] chore: use ensure_cached() instead of ensure() ensure() removed in ff838c0e4266d3c804204fa6915f3ef3e8566a88 --- src/prisma/generator/templates/engine/query.py.jinja | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/prisma/generator/templates/engine/query.py.jinja b/src/prisma/generator/templates/engine/query.py.jinja index d7bf64fde..ff3579cec 100644 --- a/src/prisma/generator/templates/engine/query.py.jinja +++ b/src/prisma/generator/templates/engine/query.py.jinja @@ -14,7 +14,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import platform +from ..binaries import settings as binaries_settings, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -77,10 +77,12 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + + ensure_cached() + self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY try: - {{ maybe_await }}self.spawn(file, timeout=timeout, datasources=datasources) + {{ maybe_await }}self.spawn(self.file, timeout=timeout, datasources=datasources) except Exception: self.close() raise From ced6e60225cbf7ebb2e48878c817b79aee4cd158 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:49:50 +0100 Subject: [PATCH 13/55] chore: comment out local path test --- tests/test_engine.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index b70832538..3911caf84 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -87,26 +87,26 @@ def test_mismatched_version_error(fake_process: FakeProcess) -> None: ) -def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: - """Query engine in current directory required to be the expected version""" - fake_engine = testdir.path / platform.check_for_extension( - f'prisma-query-engine-{platform.binary_platform()}' - ) - fake_engine.touch() - - fake_process.register_subprocess( - [fake_engine, '--version'], # type: ignore[list-item] - stdout='query-engine a-different-hash', - ) - with pytest.raises(errors.MismatchedVersionsError): - path = utils.ensure() - - fake_process.register_subprocess( - [fake_engine, '--version'], # type: ignore[list-item] - stdout=f'query-engine {ENGINE_VERSION}', - ) - path = utils.ensure() - assert path == fake_engine +# def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: +# """Query engine in current directory required to be the expected version""" +# fake_engine = testdir.path / platform.check_for_extension( +# f'prisma-query-engine-{platform.binary_platform()}' +# ) +# fake_engine.touch() + +# fake_process.register_subprocess( +# [fake_engine, '--version'], # type: ignore[list-item] +# stdout='query-engine a-different-hash', +# ) +# with pytest.raises(errors.MismatchedVersionsError): +# path = utils.ensure() + +# fake_process.register_subprocess( +# [fake_engine, '--version'], # type: ignore[list-item] +# stdout=f'query-engine {ENGINE_VERSION}', +# ) +# path = utils.ensure() +# assert path == fake_engine def test_ensure_env_override(testdir: Testdir, fake_process: FakeProcess) -> None: From df6ef396434ac5d88f744d817f2bc532978d6dfd Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 22:50:25 +0100 Subject: [PATCH 14/55] chore: import PRISMA_VERSION from prisma.binaries --- scripts/docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docs.py b/scripts/docs.py index e8f5eee01..53ac49aca 100644 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -1,6 +1,6 @@ import re from pathlib import Path -from prisma.binaries.constants import PRISMA_VERSION +from prisma.binaries import PRISMA_VERSION ROOTDIR = Path(__file__).parent.parent From f60e9b06b062ec052f642d309c1310f6dc7d9fcd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Jan 2022 21:50:42 +0000 Subject: [PATCH 15/55] chore(pre-commit.ci): auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/prisma/generator/templates/engine/query.py.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prisma/generator/templates/engine/query.py.jinja b/src/prisma/generator/templates/engine/query.py.jinja index ff3579cec..bfd222517 100644 --- a/src/prisma/generator/templates/engine/query.py.jinja +++ b/src/prisma/generator/templates/engine/query.py.jinja @@ -79,7 +79,7 @@ class QueryEngine(HTTPEngine): start = time.monotonic() ensure_cached() - self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY + self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY try: {{ maybe_await }}self.spawn(self.file, timeout=timeout, datasources=datasources) From e16086062d643cf60ad544aea399acac4deda84c Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 23:00:27 +0100 Subject: [PATCH 16/55] feat: add `is_windows` method on OsSettings --- src/prisma/binaries/binaries.py | 3 ++- src/prisma/binaries/platform.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index b72bc4006..c517c5dd0 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -51,7 +51,7 @@ GLOBAL_TEMP_DIR = ( Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) -PLATFORM_EXE_EXTENSION = ".exe" if PLATFORM == "windows" else "" +PLATFORM_EXE_EXTENSION = ".exe" if OS_SETTINGS.is_windows() else "" def default_in_temp(name: str): @@ -76,6 +76,7 @@ class PrismaSettings(BaseSettings): settings = PrismaSettings() + PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" PRISMA_CLI_PATH = GLOBAL_TEMP_DIR / PRISMA_CLI_NAME PRISMA_CLI_URL = f"{settings.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 2435c60f3..f98fd215f 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -21,6 +21,9 @@ def __init__( self.libssl = libssl self.distro = distro + def is_windows(self) -> bool: + return self.system.lower() == 'windows' + @lru_cache() def get_openssl() -> str: From 18f2a80a61aa1ed9e11583a08a9d4804695f148d Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 20 Jan 2022 23:00:40 +0100 Subject: [PATCH 17/55] chore: use is_windows in query template --- src/prisma/generator/templates/engine/query.py.jinja | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/prisma/generator/templates/engine/query.py.jinja b/src/prisma/generator/templates/engine/query.py.jinja index ff3579cec..901d24217 100644 --- a/src/prisma/generator/templates/engine/query.py.jinja +++ b/src/prisma/generator/templates/engine/query.py.jinja @@ -14,7 +14,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import settings as binaries_settings, ensure_cached +from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -44,9 +44,8 @@ class QueryEngine(HTTPEngine): def close(self) -> None: log.debug('Disconnecting query engine...') - if self.process is not None: - if platform.name() == 'windows': + if OS_SETTINGS.is_windows(): self.process.kill() else: self.process.send_signal(signal.SIGINT) From a30b98fc57016e6f9fe1c7aecb072f4ae6a2c6b8 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:11:43 +0100 Subject: [PATCH 18/55] chore: linter use lazy string formatting for logging --- src/prisma/binaries/binaries.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index c517c5dd0..c614d16cb 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -119,9 +119,9 @@ def ensure_cached() -> Path: to_download: List[Binary] = [] for binary in BINARIES: if binary.path.exists(): - log.debug(f"{binary.name} is cached, skipping download") + log.debug(f"%s is cached, skipping download" % binary.name) continue - log.debug(f"{binary.name} is not cached, will download") + log.debug(f"%s is not cached, will download" % binary.name) to_download.append(binary) if len(to_download) == 0: @@ -138,7 +138,7 @@ def show_item(item: Optional[Binary]) -> str: item_show_func=show_item, ) as iterator: for binary in iterator: - print(f"Downloading {binary.url}") + log.debug("Downloading %s from %s" % (binary.name, binary.url)) download(binary.url, binary.path) return GLOBAL_TEMP_DIR From 98605a0a68d3ffe93e4513af657660cf02030348 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:17:49 +0100 Subject: [PATCH 19/55] chore: disable tests that were using `utils.ensure()` TODO: can't we use `ensureCached` like in 5dea8cfb1ac5ae6963a826762d4508fcfb812766? --- tests/test_engine.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 3911caf84..5253b4f9e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -64,12 +64,13 @@ def mock_exists(path: Path) -> bool: monkeypatch.setattr(Path, 'exists', mock_exists, raising=True) - with pytest.raises(errors.BinaryNotFoundError) as exc: - utils.ensure() + # TODO: What to do with this? + # with pytest.raises(errors.BinaryNotFoundError) as exc: + # utils.ensure() - assert exc.match( - r'Expected .* or .* but neither were found\.\nTry running prisma py fetch' - ) + # assert exc.match( + # r'Expected .* or .* but neither were found\.\nTry running prisma py fetch' + # ) def test_mismatched_version_error(fake_process: FakeProcess) -> None: @@ -79,12 +80,13 @@ def test_mismatched_version_error(fake_process: FakeProcess) -> None: stdout='query-engine unexpected-hash', ) - with pytest.raises(errors.MismatchedVersionsError) as exc: - utils.ensure() + # TODO: What to do with this? + # with pytest.raises(errors.MismatchedVersionsError) as exc: + # utils.ensure() - assert exc.match( - f'Expected query engine version `{ENGINE_VERSION}` but got `unexpected-hash`' - ) + # assert exc.match( + # f'Expected query engine version `{ENGINE_VERSION}` but got `unexpected-hash`' + # ) # def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: @@ -119,18 +121,21 @@ def test_ensure_env_override(testdir: Testdir, fake_process: FakeProcess) -> Non stdout='query-engine a-different-hash', ) - with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': str(fake_engine)}): - path = utils.ensure() + # TODO: What to do with this? + # with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': str(fake_engine)}): + # path = utils.ensure() - assert path == fake_engine + # assert path == fake_engine def test_ensure_env_override_does_not_exist() -> None: """Query engine path in environment variable not found raises an error""" - with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': 'foo'}): - with pytest.raises(errors.BinaryNotFoundError) as exc: - utils.ensure() - assert exc.match( - r'PRISMA_QUERY_ENGINE_BINARY was provided, but no query engine was found at foo' - ) + # TODO: What to do with this? + # with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': 'foo'}): + # with pytest.raises(errors.BinaryNotFoundError) as exc: + # utils.ensure() + + # assert exc.match( + # r'PRISMA_QUERY_ENGINE_BINARY was provided, but no query engine was found at foo' + # ) From 34855165f46b9063bf96a080a124e563905e0a69 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:22:33 +0100 Subject: [PATCH 20/55] chore: add types to prisma/binaries/binaries.py --- src/prisma/binaries/binaries.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index c614d16cb..b79095dba 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -4,7 +4,7 @@ import logging import tempfile from pathlib import Path -from typing import Optional, List +from typing import Callable, Optional, List import click from pydantic import BaseSettings, Field @@ -35,26 +35,26 @@ # ] # Get system information -OS_SETTINGS = platform.get_os_settings() -PLATFORM = platform.resolve_platform(OS_SETTINGS) -CLI_PLATFORM = OS_SETTINGS.system +OS_SETTINGS: platform.OsSettings = platform.get_os_settings() +PLATFORM: str = platform.resolve_platform(OS_SETTINGS) +CLI_PLATFORM: str = OS_SETTINGS.system # TODO: if this version changes but the engine version # doesn't change then the CLI is incorrectly cached # hardcoded CLI version version -PRISMA_VERSION = '3.7.0' +PRISMA_VERSION: str = '3.7.0' # versions can be found under https://github.com/prisma/prisma-engine/commits/main ENGINE_VERSION = os.environ.get( 'PRISMA_ENGINE_VERSION', '8746e055198f517658c08a0c426c7eec87f5a85f' ) -GLOBAL_TEMP_DIR = ( +GLOBAL_TEMP_DIR: Path = ( Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) -PLATFORM_EXE_EXTENSION = ".exe" if OS_SETTINGS.is_windows() else "" +PLATFORM_EXE_EXTENSION: str = ".exe" if OS_SETTINGS.is_windows() else "" -def default_in_temp(name: str): +def default_in_temp(name: str) -> Callable[[], Path]: return lambda: (GLOBAL_TEMP_DIR / name).with_suffix(PLATFORM_EXE_EXTENSION) @@ -74,7 +74,7 @@ class PrismaSettings(BaseSettings): PRISMA_CLI_BINARY_TARGETS: List[str] = Field(default_factory=list) -settings = PrismaSettings() +settings: PrismaSettings = PrismaSettings() PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" From aa121ba3b870714c76abf1df9139fc53bc4d2128 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:30:40 +0100 Subject: [PATCH 21/55] chore: change old usage of binary_platform to PLATFORM --- src/prisma/cli/commands/version.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/prisma/cli/commands/version.py b/src/prisma/cli/commands/version.py index cc4afdf4e..9c6d7aa9f 100644 --- a/src/prisma/cli/commands/version.py +++ b/src/prisma/cli/commands/version.py @@ -7,8 +7,7 @@ from ..utils import pretty_info from ... import __version__ -from ...binaries import PRISMA_VERSION, ENGINE_VERSION -from ...binaries.platform import binary_platform +from ...binaries import PRISMA_VERSION, ENGINE_VERSION, PLATFORM @click.command( @@ -38,7 +37,7 @@ def cli(output_json: bool) -> None: info = { 'prisma': PRISMA_VERSION, 'prisma client python': __version__, - 'platform': binary_platform(), + 'platform': PLATFORM, 'engines': ENGINE_VERSION, 'install path': str(Path(__file__).resolve().parent.parent.parent), 'installed extras': installed, From ee1d2fda9b3fe4ae6c233671aa628e91644dac3f Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:31:29 +0100 Subject: [PATCH 22/55] chore: sort imports in prisma/binaries/ --- src/prisma/binaries/binaries.py | 5 ++--- src/prisma/binaries/download.py | 2 +- src/prisma/binaries/platform.py | 5 +++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index b79095dba..f43c57f62 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -import os import logging +import os import tempfile from pathlib import Path -from typing import Callable, Optional, List +from typing import Callable, List, Optional import click from pydantic import BaseSettings, Field @@ -12,7 +12,6 @@ from . import platform from .download import download - # PLATFORMS: List[str] = [ # 'darwin', # 'darwin-arm64', diff --git a/src/prisma/binaries/download.py b/src/prisma/binaries/download.py index 399ed6cb2..85f367027 100644 --- a/src/prisma/binaries/download.py +++ b/src/prisma/binaries/download.py @@ -1,5 +1,5 @@ -import os import gzip +import os import shutil from pathlib import Path diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index f98fd215f..653d1ccd4 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -1,11 +1,12 @@ +import platform import re import subprocess -import platform import sys -import distro from functools import lru_cache from typing import Optional +import distro + class OsSettings: system: str From 6fa32ff03f5d8e5e548c5f44fad015c44ee7b58f Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 21 Jan 2022 18:58:51 +0100 Subject: [PATCH 23/55] feat: implement download method on `Binary` --- src/prisma/binaries/binaries.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 76a68d240..08c104c3d 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -98,6 +98,9 @@ def __init__(self, name: str, path: Path, *, url: Optional[str] = None): self.path = path self.url = engine_url_for(name) if url is None else url + def download(self) -> None: + download(self.url, self.path) + ENGINES: List[Binary] = [ Binary(name='query-engine', path=settings.PRISMA_QUERY_ENGINE_BINARY), @@ -138,7 +141,7 @@ def show_item(item: Optional[Binary]) -> str: ) as iterator: for binary in iterator: log.debug("Downloading %s from %s" % (binary.name, binary.url)) - download(binary.url, binary.path) + binary.download() return GLOBAL_TEMP_DIR From c9e6ac32aa12be8b04ffe8d7d55ad94895d002ad Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 02:59:35 +0100 Subject: [PATCH 24/55] feat: add `remove` method on Binary classs --- src/prisma/binaries/binaries.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 08c104c3d..1276d3613 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -2,7 +2,6 @@ import logging import os -import tempfile from pathlib import Path from typing import Callable, List, Optional @@ -101,6 +100,10 @@ def __init__(self, name: str, path: Path, *, url: Optional[str] = None): def download(self) -> None: download(self.url, self.path) + def remove(self) -> None: + # This might fail if file is still in use, which happens during tests (somehow)! + self.path.unlink(missing_ok=True) + ENGINES: List[Binary] = [ Binary(name='query-engine', path=settings.PRISMA_QUERY_ENGINE_BINARY), @@ -149,5 +152,4 @@ def show_item(item: Optional[Binary]) -> str: def remove_all() -> None: """Remove all downloaded binaries""" for binary in BINARIES: - if binary.path.exists(): - binary.path.unlink() + binary.remove() From 2b4405b60217e3401ff822a95dc2381da5411d64 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 02:59:57 +0100 Subject: [PATCH 25/55] chore: make download more streamlined --- src/prisma/binaries/download.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/prisma/binaries/download.py b/src/prisma/binaries/download.py index 85f367027..1012e85b9 100644 --- a/src/prisma/binaries/download.py +++ b/src/prisma/binaries/download.py @@ -10,22 +10,20 @@ def download(url: str, to: Path) -> None: to.parent.mkdir(parents=True, exist_ok=True) - tmp = to.with_suffix('.tmp') tar = to.with_suffix('.gz.tmp') maybe_async_run(client.download, url, tar) # decompress to a tmp file before replacing the original with gzip.open(tar, 'rb') as f_in: - with open(tmp, 'wb') as f_out: + with open(to, 'wb') as f_out: shutil.copyfileobj(f_in, f_out) # chmod +x - status = os.stat(tmp) - os.chmod(tmp, status.st_mode | 0o111) + from stat import S_IXUSR, S_IWUSR, S_IRUSR + + os.chmod(to, S_IXUSR | S_IWUSR | S_IRUSR) # override the original - shutil.copy(tmp, to) # remove temporary files os.remove(tar) - os.remove(tmp) From d88afb64839b5d27ff45ad1e9962c1c5e775d6e6 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 03:00:14 +0100 Subject: [PATCH 26/55] chore(binaries.py): forgot an import --- src/prisma/binaries/binaries.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 1276d3613..c37798be7 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -2,6 +2,7 @@ import logging import os +import tempfile from pathlib import Path from typing import Callable, List, Optional From c9422ccada844f6151bca56ec5c6520796354d9b Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 03:00:31 +0100 Subject: [PATCH 27/55] chore(platform.py): add pyright directive to suppress stub errors --- src/prisma/binaries/platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 653d1ccd4..32f2267fc 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -5,7 +5,7 @@ from functools import lru_cache from typing import Optional -import distro +import distro # pyright: reportMissingTypeStubs=false class OsSettings: From 1069d1d656a67387ca6c1259b4246c4b7b7ae1f1 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 03:01:24 +0100 Subject: [PATCH 28/55] chore(test_fetch.py): skip fetch tests when on windows Cannot remove files when running tests as they're somewhere still accessed --- tests/test_cli/test_fetch.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index b932dd15e..6237a3f9b 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -1,12 +1,12 @@ import random import shutil +import pytest from click.testing import Result - from prisma import binaries +from prisma.binaries.binaries import OS_SETTINGS from tests.utils import Runner - # TODO: this could probably mess up other tests if one of these # tests fails mid run, as the global binaries are deleted @@ -26,6 +26,9 @@ def test_fetch(runner: Runner) -> None: assert_success(runner.invoke(['py', 'fetch'])) +@pytest.mark.skipif( + OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" +) def test_fetch_one_binary_missing(runner: Runner) -> None: """Downloads a binary if it is missing""" binary = random.choice(binaries.BINARIES) @@ -36,6 +39,9 @@ def test_fetch_one_binary_missing(runner: Runner) -> None: assert_success(runner.invoke(['py', 'fetch'])) +@pytest.mark.skipif( + OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" +) def test_fetch_force(runner: Runner) -> None: """Passing --force re-downloads an already existing binary""" binary = random.choice(binaries.BINARIES) @@ -51,13 +57,14 @@ def test_fetch_force(runner: Runner) -> None: # ensure downloaded the same as before assert old_stat.st_size == new_stat.st_size +d - +@pytest.mark.skipif( + OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" +) def test_fetch_force_no_dir(runner: Runner) -> None: """Passing --force when the base directory does not exist""" - binaries.remove_all() shutil.rmtree(str(binaries.GLOBAL_TEMP_DIR)) - binary = binaries.BINARIES[0] assert not binary.path.exists() From 3cfadd2ce034d3c4715833b5e9602c223f1479e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 02:01:40 +0000 Subject: [PATCH 29/55] chore(pre-commit.ci): auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cli/test_fetch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index 6237a3f9b..e0cdaa759 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -57,8 +57,11 @@ def test_fetch_force(runner: Runner) -> None: # ensure downloaded the same as before assert old_stat.st_size == new_stat.st_size + + d + @pytest.mark.skipif( OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" ) From 0158efbc3cb75a50b9cc07603e31efaa3e868527 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Mon, 24 Jan 2022 03:05:04 +0100 Subject: [PATCH 30/55] chore(test_fetch.py): remove randomly placed letter --- tests/test_cli/test_fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index 6237a3f9b..9c068b658 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -57,7 +57,7 @@ def test_fetch_force(runner: Runner) -> None: # ensure downloaded the same as before assert old_stat.st_size == new_stat.st_size -d + @pytest.mark.skipif( OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" From d319391c87460831c1ac55a0294233c18b8643b1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 02:06:08 +0000 Subject: [PATCH 31/55] chore(pre-commit.ci): auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cli/test_fetch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index 60c778878..9c068b658 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -59,8 +59,6 @@ def test_fetch_force(runner: Runner) -> None: assert old_stat.st_size == new_stat.st_size - - @pytest.mark.skipif( OS_SETTINGS.is_windows(), reason="Open file issue, can't remove files on windows" ) From c1c321a7511125b1630762630de0e251ae0ca4dd Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 01:00:24 +0100 Subject: [PATCH 32/55] feat(platform.py): download 1.1.x binaries when using openssl 3 --- src/prisma/binaries/platform.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 32f2267fc..ca53f5dcb 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -31,7 +31,14 @@ def get_openssl() -> str: process = subprocess.run( ['openssl', 'version', '-v'], stdout=subprocess.PIPE, check=True ) - return parse_openssl_version(str(process.stdout, sys.getdefaultencoding())) + version = parse_openssl_version(str(process.stdout, sys.getdefaultencoding())) + if version not in ("1.0.x", "1.1.x"): + # If not 1.0 or 1.1 then it's most likely 3.0 + # Currently prisma doesn't provide binaries for 3.0 + # But we can use the latest stable 1.1.x + # See: https://github.com/prisma/prisma/issues/11356 + return "1.1.x" + return version @lru_cache() From c78934210f4dbc74627f02d0cefea6ba9d272575 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 01:00:42 +0100 Subject: [PATCH 33/55] chore: update snapshots --- .../test_exhaustive/test_async[enginequery.py].raw | 11 ++++++----- .../test_exhaustive/test_sync[enginequery.py].raw | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw index 542e02751..bbcc4325b 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw @@ -42,7 +42,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import platform +from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -72,9 +72,8 @@ class QueryEngine(HTTPEngine): def close(self) -> None: log.debug('Disconnecting query engine...') - if self.process is not None: - if platform.name() == 'windows': + if OS_SETTINGS.is_windows(): self.process.kill() else: self.process.send_signal(signal.SIGINT) @@ -102,10 +101,12 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + + ensure_cached() + self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY try: - await self.spawn(file, timeout=timeout, datasources=datasources) + await self.spawn(self.file, timeout=timeout, datasources=datasources) except Exception: self.close() raise diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw index 46b930283..8087e173b 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw @@ -42,7 +42,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import platform +from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -72,9 +72,8 @@ class QueryEngine(HTTPEngine): def close(self) -> None: log.debug('Disconnecting query engine...') - if self.process is not None: - if platform.name() == 'windows': + if OS_SETTINGS.is_windows(): self.process.kill() else: self.process.send_signal(signal.SIGINT) @@ -103,10 +102,12 @@ class QueryEngine(HTTPEngine): raise errors.AlreadyConnectedError('Already connected to the query engine') start = time.monotonic() - self.file = file = utils.ensure() + + ensure_cached() + self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY try: - self.spawn(file, timeout=timeout, datasources=datasources) + self.spawn(self.file, timeout=timeout, datasources=datasources) except Exception: self.close() raise From 05217e2b8284784a85c5bec29d5dfab5f794f8fb Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 01:16:58 +0100 Subject: [PATCH 34/55] chore: remove missing_ok from binary.remove --- src/prisma/binaries/binaries.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index c37798be7..9a38b90c9 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -103,7 +103,10 @@ def download(self) -> None: def remove(self) -> None: # This might fail if file is still in use, which happens during tests (somehow)! - self.path.unlink(missing_ok=True) + try: + self.path.unlink() + except FileNotFoundError: + pass ENGINES: List[Binary] = [ From 125e607543dcad483f34b3aec5f7aee92df93242 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 01:57:18 +0100 Subject: [PATCH 35/55] chore(binaries.py): add lazy logging format --- src/prisma/binaries/binaries.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 9a38b90c9..eee34366b 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -128,9 +128,9 @@ def ensure_cached() -> Path: to_download: List[Binary] = [] for binary in BINARIES: if binary.path.exists(): - log.debug(f"%s is cached, skipping download" % binary.name) + log.debug("%s is cached, skipping download", binary.name) continue - log.debug(f"%s is not cached, will download" % binary.name) + log.debug("%s is not cached, will download", binary.name) to_download.append(binary) if len(to_download) == 0: @@ -147,7 +147,7 @@ def show_item(item: Optional[Binary]) -> str: item_show_func=show_item, ) as iterator: for binary in iterator: - log.debug("Downloading %s from %s" % (binary.name, binary.url)) + log.debug("Downloading %s from %s", binary.name, binary.url) binary.download() return GLOBAL_TEMP_DIR From 4b4fc5f9894b2fca95bbf20b2ffbd1b3a10ce8f0 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:03:12 +0100 Subject: [PATCH 36/55] chore(platform.py): fix linter errors by modularizing platform resolving --- src/prisma/binaries/platform.py | 69 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index ca53f5dcb..3ca2d586e 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -54,18 +54,18 @@ def parse_openssl_version(string: str) -> str: def resolve_known_distro(distro_id: str, distro_like: str) -> Optional[str]: if distro_id == "alpine": return "musl" - elif distro_id == "raspbian": + if distro_id == "raspbian": return "arm" - elif distro_id == "nixos": + if distro_id == "nixos": return "nixos" - elif ( + if ( distro_id == "fedora" or "fedora" in distro_like or "rhel" in distro_like or "centros" in distro_like ): return "rhel" - elif ( + if ( distro_id == "ubuntu" or distro_id == "debian" or "ubuntu" in distro_like @@ -87,34 +87,39 @@ def get_os_settings() -> OsSettings: ) -def resolve_platform(os: OsSettings) -> str: - system, machine, libssl, distro = ( - os.system, - os.machine, - os.libssl, - os.distro, - ) - - if system == "darwin" and machine == "aarch64": +def resolve_darwin(os_settings: OsSettings) -> Optional[str]: + if os_settings.system == "darwin" and os_settings.machine == "aarch64": return "darwin-arm64" - elif system == "darwin": + if os_settings.system == "darwin": return "darwin" - elif system == "windows": - return "windows" - elif system == "freebsd": - return "freebsd" - elif system == "openbsd": - return "openbsd" - elif system == "netbsd": - return "netbsd" - elif system == "linux" and machine == "aarch64": - return f"linux-arm64-openssl-{libssl}" - elif system == "linux" and machine == "arm": - return f"linux-arm-openssl-{libssl}" - elif system == "linux" and distro == "musl": - return "linux-musl" - elif system == "linux" and distro == "nixos": - return "linux-nixos" - elif distro: - return f"{distro}-openssl-{libssl}" + return None + + +def resolve_linux(os_settings: OsSettings) -> Optional[str]: + if os_settings.system == "linux": + if os_settings.machine == "aarch64": + return f"linux-arm64-openssl-{os_settings.libssl}" + if os_settings.machine == "arm": + return f"linux-arm-openssl-{os_settings.libssl}" + if os_settings.distro == "musl": + return "linux-musl" + if os_settings.distro == "nixos": + return "linux-nixos" + if os_settings.distro: + return f"{os_settings.distro}-openssl-{os_settings.libssl}" + return None + + +def resolve_other_platforms(os_settings: OsSettings) -> Optional[str]: + if os_settings.system in ("windows", "freebsd", "openbsd", "netbsd"): + return os_settings.system + return None + + +def resolve_platform(os_settings: OsSettings) -> str: + resolvers = (resolve_darwin, resolve_linux, resolve_other_platforms) + for resolver in resolvers: + platform = resolver(os_settings) + if platform is not None: + return platform return "debian-openssl-1.1.x" # default fallback From 115a5dd667ee43340789de916b9befcc4f6dbc38 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:03:35 +0100 Subject: [PATCH 37/55] chore(binaries.py): fix linter errors by splitting up long string --- src/prisma/binaries/binaries.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index eee34366b..29b381296 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -82,7 +82,13 @@ class PrismaSettings(BaseSettings): def engine_url_for(name: str) -> str: - return f"{settings.PRISMA_ENGINES_MIRROR}/all_commits/{ENGINE_VERSION}/{PLATFORM}/{name}{PLATFORM_EXE_EXTENSION}.gz" + return ( + f"{settings.PRISMA_ENGINES_MIRROR}/" + "all_commits/" + f"{ENGINE_VERSION}/" + f"{PLATFORM}/" + f"{name}{PLATFORM_EXE_EXTENSION}.gz" + ) log: logging.Logger = logging.getLogger(__name__) From 63e8ec51e0b784519ef7a2e5a79143154027ac16 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:11:02 +0100 Subject: [PATCH 38/55] chore: remove empty test --- tests/test_binaries.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 07ee54b47..e69de29bb 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -1,6 +0,0 @@ -from pathlib import Path - -import pytest -from _pytest.logging import LogCaptureFixture - -from prisma.utils import temp_env_update From d4d5df1926082c31d86def7d8331a219ed3b7178 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:11:09 +0100 Subject: [PATCH 39/55] chore: remove unused imports --- tests/test_engine.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 5253b4f9e..4f5d52cc6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -8,10 +8,11 @@ from pytest_subprocess import FakeProcess from prisma import Client -from prisma.utils import temp_env_update -from prisma.binaries import platform -from prisma.binaries import BINARIES, ENGINE_VERSION -from prisma.engine import errors, utils + +# from prisma.utils import temp_env_update +# from prisma.binaries import platform +from prisma.binaries import BINARIES # , ENGINE_VERSION +from prisma.engine import errors # , utils from prisma.engine.query import QueryEngine from prisma._compat import get_running_loop From 550d2fff8091bf9d94257792cf237671fb8b612a Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:27:27 +0100 Subject: [PATCH 40/55] chore(platform.py): mypy ignore stubs from distro module --- src/prisma/binaries/platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 3ca2d586e..8f9e22fba 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -5,7 +5,7 @@ from functools import lru_cache from typing import Optional -import distro # pyright: reportMissingTypeStubs=false +import distro # type: ignore # pyright: reportMissingTypeStubs=false class OsSettings: From 74c3381b193b9f544b4936d49838aa46fbdc0e91 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 22:47:57 +0100 Subject: [PATCH 41/55] chore: fix _compat typechecking --- src/prisma/_compat.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/prisma/_compat.py b/src/prisma/_compat.py index 4dcbb8521..48af73a60 100644 --- a/src/prisma/_compat.py +++ b/src/prisma/_compat.py @@ -47,6 +47,11 @@ def validator( if TYPE_CHECKING: cached_property = property else: - from cached_property import cached_property as cached_property + try: + from cached_property import cached_property as cached_property + except ImportError: + # If we can't import cached_property, fallback to the standard property + cached_property = property else: + from functools import cached_property as cached_property From 9bf32e1d44c5a731d2a901a48d9e8ee91ab65d4d Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 22:49:38 +0100 Subject: [PATCH 42/55] chore: fix typo in "centos" --- src/prisma/binaries/platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index 8f9e22fba..c133c5c54 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -62,7 +62,7 @@ def resolve_known_distro(distro_id: str, distro_like: str) -> Optional[str]: distro_id == "fedora" or "fedora" in distro_like or "rhel" in distro_like - or "centros" in distro_like + or "centos" in distro_like ): return "rhel" if ( From 9d74f4119f30e1b30be3b1b3ba21c5b16bb3bf01 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 22:58:10 +0100 Subject: [PATCH 43/55] feat: major refactor of binaries.py --- src/prisma/binaries/binaries.py | 178 ++++++++++++------ src/prisma/cli/prisma.py | 6 +- .../generator/templates/engine/query.py.jinja | 4 +- 3 files changed, 129 insertions(+), 59 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 29b381296..ac5223ea9 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -2,37 +2,22 @@ import logging import os +import subprocess import tempfile from pathlib import Path -from typing import Callable, List, Optional +import time +from typing import Callable, List, Optional, Tuple import click from pydantic import BaseSettings, Field +from prisma.errors import PrismaError + +from prisma.utils import time_since + from . import platform from .download import download -# PLATFORMS: List[str] = [ -# 'darwin', -# 'darwin-arm64', -# 'debian-openssl-1.0.x', -# 'debian-openssl-1.1.x', -# 'rhel-openssl-1.0.x', -# 'rhel-openssl-1.1.x', -# 'linux-arm64-openssl-1.1.x', -# 'linux-arm64-openssl-1.0.x', -# 'linux-arm-openssl-1.1.x', -# 'linux-arm-openssl-1.0.x', -# 'linux-musl', -# 'linux-nixos', -# 'windows', -# 'freebsd11', -# 'freebsd12', -# 'openbsd', -# 'netbsd', -# 'arm', -# ] - # Get system information OS_SETTINGS: platform.OsSettings = platform.get_os_settings() PLATFORM: str = platform.resolve_platform(OS_SETTINGS) @@ -52,57 +37,77 @@ ) PLATFORM_EXE_EXTENSION: str = ".exe" if OS_SETTINGS.is_windows() else "" +PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" + + +def default_prisma_cli_path() -> Path: + return GLOBAL_TEMP_DIR / PRISMA_CLI_NAME + -def default_in_temp(name: str) -> Callable[[], Path]: +def default_engine_path(name: str) -> Callable[[], Path]: return lambda: (GLOBAL_TEMP_DIR / name).with_suffix(PLATFORM_EXE_EXTENSION) class PrismaSettings(BaseSettings): PRISMA_CLI_MIRROR: str = "https://prisma-photongo.s3-eu-west-1.amazonaws.com" PRISMA_ENGINES_MIRROR: str = "https://binaries.prisma.sh" + PRISMA_QUERY_ENGINE_BINARY: Path = Field( - default_factory=default_in_temp("query-engine") + default_factory=default_engine_path("query-engine") ) PRISMA_MIGRATION_ENGINE_BINARY: Path = Field( - default_factory=default_in_temp("migration-engine") + default_factory=default_engine_path("migration-engine") ) PRISMA_INTROSPECTION_ENGINE_BINARY: Path = Field( - default_factory=default_in_temp("introspection-engine") + default_factory=default_engine_path("introspection-engine") ) - PRISMA_FMT_BINARY: Path = Field(default_factory=default_in_temp("prisma-fmt")) + PRISMA_CLI_BINARY: Path = Field(default_factory=default_prisma_cli_path) + PRISMA_FMT_BINARY: Path = Field(default_factory=default_engine_path("prisma-fmt")) PRISMA_CLI_BINARY_TARGETS: List[str] = Field(default_factory=list) + def engine_url(self, name: str) -> str: + return ( + f"{self.PRISMA_ENGINES_MIRROR}/" + "all_commits/" + f"{ENGINE_VERSION}/" + f"{PLATFORM}/" + f"{name}{PLATFORM_EXE_EXTENSION}.gz" + ) -settings: PrismaSettings = PrismaSettings() + def prisma_cli_url(self) -> str: + return f"{self.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" -PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" -PRISMA_CLI_PATH = GLOBAL_TEMP_DIR / PRISMA_CLI_NAME -PRISMA_CLI_URL = f"{settings.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" +SETTINGS: PrismaSettings = PrismaSettings() -def engine_url_for(name: str) -> str: - return ( - f"{settings.PRISMA_ENGINES_MIRROR}/" - "all_commits/" - f"{ENGINE_VERSION}/" - f"{PLATFORM}/" - f"{name}{PLATFORM_EXE_EXTENSION}.gz" - ) +log: logging.Logger = logging.getLogger(__name__) -log: logging.Logger = logging.getLogger(__name__) +class InvalidBinaryVersion(PrismaError): + pass class Binary: name: str url: str path: Path - - def __init__(self, name: str, path: Path, *, url: Optional[str] = None): + settings: PrismaSettings + + def __init__( + self, + name: str, + path: Path, + *, + url: Optional[str] = None, + settings: Optional[PrismaSettings] = None, + ): + if settings is None: + settings = SETTINGS + self.settings = settings self.name = name self.path = path - self.url = engine_url_for(name) if url is None else url + self.url = settings.engine_url(name) if url is None else url def download(self) -> None: download(self.url, self.path) @@ -114,22 +119,87 @@ def remove(self) -> None: except FileNotFoundError: pass - -ENGINES: List[Binary] = [ - Binary(name='query-engine', path=settings.PRISMA_QUERY_ENGINE_BINARY), - Binary(name='migration-engine', path=settings.PRISMA_MIGRATION_ENGINE_BINARY), - Binary( - name='introspection-engine', path=settings.PRISMA_INTROSPECTION_ENGINE_BINARY - ), - Binary(name='prisma-fmt', path=settings.PRISMA_FMT_BINARY), + def ensure_binary(self) -> None: + """ + If binary doesn't exist or is not a file, raise an error + + Args: + binary (Binary): a binary to check + """ + if not self.path.exists(): + raise FileNotFoundError( + f"{self.name} binary not found at {self.path}\nTry running `prisma fetch`" + ) + if not self.path.is_file(): + raise IsADirectoryError(f"{self.name} binary is a directory") + # Run binary with --version to check if it's the right version and capture stdout + start_version = time.monotonic() + process = subprocess.run( + [str(self.path), '--version'], stdout=subprocess.PIPE, check=True + ) + log.debug('Version check took %s', time_since(start_version)) + + # Version format is: " " + # e.g. migration-engine-cli 34df67547cf5598f5a6cd3eb45f14ee70c3fb86f + # or query-engine 34df67547cf5598f5a6cd3eb45f14ee70c3fb86f + version = process.stdout.decode().split(' ')[1] + if version != ENGINE_VERSION: + raise InvalidBinaryVersion( + f"{self.name} binary version {version} is not {ENGINE_VERSION}" + ) + + +EnginesType = Tuple[ + Binary, # query-engine + Binary, # migration-engine + Binary, # introspection-engine + Binary, # prisma-fmt ] -BINARIES: List[Binary] = [ - *ENGINES, - Binary(name="prisma-cli", path=PRISMA_CLI_PATH, url=PRISMA_CLI_URL), +# Maybe we should use a dict instead of a tuple? or a namedtuple? +BinariesType = Tuple[ + Binary, # query-engine + Binary, # migration-engine + Binary, # introspection-engine + Binary, # prisma-fmt + Binary, # prisma-cli ] +def engines_from_settings( + settings: PrismaSettings, +) -> EnginesType: + return ( + Binary(name='query-engine', path=settings.PRISMA_QUERY_ENGINE_BINARY), + Binary(name='migration-engine', path=settings.PRISMA_MIGRATION_ENGINE_BINARY), + Binary( + name='introspection-engine', + path=settings.PRISMA_INTROSPECTION_ENGINE_BINARY, + ), + Binary(name='prisma-fmt', path=settings.PRISMA_FMT_BINARY), + ) + + +def binaries_from_settings( + settings: PrismaSettings, *, engines: Optional[EnginesType] = None +) -> BinariesType: + if engines is None: + engines = engines_from_settings(settings) + return ( + *engines, + Binary( + name='prisma-cli', + path=settings.PRISMA_CLI_BINARY, + url=settings.prisma_cli_url(), + ), + ) + + +ENGINES: EnginesType = engines_from_settings(SETTINGS) + +BINARIES: BinariesType = binaries_from_settings(SETTINGS, engines=ENGINES) + + def ensure_cached() -> Path: to_download: List[Binary] = [] for binary in BINARIES: diff --git a/src/prisma/cli/prisma.py b/src/prisma/cli/prisma.py index 3f2cee8e5..48acb56de 100644 --- a/src/prisma/cli/prisma.py +++ b/src/prisma/cli/prisma.py @@ -17,8 +17,8 @@ def run( check: bool = False, env: Optional[Dict[str, str]] = None, ) -> int: - directory = binaries.ensure_cached() - path = directory.joinpath(binaries.PRISMA_CLI_NAME) + binaries.ensure_cached() + path = binaries.SETTINGS.PRISMA_CLI_BINARY if not path.exists(): raise RuntimeError( f'The Prisma CLI is not downloaded, expected {path} to exist.' @@ -35,7 +35,7 @@ def run( env = {**default_env, **env} if env is not None else default_env # ensure the client uses our engine binaries # TODO: this is a hack, probably there's a better way to do this - engine_env_dict = binaries.settings.dict() + engine_env_dict = binaries.SETTINGS.dict() for engine in binaries.ENGINES: for engine_env, engine_path in engine_env_dict.items(): if engine_path == engine.path: diff --git a/src/prisma/generator/templates/engine/query.py.jinja b/src/prisma/generator/templates/engine/query.py.jinja index a168031ac..8a216d750 100644 --- a/src/prisma/generator/templates/engine/query.py.jinja +++ b/src/prisma/generator/templates/engine/query.py.jinja @@ -14,7 +14,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached +from ..binaries import SETTINGS, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -78,7 +78,7 @@ class QueryEngine(HTTPEngine): start = time.monotonic() ensure_cached() - self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY + self.file = SETTINGS.PRISMA_QUERY_ENGINE_BINARY try: {{ maybe_await }}self.spawn(self.file, timeout=timeout, datasources=datasources) From 42aac70f38562b7283cdb4509ecea5a3c8a2e48e Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 22:58:47 +0100 Subject: [PATCH 44/55] chore: update snapshots --- .../test_exhaustive/test_async[enginequery.py].raw | 4 ++-- .../test_exhaustive/test_sync[enginequery.py].raw | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw index bbcc4325b..a3978d2fd 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_async[enginequery.py].raw @@ -42,7 +42,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached +from ..binaries import SETTINGS, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -103,7 +103,7 @@ class QueryEngine(HTTPEngine): start = time.monotonic() ensure_cached() - self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY + self.file = SETTINGS.PRISMA_QUERY_ENGINE_BINARY try: await self.spawn(self.file, timeout=timeout, datasources=datasources) diff --git a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw index 8087e173b..6aeba86bc 100644 --- a/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw +++ b/tests/test_generation/exhaustive/__snapshots__/test_exhaustive/test_sync[enginequery.py].raw @@ -42,7 +42,7 @@ from pathlib import Path from . import utils, errors from .http import HTTPEngine from ..utils import DEBUG -from ..binaries import settings as binaries_settings, OS_SETTINGS, ensure_cached +from ..binaries import SETTINGS, OS_SETTINGS, ensure_cached from ..utils import time_since, _env_bool from ..types import DatasourceOverride from ..builder import dumps @@ -104,7 +104,7 @@ class QueryEngine(HTTPEngine): start = time.monotonic() ensure_cached() - self.file = binaries_settings.PRISMA_QUERY_ENGINE_BINARY + self.file = SETTINGS.PRISMA_QUERY_ENGINE_BINARY try: self.spawn(self.file, timeout=timeout, datasources=datasources) From 72a93e51b7c275ce4e22948e70390a7855fec98f Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 22:59:32 +0100 Subject: [PATCH 45/55] feat: add and rework important tests --- tests/test_binaries.py | 118 ++++++++++++++++++++++++++++++++++++++++ tests/test_engine.py | 121 ++++++++++++++++++++++------------------- 2 files changed, 182 insertions(+), 57 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index e69de29bb..8974b9dae 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -0,0 +1,118 @@ +import platform +from typing import Any, List, Optional, Tuple +import pytest + +import distro # type: ignore # pyright: reportMissingTypeStubs=false +import prisma.binaries.platform as binaries_platform + + +def mock(obj: Any): + return lambda: obj + + +def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(platform, "system", mock("Windows")) + monkeypatch.setattr(platform, "machine", mock("x86_64")) + monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) + monkeypatch.setattr(distro, "id", mock("")) + monkeypatch.setattr(distro, "like", mock("")) + + assert platform.system() == "Windows" + os_settings = binaries_platform.get_os_settings() + assert os_settings.is_windows() + + assert binaries_platform.resolve_darwin(os_settings) is None + assert binaries_platform.resolve_linux(os_settings) is None + assert binaries_platform.resolve_other_platforms(os_settings) == "windows" + + +MACHINES = ["aarch64", "x86_64"] + + +@pytest.mark.parametrize(("machine,"), MACHINES) +def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): + monkeypatch.setattr(platform, "system", mock("Darwin")) + monkeypatch.setattr(platform, "machine", mock(machine)) + monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) + monkeypatch.setattr(distro, "id", mock("")) + monkeypatch.setattr(distro, "like", mock("")) + + assert platform.system() == "Darwin" + + os_settings = binaries_platform.get_os_settings() + + assert not os_settings.is_windows() + if machine == "aarch64": + assert binaries_platform.resolve_darwin(os_settings) == "darwin-arm64" + else: + assert binaries_platform.resolve_darwin(os_settings) == "darwin" + assert binaries_platform.resolve_linux(os_settings) is None + assert binaries_platform.resolve_other_platforms(os_settings) is None + + +# DISTRO_ID, DISTRO_LIKE, EXPECTED +DISTRO_ID_LIKE: List[Tuple[str, str, str]] = [ + ("alpine", "", "musl"), + ("raspbian", "", "arm"), + ("nixos", "", "nixos"), + ("ubuntu", "debian", "debian"), + ("debian", "debian", "debian"), + ("fedora", "fedora", "rhel"), + ("rhel", "rhel", "rhel"), + ("centos", "centos", "rhel"), + ("oracle", "fedora", "rhel"), +] + + +@pytest.mark.parametrize(("distro_id,distro_like,expected"), DISTRO_ID_LIKE) +def test_resolve_known_distro( + monkeypatch: pytest.MonkeyPatch, distro_id: str, distro_like: str, expected: str +): + monkeypatch.setattr(platform, "system", mock("linux")) + monkeypatch.setattr(platform, "machine", mock("x86_64")) + monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) + monkeypatch.setattr(distro, "id", mock(distro_id)) + monkeypatch.setattr(distro, "like", mock(distro_like)) + + os_settings = binaries_platform.get_os_settings() + assert os_settings.distro == expected + + +# TODO: add openssl to parameters +# TODO: add more arm datapoints +# MACHINE, DISTRO_ID, DISTRO_LIKE, RESOLVED_DISTRO, EPXECTED +LINUX_PLATFORMS: List[Tuple[str, str, str, Optional[str], str]] = [ + ("x86_64", "alpine", "", "musl", "linux-musl"), + ("aarch64", "ubuntu", "debian", "debian", "linux-arm64-openssl-1.1.x"), + ("arm", "raspbian", "", "arm", "linux-arm-openssl-1.1.x"), + ("x86_64", "nixos", "", "nixos", "linux-nixos"), + ("x86_64", "fedora", "fedora", "rhel", "rhel-openssl-1.1.x"), + ("x86_64", "debian", "debian", "debian", "debian-openssl-1.1.x"), + ("x86_64", "niche-distro", "niche-like", None, "debian-openssl-1.1.x"), + ("arm", "niche-distro", "niche-like", None, "linux-arm-openssl-1.1.x"), + ("aarch64", "niche-distro", "niche-like", None, "linux-arm64-openssl-1.1.x"), +] + + +@pytest.mark.parametrize( + "machine,distro_id,distro_like,resolved_distro,expected_platform", + LINUX_PLATFORMS, +) +def test_resolve_linux( + monkeypatch: pytest.MonkeyPatch, + distro_id: str, + distro_like: str, + machine: str, + resolved_distro: str, + expected_platform: str, +): + # pylint: disable=too-many-arguments + # TODO: arguments can be simplified to a namedtuple or something + monkeypatch.setattr(platform, "system", mock("Linux")) + monkeypatch.setattr(platform, "machine", mock(machine)) + monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) + monkeypatch.setattr(distro, "id", mock(distro_id)) + monkeypatch.setattr(distro, "like", mock(distro_like)) + os_settings = binaries_platform.get_os_settings() + assert os_settings.distro == resolved_distro + assert binaries_platform.resolve_platform(os_settings) == expected_platform diff --git a/tests/test_engine.py b/tests/test_engine.py index 4f5d52cc6..f33af8a69 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -8,29 +8,28 @@ from pytest_subprocess import FakeProcess from prisma import Client - -# from prisma.utils import temp_env_update -# from prisma.binaries import platform -from prisma.binaries import BINARIES # , ENGINE_VERSION -from prisma.engine import errors # , utils +from prisma.binaries import BINARIES, PLATFORM_EXE_EXTENSION +from prisma.binaries.binaries import ( + ENGINE_VERSION, + ENGINES, + InvalidBinaryVersion, + PrismaSettings, + engines_from_settings, +) +from prisma.engine import errors from prisma.engine.query import QueryEngine from prisma._compat import get_running_loop +from prisma.utils import temp_env_update from .utils import Testdir -QUERY_ENGINE = next( # pragma: no branch - b for b in BINARIES if b.name == 'query-engine' -) - - @contextlib.contextmanager def no_event_loop() -> Iterator[None]: try: current: Optional[asyncio.AbstractEventLoop] = get_running_loop() except RuntimeError: current = None - try: asyncio.set_event_loop(None) yield @@ -50,7 +49,7 @@ async def test_engine_connects() -> None: await db.disconnect() -def test_stopping_engine_on_closed_loop() -> None: +def test_stopping_engine_on_closed_loop(): """Stopping the engine with no event loop available does not raise an error""" with no_event_loop(): engine = QueryEngine(dml='') @@ -65,78 +64,86 @@ def mock_exists(path: Path) -> bool: monkeypatch.setattr(Path, 'exists', mock_exists, raising=True) - # TODO: What to do with this? - # with pytest.raises(errors.BinaryNotFoundError) as exc: - # utils.ensure() + with pytest.raises(FileNotFoundError) as exc: + BINARIES[0].ensure_binary() - # assert exc.match( - # r'Expected .* or .* but neither were found\.\nTry running prisma py fetch' - # ) + assert exc.match(r'.* binary not found at .*\nTry running `prisma fetch`') def test_mismatched_version_error(fake_process: FakeProcess) -> None: """Mismatched query engine versions raises an error""" + query_engine = ENGINES[0] fake_process.register_subprocess( - [QUERY_ENGINE.path, '--version'], # type: ignore[list-item] + [str(query_engine.path), '--version'], stdout='query-engine unexpected-hash', ) - # TODO: What to do with this? - # with pytest.raises(errors.MismatchedVersionsError) as exc: - # utils.ensure() + with pytest.raises(InvalidBinaryVersion) as exc: + query_engine.ensure_binary() + # "{self.name} binary version {version} is not {ENGINE_VERSION}" + assert exc.match( + f'{query_engine.name} binary version unexpected-hash is not {ENGINE_VERSION}' + ) - # assert exc.match( - # f'Expected query engine version `{ENGINE_VERSION}` but got `unexpected-hash`' - # ) +# TODO: reimplement this test +@pytest.mark.skip( + reason='Local binary paths are no longer supported, this test still exists as support will be added again in the future' +) +def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: + """Query engine in current directory required to be the expected version""" -# def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: -# """Query engine in current directory required to be the expected version""" -# fake_engine = testdir.path / platform.check_for_extension( -# f'prisma-query-engine-{platform.binary_platform()}' -# ) -# fake_engine.touch() + fake_engine = testdir.path / ("prisma-query-engine" + PLATFORM_EXE_EXTENSION) + fake_engine.touch() -# fake_process.register_subprocess( -# [fake_engine, '--version'], # type: ignore[list-item] -# stdout='query-engine a-different-hash', -# ) -# with pytest.raises(errors.MismatchedVersionsError): -# path = utils.ensure() + fake_process.register_subprocess( + [str(fake_engine), '--version'], + stdout='query-engine a-different-hash', + ) + # with pytest.raises(errors.MismatchedVersionsError): + # TODO: handle this case + # pass -# fake_process.register_subprocess( -# [fake_engine, '--version'], # type: ignore[list-item] -# stdout=f'query-engine {ENGINE_VERSION}', -# ) -# path = utils.ensure() -# assert path == fake_engine + fake_process.register_subprocess( + [fake_engine, '--version'], # type: ignore[list-item] + stdout=f'query-engine {ENGINE_VERSION}', + ) + # path = utils.ensure() + # assert path == fake_engine def test_ensure_env_override(testdir: Testdir, fake_process: FakeProcess) -> None: """Query engine path in environment variable can be any version""" fake_engine = testdir.path / 'my-query-engine' fake_engine.touch() - fake_process.register_subprocess( - [fake_engine, '--version'], # type: ignore[list-item] + [str(fake_engine), '--version'], stdout='query-engine a-different-hash', ) - # TODO: What to do with this? - # with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': str(fake_engine)}): - # path = utils.ensure() - - # assert path == fake_engine + with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': str(fake_engine)}): + prisma_settings = PrismaSettings() + engines = engines_from_settings(prisma_settings) + query_engine = engines[0] # query engine is the first engine + with pytest.raises(InvalidBinaryVersion) as exc: + query_engine.ensure_binary() + assert exc.match( + f'{query_engine.name} binary version a-different-hash is not {ENGINE_VERSION}' + ) + # query_engine.ensure_binary() def test_ensure_env_override_does_not_exist() -> None: """Query engine path in environment variable not found raises an error""" - # TODO: What to do with this? - # with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': 'foo'}): - # with pytest.raises(errors.BinaryNotFoundError) as exc: - # utils.ensure() + with temp_env_update({'PRISMA_QUERY_ENGINE_BINARY': 'foo'}): + prisma_settings = PrismaSettings() + engines = engines_from_settings(prisma_settings) + query_engine = engines[0] # query engine is the first engine + with pytest.raises(FileNotFoundError) as exc: + query_engine.ensure_binary() - # assert exc.match( - # r'PRISMA_QUERY_ENGINE_BINARY was provided, but no query engine was found at foo' - # ) + # f"{self.name} binary not found at {self.path}\nTry running `prisma fetch`" + assert exc.match( + f'{query_engine.name} binary not found at {query_engine.path}\nTry running `prisma fetch`' + ) From 61c2c12e0f7f7794f291126042029e33dfa79ef2 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Thu, 27 Jan 2022 23:44:49 +0100 Subject: [PATCH 46/55] chore: add pragmas to tests --- tests/test_binaries.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 8974b9dae..e74a61f3b 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -10,7 +10,7 @@ def mock(obj: Any): return lambda: obj -def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): +def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): # pragma: no cover monkeypatch.setattr(platform, "system", mock("Windows")) monkeypatch.setattr(platform, "machine", mock("x86_64")) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -30,7 +30,9 @@ def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): @pytest.mark.parametrize(("machine,"), MACHINES) -def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): +def test_mock_darwin_os( + monkeypatch: pytest.MonkeyPatch, machine: str +): # pragma: no cover monkeypatch.setattr(platform, "system", mock("Darwin")) monkeypatch.setattr(platform, "machine", mock(machine)) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -67,7 +69,7 @@ def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): @pytest.mark.parametrize(("distro_id,distro_like,expected"), DISTRO_ID_LIKE) def test_resolve_known_distro( monkeypatch: pytest.MonkeyPatch, distro_id: str, distro_like: str, expected: str -): +): # pragma: no cover monkeypatch.setattr(platform, "system", mock("linux")) monkeypatch.setattr(platform, "machine", mock("x86_64")) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -105,7 +107,7 @@ def test_resolve_linux( machine: str, resolved_distro: str, expected_platform: str, -): +): # pragma: no cover # pylint: disable=too-many-arguments # TODO: arguments can be simplified to a namedtuple or something monkeypatch.setattr(platform, "system", mock("Linux")) From 48ccd5ad436c50d12fddc106b1a70a1eb19e8914 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 28 Jan 2022 00:06:16 +0100 Subject: [PATCH 47/55] chore: actually fix coverage errors --- tests/test_binaries.py | 46 +++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index e74a61f3b..73403efb8 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -10,7 +10,16 @@ def mock(obj: Any): return lambda: obj -def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): # pragma: no cover +def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): + """ + Mock the platform.system() to return "Windows" and platform.machine() + to return default "x86_64". + + Tests resolve_other_platforms() and asserts that it returns the expected value. + + Additionaly tests resolve_linux() and resolve_darwin() + and makes sure they return None + """ monkeypatch.setattr(platform, "system", mock("Windows")) monkeypatch.setattr(platform, "machine", mock("x86_64")) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -30,9 +39,17 @@ def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): # pragma: no cover @pytest.mark.parametrize(("machine,"), MACHINES) -def test_mock_darwin_os( - monkeypatch: pytest.MonkeyPatch, machine: str -): # pragma: no cover +def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): + """ + Mock the platform.system() to return "Darwin" and platform.machine() to return + either "aarch64" or "x86_64". + + Tests resolve_darwin() and makes sure it returns "darwin-arm64" for aarch64 + and "darwin" for x86_64. + + Additionaly tests resolve_linux() and resolve_other_platforms() + and makes sure they return None + """ monkeypatch.setattr(platform, "system", mock("Darwin")) monkeypatch.setattr(platform, "machine", mock(machine)) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -69,7 +86,14 @@ def test_mock_darwin_os( @pytest.mark.parametrize(("distro_id,distro_like,expected"), DISTRO_ID_LIKE) def test_resolve_known_distro( monkeypatch: pytest.MonkeyPatch, distro_id: str, distro_like: str, expected: str -): # pragma: no cover +): + """ + Mock the platform.system() to return "linux" and platform.machine() + to return default "x86_64". + + Tests resolve_known_distro() and asserts that it + returns expected distro names. + """ monkeypatch.setattr(platform, "system", mock("linux")) monkeypatch.setattr(platform, "machine", mock("x86_64")) monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) @@ -107,8 +131,16 @@ def test_resolve_linux( machine: str, resolved_distro: str, expected_platform: str, -): # pragma: no cover - # pylint: disable=too-many-arguments +): # pylint: disable=too-many-arguments + + """ + Mocks the platform.system() to return "linux" and platform.machine() + to return either "aarch64" or "x86_64". + + Tests resolve_platform() and asserts that it returns expected binary platform. + + Additionaly tests resolve_known_distro() + """ # TODO: arguments can be simplified to a namedtuple or something monkeypatch.setattr(platform, "system", mock("Linux")) monkeypatch.setattr(platform, "machine", mock(machine)) From 083252b64d66adb92025b96ee2586f04978d1bd7 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Fri, 28 Jan 2022 00:19:43 +0100 Subject: [PATCH 48/55] chore: painstakingly change double quotes to single quotes --- src/prisma/binaries/binaries.py | 44 ++++++------ src/prisma/binaries/platform.py | 68 +++++++++---------- tests/test_binaries.py | 116 ++++++++++++++++---------------- tests/test_engine.py | 4 +- 4 files changed, 116 insertions(+), 116 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index ac5223ea9..77d07f779 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -35,9 +35,9 @@ GLOBAL_TEMP_DIR: Path = ( Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) -PLATFORM_EXE_EXTENSION: str = ".exe" if OS_SETTINGS.is_windows() else "" +PLATFORM_EXE_EXTENSION: str = '.exe' if OS_SETTINGS.is_windows() else '' -PRISMA_CLI_NAME = f"prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}" +PRISMA_CLI_NAME = f'prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}' def default_prisma_cli_path() -> Path: @@ -49,33 +49,33 @@ def default_engine_path(name: str) -> Callable[[], Path]: class PrismaSettings(BaseSettings): - PRISMA_CLI_MIRROR: str = "https://prisma-photongo.s3-eu-west-1.amazonaws.com" - PRISMA_ENGINES_MIRROR: str = "https://binaries.prisma.sh" + PRISMA_CLI_MIRROR: str = 'https://prisma-photongo.s3-eu-west-1.amazonaws.com' + PRISMA_ENGINES_MIRROR: str = 'https://binaries.prisma.sh' PRISMA_QUERY_ENGINE_BINARY: Path = Field( - default_factory=default_engine_path("query-engine") + default_factory=default_engine_path('query-engine') ) PRISMA_MIGRATION_ENGINE_BINARY: Path = Field( - default_factory=default_engine_path("migration-engine") + default_factory=default_engine_path('migration-engine') ) PRISMA_INTROSPECTION_ENGINE_BINARY: Path = Field( - default_factory=default_engine_path("introspection-engine") + default_factory=default_engine_path('introspection-engine') ) PRISMA_CLI_BINARY: Path = Field(default_factory=default_prisma_cli_path) - PRISMA_FMT_BINARY: Path = Field(default_factory=default_engine_path("prisma-fmt")) + PRISMA_FMT_BINARY: Path = Field(default_factory=default_engine_path('prisma-fmt')) PRISMA_CLI_BINARY_TARGETS: List[str] = Field(default_factory=list) def engine_url(self, name: str) -> str: return ( - f"{self.PRISMA_ENGINES_MIRROR}/" - "all_commits/" - f"{ENGINE_VERSION}/" - f"{PLATFORM}/" - f"{name}{PLATFORM_EXE_EXTENSION}.gz" + f'{self.PRISMA_ENGINES_MIRROR}/' + 'all_commits/' + f'{ENGINE_VERSION}/' + f'{PLATFORM}/' + f'{name}{PLATFORM_EXE_EXTENSION}.gz' ) def prisma_cli_url(self) -> str: - return f"{self.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz" + return f'{self.PRISMA_CLI_MIRROR}/{PRISMA_CLI_NAME}.gz' SETTINGS: PrismaSettings = PrismaSettings() @@ -128,10 +128,10 @@ def ensure_binary(self) -> None: """ if not self.path.exists(): raise FileNotFoundError( - f"{self.name} binary not found at {self.path}\nTry running `prisma fetch`" + f'{self.name} binary not found at {self.path}\nTry running `prisma fetch`' ) if not self.path.is_file(): - raise IsADirectoryError(f"{self.name} binary is a directory") + raise IsADirectoryError(f'{self.name} binary is a directory') # Run binary with --version to check if it's the right version and capture stdout start_version = time.monotonic() process = subprocess.run( @@ -145,7 +145,7 @@ def ensure_binary(self) -> None: version = process.stdout.decode().split(' ')[1] if version != ENGINE_VERSION: raise InvalidBinaryVersion( - f"{self.name} binary version {version} is not {ENGINE_VERSION}" + f'{self.name} binary version {version} is not {ENGINE_VERSION}' ) @@ -204,17 +204,17 @@ def ensure_cached() -> Path: to_download: List[Binary] = [] for binary in BINARIES: if binary.path.exists(): - log.debug("%s is cached, skipping download", binary.name) + log.debug('%s is cached, skipping download', binary.name) continue - log.debug("%s is not cached, will download", binary.name) + log.debug('%s is not cached, will download', binary.name) to_download.append(binary) if len(to_download) == 0: - log.debug("All binaries are cached, skipping download") + log.debug('All binaries are cached, skipping download') return GLOBAL_TEMP_DIR def show_item(item: Optional[Binary]) -> str: - return "" if item is None else item.name + return '' if item is None else item.name with click.progressbar( to_download, @@ -223,7 +223,7 @@ def show_item(item: Optional[Binary]) -> str: item_show_func=show_item, ) as iterator: for binary in iterator: - log.debug("Downloading %s from %s", binary.name, binary.url) + log.debug('Downloading %s from %s', binary.name, binary.url) binary.download() return GLOBAL_TEMP_DIR diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index c133c5c54..a2a9b79e9 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -32,12 +32,12 @@ def get_openssl() -> str: ['openssl', 'version', '-v'], stdout=subprocess.PIPE, check=True ) version = parse_openssl_version(str(process.stdout, sys.getdefaultencoding())) - if version not in ("1.0.x", "1.1.x"): + if version not in ('1.0.x', '1.1.x'): # If not 1.0 or 1.1 then it's most likely 3.0 # Currently prisma doesn't provide binaries for 3.0 # But we can use the latest stable 1.1.x # See: https://github.com/prisma/prisma/issues/11356 - return "1.1.x" + return '1.1.x' return version @@ -52,26 +52,26 @@ def parse_openssl_version(string: str) -> str: def resolve_known_distro(distro_id: str, distro_like: str) -> Optional[str]: - if distro_id == "alpine": - return "musl" - if distro_id == "raspbian": - return "arm" - if distro_id == "nixos": - return "nixos" + if distro_id == 'alpine': + return 'musl' + if distro_id == 'raspbian': + return 'arm' + if distro_id == 'nixos': + return 'nixos' if ( - distro_id == "fedora" - or "fedora" in distro_like - or "rhel" in distro_like - or "centos" in distro_like + distro_id == 'fedora' + or 'fedora' in distro_like + or 'rhel' in distro_like + or 'centos' in distro_like ): - return "rhel" + return 'rhel' if ( - distro_id == "ubuntu" - or distro_id == "debian" - or "ubuntu" in distro_like - or "debian" in distro_like + distro_id == 'ubuntu' + or distro_id == 'debian' + or 'ubuntu' in distro_like + or 'debian' in distro_like ): - return "debian" + return 'debian' return None @@ -88,30 +88,30 @@ def get_os_settings() -> OsSettings: def resolve_darwin(os_settings: OsSettings) -> Optional[str]: - if os_settings.system == "darwin" and os_settings.machine == "aarch64": - return "darwin-arm64" - if os_settings.system == "darwin": - return "darwin" + if os_settings.system == 'darwin' and os_settings.machine == 'aarch64': + return 'darwin-arm64' + if os_settings.system == 'darwin': + return 'darwin' return None def resolve_linux(os_settings: OsSettings) -> Optional[str]: - if os_settings.system == "linux": - if os_settings.machine == "aarch64": - return f"linux-arm64-openssl-{os_settings.libssl}" - if os_settings.machine == "arm": - return f"linux-arm-openssl-{os_settings.libssl}" - if os_settings.distro == "musl": - return "linux-musl" - if os_settings.distro == "nixos": - return "linux-nixos" + if os_settings.system == 'linux': + if os_settings.machine == 'aarch64': + return f'linux-arm64-openssl-{os_settings.libssl}' + if os_settings.machine == 'arm': + return f'linux-arm-openssl-{os_settings.libssl}' + if os_settings.distro == 'musl': + return 'linux-musl' + if os_settings.distro == 'nixos': + return 'linux-nixos' if os_settings.distro: - return f"{os_settings.distro}-openssl-{os_settings.libssl}" + return f'{os_settings.distro}-openssl-{os_settings.libssl}' return None def resolve_other_platforms(os_settings: OsSettings) -> Optional[str]: - if os_settings.system in ("windows", "freebsd", "openbsd", "netbsd"): + if os_settings.system in ('windows', 'freebsd', 'openbsd', 'netbsd'): return os_settings.system return None @@ -122,4 +122,4 @@ def resolve_platform(os_settings: OsSettings) -> str: platform = resolver(os_settings) if platform is not None: return platform - return "debian-openssl-1.1.x" # default fallback + return 'debian-openssl-1.1.x' # default fallback diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 73403efb8..457b8f1fd 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -12,93 +12,93 @@ def mock(obj: Any): def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): """ - Mock the platform.system() to return "Windows" and platform.machine() - to return default "x86_64". + Mock the platform.system() to return 'Windows' and platform.machine() + to return default 'x86_64'. Tests resolve_other_platforms() and asserts that it returns the expected value. Additionaly tests resolve_linux() and resolve_darwin() and makes sure they return None """ - monkeypatch.setattr(platform, "system", mock("Windows")) - monkeypatch.setattr(platform, "machine", mock("x86_64")) - monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) - monkeypatch.setattr(distro, "id", mock("")) - monkeypatch.setattr(distro, "like", mock("")) + monkeypatch.setattr(platform, 'system', mock('Windows')) + monkeypatch.setattr(platform, 'machine', mock('x86_64')) + monkeypatch.setattr(binaries_platform, 'get_openssl', mock('1.1.x')) + monkeypatch.setattr(distro, 'id', mock('')) + monkeypatch.setattr(distro, 'like', mock('')) - assert platform.system() == "Windows" + assert platform.system() == 'Windows' os_settings = binaries_platform.get_os_settings() assert os_settings.is_windows() assert binaries_platform.resolve_darwin(os_settings) is None assert binaries_platform.resolve_linux(os_settings) is None - assert binaries_platform.resolve_other_platforms(os_settings) == "windows" + assert binaries_platform.resolve_other_platforms(os_settings) == 'windows' -MACHINES = ["aarch64", "x86_64"] +MACHINES = ['aarch64', 'x86_64'] -@pytest.mark.parametrize(("machine,"), MACHINES) +@pytest.mark.parametrize(('machine,'), MACHINES) def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): """ - Mock the platform.system() to return "Darwin" and platform.machine() to return - either "aarch64" or "x86_64". + Mock the platform.system() to return 'Darwin' and platform.machine() to return + either 'aarch64' or 'x86_64'. - Tests resolve_darwin() and makes sure it returns "darwin-arm64" for aarch64 - and "darwin" for x86_64. + Tests resolve_darwin() and makes sure it returns 'darwin-arm64' for aarch64 + and 'darwin' for x86_64. Additionaly tests resolve_linux() and resolve_other_platforms() and makes sure they return None """ - monkeypatch.setattr(platform, "system", mock("Darwin")) - monkeypatch.setattr(platform, "machine", mock(machine)) - monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) - monkeypatch.setattr(distro, "id", mock("")) - monkeypatch.setattr(distro, "like", mock("")) + monkeypatch.setattr(platform, 'system', mock('Darwin')) + monkeypatch.setattr(platform, 'machine', mock(machine)) + monkeypatch.setattr(binaries_platform, 'get_openssl', mock('1.1.x')) + monkeypatch.setattr(distro, 'id', mock('')) + monkeypatch.setattr(distro, 'like', mock('')) - assert platform.system() == "Darwin" + assert platform.system() == 'Darwin' os_settings = binaries_platform.get_os_settings() assert not os_settings.is_windows() - if machine == "aarch64": - assert binaries_platform.resolve_darwin(os_settings) == "darwin-arm64" + if machine == 'aarch64': + assert binaries_platform.resolve_darwin(os_settings) == 'darwin-arm64' else: - assert binaries_platform.resolve_darwin(os_settings) == "darwin" + assert binaries_platform.resolve_darwin(os_settings) == 'darwin' assert binaries_platform.resolve_linux(os_settings) is None assert binaries_platform.resolve_other_platforms(os_settings) is None # DISTRO_ID, DISTRO_LIKE, EXPECTED DISTRO_ID_LIKE: List[Tuple[str, str, str]] = [ - ("alpine", "", "musl"), - ("raspbian", "", "arm"), - ("nixos", "", "nixos"), - ("ubuntu", "debian", "debian"), - ("debian", "debian", "debian"), - ("fedora", "fedora", "rhel"), - ("rhel", "rhel", "rhel"), - ("centos", "centos", "rhel"), - ("oracle", "fedora", "rhel"), + ('alpine', '', 'musl'), + ('raspbian', "", 'arm'), + ('nixos', "", 'nixos'), + ('ubuntu', "debian", 'debian'), + ('debian', "debian", 'debian'), + ('fedora', "fedora", 'rhel'), + ('rhel', "rhel", 'rhel'), + ('centos', "centos", 'rhel'), + ('oracle', "fedora", 'rhel'), ] -@pytest.mark.parametrize(("distro_id,distro_like,expected"), DISTRO_ID_LIKE) +@pytest.mark.parametrize(('distro_id,distro_like,expected'), DISTRO_ID_LIKE) def test_resolve_known_distro( monkeypatch: pytest.MonkeyPatch, distro_id: str, distro_like: str, expected: str ): """ - Mock the platform.system() to return "linux" and platform.machine() - to return default "x86_64". + Mock the platform.system() to return 'linux' and platform.machine() + to return default 'x86_64'. Tests resolve_known_distro() and asserts that it returns expected distro names. """ - monkeypatch.setattr(platform, "system", mock("linux")) - monkeypatch.setattr(platform, "machine", mock("x86_64")) - monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) - monkeypatch.setattr(distro, "id", mock(distro_id)) - monkeypatch.setattr(distro, "like", mock(distro_like)) + monkeypatch.setattr(platform, 'system', mock('linux')) + monkeypatch.setattr(platform, 'machine', mock('x86_64')) + monkeypatch.setattr(binaries_platform, 'get_openssl', mock('1.1.x')) + monkeypatch.setattr(distro, 'id', mock(distro_id)) + monkeypatch.setattr(distro, 'like', mock(distro_like)) os_settings = binaries_platform.get_os_settings() assert os_settings.distro == expected @@ -108,20 +108,20 @@ def test_resolve_known_distro( # TODO: add more arm datapoints # MACHINE, DISTRO_ID, DISTRO_LIKE, RESOLVED_DISTRO, EPXECTED LINUX_PLATFORMS: List[Tuple[str, str, str, Optional[str], str]] = [ - ("x86_64", "alpine", "", "musl", "linux-musl"), - ("aarch64", "ubuntu", "debian", "debian", "linux-arm64-openssl-1.1.x"), - ("arm", "raspbian", "", "arm", "linux-arm-openssl-1.1.x"), - ("x86_64", "nixos", "", "nixos", "linux-nixos"), - ("x86_64", "fedora", "fedora", "rhel", "rhel-openssl-1.1.x"), - ("x86_64", "debian", "debian", "debian", "debian-openssl-1.1.x"), - ("x86_64", "niche-distro", "niche-like", None, "debian-openssl-1.1.x"), - ("arm", "niche-distro", "niche-like", None, "linux-arm-openssl-1.1.x"), - ("aarch64", "niche-distro", "niche-like", None, "linux-arm64-openssl-1.1.x"), + ('x86_64', "alpine", "", "musl", 'linux-musl'), + ('aarch64', "ubuntu", "debian", "debian", 'linux-arm64-openssl-1.1.x'), + ('arm', "raspbian", "", "arm", 'linux-arm-openssl-1.1.x'), + ('x86_64', "nixos", "", "nixos", 'linux-nixos'), + ('x86_64', "fedora", "fedora", "rhel", 'rhel-openssl-1.1.x'), + ('x86_64', "debian", "debian", "debian", 'debian-openssl-1.1.x'), + ('x86_64', "niche-distro", "niche-like", None, 'debian-openssl-1.1.x'), + ('arm', "niche-distro", "niche-like", None, 'linux-arm-openssl-1.1.x'), + ('aarch64', "niche-distro", "niche-like", None, 'linux-arm64-openssl-1.1.x'), ] @pytest.mark.parametrize( - "machine,distro_id,distro_like,resolved_distro,expected_platform", + 'machine,distro_id,distro_like,resolved_distro,expected_platform', LINUX_PLATFORMS, ) def test_resolve_linux( @@ -134,19 +134,19 @@ def test_resolve_linux( ): # pylint: disable=too-many-arguments """ - Mocks the platform.system() to return "linux" and platform.machine() - to return either "aarch64" or "x86_64". + Mocks the platform.system() to return 'linux' and platform.machine() + to return either 'aarch64' or 'x86_64'. Tests resolve_platform() and asserts that it returns expected binary platform. Additionaly tests resolve_known_distro() """ # TODO: arguments can be simplified to a namedtuple or something - monkeypatch.setattr(platform, "system", mock("Linux")) - monkeypatch.setattr(platform, "machine", mock(machine)) - monkeypatch.setattr(binaries_platform, "get_openssl", mock("1.1.x")) - monkeypatch.setattr(distro, "id", mock(distro_id)) - monkeypatch.setattr(distro, "like", mock(distro_like)) + monkeypatch.setattr(platform, 'system', mock('Linux')) + monkeypatch.setattr(platform, 'machine', mock(machine)) + monkeypatch.setattr(binaries_platform, 'get_openssl', mock('1.1.x')) + monkeypatch.setattr(distro, 'id', mock(distro_id)) + monkeypatch.setattr(distro, 'like', mock(distro_like)) os_settings = binaries_platform.get_os_settings() assert os_settings.distro == resolved_distro assert binaries_platform.resolve_platform(os_settings) == expected_platform diff --git a/tests/test_engine.py b/tests/test_engine.py index f33af8a69..41d4075a3 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -88,12 +88,12 @@ def test_mismatched_version_error(fake_process: FakeProcess) -> None: # TODO: reimplement this test @pytest.mark.skip( - reason='Local binary paths are no longer supported, this test still exists as support will be added again in the future' + reason='Local binary paths are no longer supported, this test still exists as support will be added again in the future' # pylint: disable=line-too-long ) def test_ensure_local_path(testdir: Testdir, fake_process: FakeProcess) -> None: """Query engine in current directory required to be the expected version""" - fake_engine = testdir.path / ("prisma-query-engine" + PLATFORM_EXE_EXTENSION) + fake_engine = testdir.path / ('prisma-query-engine' + PLATFORM_EXE_EXTENSION) fake_engine.touch() fake_process.register_subprocess( From d9baefaaf50600209ec5a76d4d364fe1d81b1f25 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Fri, 28 Jan 2022 10:45:53 -0500 Subject: [PATCH 49/55] chore: some minor nitpicks and refactoring --- codecov.yml | 1 + src/prisma/_compat.py | 3 +-- src/prisma/binaries/__init__.py | 1 - src/prisma/binaries/binaries.py | 21 ++++++++--------- src/prisma/binaries/download.py | 5 +---- src/prisma/binaries/platform.py | 30 ++++++++++++------------- tests/test_binaries.py | 40 ++++++++++++++++++++------------- 7 files changed, 52 insertions(+), 49 deletions(-) diff --git a/codecov.yml b/codecov.yml index e4c0c0767..876756e39 100644 --- a/codecov.yml +++ b/codecov.yml @@ -7,6 +7,7 @@ coverage: target: 100% paths: - "tests/" + - "src/prisma/binaries/platform.py" branches: - main prisma: diff --git a/src/prisma/_compat.py b/src/prisma/_compat.py index 48af73a60..c4fb25006 100644 --- a/src/prisma/_compat.py +++ b/src/prisma/_compat.py @@ -49,9 +49,8 @@ def validator( else: try: from cached_property import cached_property as cached_property - except ImportError: + except ImportError: # pragma: no cover # If we can't import cached_property, fallback to the standard property cached_property = property else: - from functools import cached_property as cached_property diff --git a/src/prisma/binaries/__init__.py b/src/prisma/binaries/__init__.py index cb0849161..28b54a7cc 100644 --- a/src/prisma/binaries/__init__.py +++ b/src/prisma/binaries/__init__.py @@ -1,4 +1,3 @@ # -*- coding: utf-8 -*- - from .binaries import * diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index 77d07f779..e66dbb384 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -12,14 +12,15 @@ from pydantic import BaseSettings, Field from prisma.errors import PrismaError -from prisma.utils import time_since - - from . import platform from .download import download +from ..utils import time_since -# Get system information -OS_SETTINGS: platform.OsSettings = platform.get_os_settings() + +log: logging.Logger = logging.getLogger(__name__) + +# system information +OS_SETTINGS: platform.OsSettings = platform.OsSettings.from_env() PLATFORM: str = platform.resolve_platform(OS_SETTINGS) CLI_PLATFORM: str = OS_SETTINGS.system @@ -32,11 +33,12 @@ ENGINE_VERSION = os.environ.get( 'PRISMA_ENGINE_VERSION', '34df67547cf5598f5a6cd3eb45f14ee70c3fb86f' ) + +# path constants GLOBAL_TEMP_DIR: Path = ( Path(tempfile.gettempdir()) / 'prisma' / 'binaries' / 'engines' / ENGINE_VERSION ) PLATFORM_EXE_EXTENSION: str = '.exe' if OS_SETTINGS.is_windows() else '' - PRISMA_CLI_NAME = f'prisma-cli-{PRISMA_VERSION}-{CLI_PLATFORM}{PLATFORM_EXE_EXTENSION}' @@ -81,9 +83,6 @@ def prisma_cli_url(self) -> str: SETTINGS: PrismaSettings = PrismaSettings() -log: logging.Logger = logging.getLogger(__name__) - - class InvalidBinaryVersion(PrismaError): pass @@ -130,8 +129,10 @@ def ensure_binary(self) -> None: raise FileNotFoundError( f'{self.name} binary not found at {self.path}\nTry running `prisma fetch`' ) + if not self.path.is_file(): raise IsADirectoryError(f'{self.name} binary is a directory') + # Run binary with --version to check if it's the right version and capture stdout start_version = time.monotonic() process = subprocess.run( @@ -185,6 +186,7 @@ def binaries_from_settings( ) -> BinariesType: if engines is None: engines = engines_from_settings(settings) + return ( *engines, Binary( @@ -196,7 +198,6 @@ def binaries_from_settings( ENGINES: EnginesType = engines_from_settings(SETTINGS) - BINARIES: BinariesType = binaries_from_settings(SETTINGS, engines=ENGINES) diff --git a/src/prisma/binaries/download.py b/src/prisma/binaries/download.py index 1012e85b9..1dedcaf30 100644 --- a/src/prisma/binaries/download.py +++ b/src/prisma/binaries/download.py @@ -2,6 +2,7 @@ import os import shutil from pathlib import Path +from stat import S_IXUSR, S_IWUSR, S_IRUSR from ..http import client from ..utils import maybe_async_run @@ -19,11 +20,7 @@ def download(url: str, to: Path) -> None: shutil.copyfileobj(f_in, f_out) # chmod +x - from stat import S_IXUSR, S_IWUSR, S_IRUSR - os.chmod(to, S_IXUSR | S_IWUSR | S_IRUSR) - # override the original - # remove temporary files os.remove(tar) diff --git a/src/prisma/binaries/platform.py b/src/prisma/binaries/platform.py index a2a9b79e9..87ba81917 100644 --- a/src/prisma/binaries/platform.py +++ b/src/prisma/binaries/platform.py @@ -1,11 +1,11 @@ -import platform import re -import subprocess import sys -from functools import lru_cache +import platform +import subprocess from typing import Optional +from functools import lru_cache -import distro # type: ignore # pyright: reportMissingTypeStubs=false +import distro # type: ignore[import] class OsSettings: @@ -22,6 +22,16 @@ def __init__( self.libssl = libssl self.distro = distro + @classmethod + def from_env(cls) -> 'OsSettings': + """Resolve settings from the current machine / environment""" + return cls( + libssl=get_openssl(), + system=platform.system().lower(), + machine=platform.machine().lower(), + distro=resolve_known_distro(distro.id(), distro.like()), + ) + def is_windows(self) -> bool: return self.system.lower() == 'windows' @@ -75,18 +85,6 @@ def resolve_known_distro(distro_id: str, distro_like: str) -> Optional[str]: return None -def get_os_settings() -> OsSettings: - system = platform.system().lower() - machine = platform.machine().lower() - openssl_version = get_openssl() - distro_id = distro.id() - distro_like = distro.like() - distr = resolve_known_distro(distro_id, distro_like) - return OsSettings( - system=system, machine=machine, libssl=openssl_version, distro=distr - ) - - def resolve_darwin(os_settings: OsSettings) -> Optional[str]: if os_settings.system == 'darwin' and os_settings.machine == 'aarch64': return 'darwin-arm64' diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 457b8f1fd..8911041ca 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -1,16 +1,18 @@ import platform from typing import Any, List, Optional, Tuple + import pytest +import distro # type: ignore[import] -import distro # type: ignore # pyright: reportMissingTypeStubs=false -import prisma.binaries.platform as binaries_platform +from prisma.binaries import platform as binaries_platform +from prisma.binaries.platform import OsSettings def mock(obj: Any): return lambda: obj -def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): +def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch) -> None: """ Mock the platform.system() to return 'Windows' and platform.machine() to return default 'x86_64'. @@ -27,7 +29,7 @@ def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(distro, 'like', mock('')) assert platform.system() == 'Windows' - os_settings = binaries_platform.get_os_settings() + os_settings = OsSettings.from_env() assert os_settings.is_windows() assert binaries_platform.resolve_darwin(os_settings) is None @@ -35,11 +37,17 @@ def test_mock_windows_os(monkeypatch: pytest.MonkeyPatch): assert binaries_platform.resolve_other_platforms(os_settings) == 'windows' -MACHINES = ['aarch64', 'x86_64'] +# MACHINE, EXPECTED +DARWIN_PLATFORMS = [ + ('x86_64', 'darwin'), + ('aarch64', 'darwin-arm64'), +] -@pytest.mark.parametrize(('machine,'), MACHINES) -def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): +@pytest.mark.parametrize(('machine,expected'), DARWIN_PLATFORMS) +def test_mock_darwin_os( + monkeypatch: pytest.MonkeyPatch, machine: str, expected: str +) -> None: """ Mock the platform.system() to return 'Darwin' and platform.machine() to return either 'aarch64' or 'x86_64'. @@ -58,13 +66,10 @@ def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): assert platform.system() == 'Darwin' - os_settings = binaries_platform.get_os_settings() + os_settings = OsSettings.from_env() assert not os_settings.is_windows() - if machine == 'aarch64': - assert binaries_platform.resolve_darwin(os_settings) == 'darwin-arm64' - else: - assert binaries_platform.resolve_darwin(os_settings) == 'darwin' + assert binaries_platform.resolve_darwin(os_settings) == expected assert binaries_platform.resolve_linux(os_settings) is None assert binaries_platform.resolve_other_platforms(os_settings) is None @@ -86,7 +91,7 @@ def test_mock_darwin_os(monkeypatch: pytest.MonkeyPatch, machine: str): @pytest.mark.parametrize(('distro_id,distro_like,expected'), DISTRO_ID_LIKE) def test_resolve_known_distro( monkeypatch: pytest.MonkeyPatch, distro_id: str, distro_like: str, expected: str -): +) -> None: """ Mock the platform.system() to return 'linux' and platform.machine() to return default 'x86_64'. @@ -100,7 +105,7 @@ def test_resolve_known_distro( monkeypatch.setattr(distro, 'id', mock(distro_id)) monkeypatch.setattr(distro, 'like', mock(distro_like)) - os_settings = binaries_platform.get_os_settings() + os_settings = OsSettings.from_env() assert os_settings.distro == expected @@ -131,7 +136,7 @@ def test_resolve_linux( machine: str, resolved_distro: str, expected_platform: str, -): # pylint: disable=too-many-arguments +) -> None: # pylint: disable=too-many-arguments """ Mocks the platform.system() to return 'linux' and platform.machine() @@ -147,6 +152,9 @@ def test_resolve_linux( monkeypatch.setattr(binaries_platform, 'get_openssl', mock('1.1.x')) monkeypatch.setattr(distro, 'id', mock(distro_id)) monkeypatch.setattr(distro, 'like', mock(distro_like)) - os_settings = binaries_platform.get_os_settings() + os_settings = OsSettings.from_env() assert os_settings.distro == resolved_distro assert binaries_platform.resolve_platform(os_settings) == expected_platform + + +# TODO: test engine URLs are valid From 722352a77780c5decdd9a5cfe4a262de302c36fe Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Fri, 28 Jan 2022 10:57:25 -0500 Subject: [PATCH 50/55] chore: fix lint error --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index cffe2b250..ea022398e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -33,6 +33,7 @@ disable = # I don't think this is valuable too-many-ancestors, + too-many-arguments, # this should only trigger for tests and I don't care about duplicating code for tests duplicate-code, From 4b8947bdccd41c7f5e2c95893b23692c27e5c8aa Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Fri, 28 Jan 2022 12:02:42 -0500 Subject: [PATCH 51/55] chore: fix mypy errors --- tests/test_binaries.py | 2 +- tests/test_engine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_binaries.py b/tests/test_binaries.py index 8911041ca..4ccc6d85f 100644 --- a/tests/test_binaries.py +++ b/tests/test_binaries.py @@ -8,7 +8,7 @@ from prisma.binaries.platform import OsSettings -def mock(obj: Any): +def mock(obj: Any) -> Any: return lambda: obj diff --git a/tests/test_engine.py b/tests/test_engine.py index 41d4075a3..f503a2adb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -49,7 +49,7 @@ async def test_engine_connects() -> None: await db.disconnect() -def test_stopping_engine_on_closed_loop(): +def test_stopping_engine_on_closed_loop() -> None: """Stopping the engine with no event loop available does not raise an error""" with no_event_loop(): engine = QueryEngine(dml='') From 572ea7753b82b054a171eaaa0babea924d30a1ba Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Sat, 29 Jan 2022 23:35:44 +0100 Subject: [PATCH 52/55] chore: remove unused errors --- src/prisma/engine/errors.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/prisma/engine/errors.py b/src/prisma/engine/errors.py index 738123aec..0b47c75c7 100644 --- a/src/prisma/engine/errors.py +++ b/src/prisma/engine/errors.py @@ -4,8 +4,6 @@ __all__ = ( 'EngineError', - 'BinaryNotFoundError', - 'MismatchedVersionsError', 'EngineConnectionError', 'EngineRequestError', 'AlreadyConnectedError', @@ -18,10 +16,6 @@ class EngineError(PrismaError): pass -class BinaryNotFoundError(EngineError): - pass - - class AlreadyConnectedError(EngineError): pass @@ -30,16 +24,6 @@ class NotConnectedError(EngineError): pass -class MismatchedVersionsError(EngineError): - got: str - expected: str - - def __init__(self, *, expected: str, got: str): - super().__init__(f'Expected query engine version `{expected}` but got `{got}`') - self.expected = expected - self.got = got - - class EngineConnectionError(EngineError): pass From 92a5ea33b22930dcbbdad152adf18c9a97666ab4 Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Sat, 29 Jan 2022 23:36:00 +0100 Subject: [PATCH 53/55] feat: improve InvalidBinaryVersion exception --- src/prisma/binaries/binaries.py | 13 +++++++++---- tests/test_engine.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/prisma/binaries/binaries.py b/src/prisma/binaries/binaries.py index e66dbb384..e5d2d6ec3 100644 --- a/src/prisma/binaries/binaries.py +++ b/src/prisma/binaries/binaries.py @@ -84,7 +84,14 @@ def prisma_cli_url(self) -> str: class InvalidBinaryVersion(PrismaError): - pass + binary: 'Binary' + expected: str + actual: str + + def __init__(self, binary: 'Binary', actual: str, expected: str = ENGINE_VERSION): + super().__init__(f'{binary.name}\'s binary version {actual} is not {expected}') + self.expected = expected + self.actual = actual class Binary: @@ -145,9 +152,7 @@ def ensure_binary(self) -> None: # or query-engine 34df67547cf5598f5a6cd3eb45f14ee70c3fb86f version = process.stdout.decode().split(' ')[1] if version != ENGINE_VERSION: - raise InvalidBinaryVersion( - f'{self.name} binary version {version} is not {ENGINE_VERSION}' - ) + raise InvalidBinaryVersion(self, version) EnginesType = Tuple[ diff --git a/tests/test_engine.py b/tests/test_engine.py index f503a2adb..d5c30a894 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -82,7 +82,7 @@ def test_mismatched_version_error(fake_process: FakeProcess) -> None: query_engine.ensure_binary() # "{self.name} binary version {version} is not {ENGINE_VERSION}" assert exc.match( - f'{query_engine.name} binary version unexpected-hash is not {ENGINE_VERSION}' + f'{query_engine.name}\'s binary version unexpected-hash is not {ENGINE_VERSION}' ) @@ -128,7 +128,7 @@ def test_ensure_env_override(testdir: Testdir, fake_process: FakeProcess) -> Non with pytest.raises(InvalidBinaryVersion) as exc: query_engine.ensure_binary() assert exc.match( - f'{query_engine.name} binary version a-different-hash is not {ENGINE_VERSION}' + f'{query_engine.name}\'s binary version a-different-hash is not {ENGINE_VERSION}' ) # query_engine.ensure_binary() From 2eea57f4a210ec11ee861f2976847882fe7130c9 Mon Sep 17 00:00:00 2001 From: Robert Craigie Date: Thu, 17 Feb 2022 17:08:11 -0500 Subject: [PATCH 54/55] chore: rename codecov target --- codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index 4dc53d2ed..14537dd10 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,7 +3,7 @@ coverage: project: # default is required to exist default: false - tests: + full-coverage: target: 100% paths: - "tests/" From 6f594bf4679c4e56b8c6b81b1ff948657d44f9fb Mon Sep 17 00:00:00 2001 From: HKGx <19597269+HKGx@users.noreply.github.com> Date: Sun, 10 Apr 2022 03:49:49 +0200 Subject: [PATCH 55/55] feat(AbstractHTTP): improve typing on download method --- src/prisma/_async_http.py | 4 ++-- src/prisma/_sync_http.py | 4 ++-- src/prisma/http_abstract.py | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/prisma/_async_http.py b/src/prisma/_async_http.py index 6bead236f..704e86286 100644 --- a/src/prisma/_async_http.py +++ b/src/prisma/_async_http.py @@ -4,7 +4,7 @@ import httpx from ._types import Method -from .http_abstract import AbstractResponse, AbstractHTTP +from .http_abstract import AbstractResponse, AbstractHTTP, PathToDest __all__ = ('HTTP', 'Response', 'client') @@ -15,7 +15,7 @@ class HTTP(AbstractHTTP[httpx.AsyncClient, httpx.Response]): __slots__ = () - async def download(self, url: str, dest: str) -> None: + async def download(self, url: str, dest: PathToDest) -> None: async with self.session.stream('GET', url, timeout=None) as resp: resp.raise_for_status() with open(dest, 'wb') as fd: diff --git a/src/prisma/_sync_http.py b/src/prisma/_sync_http.py index d80dfa4d7..11a39e00f 100644 --- a/src/prisma/_sync_http.py +++ b/src/prisma/_sync_http.py @@ -3,7 +3,7 @@ import httpx from ._types import Method -from .http_abstract import AbstractResponse, AbstractHTTP +from .http_abstract import AbstractResponse, AbstractHTTP, PathToDest __all__ = ('HTTP', 'Response', 'client') @@ -14,7 +14,7 @@ class HTTP(AbstractHTTP[httpx.Client, httpx.Response]): __slots__ = () - def download(self, url: str, dest: str) -> None: + def download(self, url: str, dest: PathToDest) -> None: with self.session.stream('GET', url, timeout=None) as resp: resp.raise_for_status() with open(dest, 'wb') as fd: diff --git a/src/prisma/http_abstract.py b/src/prisma/http_abstract.py index c8d9c3426..b1be19db6 100644 --- a/src/prisma/http_abstract.py +++ b/src/prisma/http_abstract.py @@ -1,4 +1,5 @@ from abc import abstractmethod, ABC +import os from typing import ( Any, Union, @@ -22,6 +23,7 @@ Response = TypeVar('Response') ReturnType = TypeVar('ReturnType') MaybeCoroutine = Union[Coroutine[Any, Any, ReturnType], ReturnType] +PathToDest = Union[str, bytes, 'os.PathLike[str]'] DEFAULT_CONFIG: Dict[str, Any] = { 'limits': Limits(max_connections=1000), @@ -52,7 +54,7 @@ def __init__(self, **kwargs: Any) -> None: } @abstractmethod - def download(self, url: str, dest: str) -> MaybeCoroutine[None]: + def download(self, url: str, dest: PathToDest) -> MaybeCoroutine[None]: ... @abstractmethod