Skip to content

Commit

Permalink
env: default to enabling pip/wheels/setuptools
Browse files Browse the repository at this point in the history
For project virtual environments, default to enabling pip, setuptools
and wheel packages to retain existing stable behaviour to prevent
unexpected breakages caused by development environments making
assumptions of base package availability in virtual environments.

Poetry itself does not require the use of these packages and will
execute correctly within environments that do not have these packages.

This change retains the ability to manage these packages as direct
project dependency as introduced in python-poetry#2826. All poetry internal
execution of pip is retaining the use of the wheel embedded within
the virtualenv package used by poetry.

In cases where a one of these reserved packages are being managed as a
project dependency, the will be treated as any other project
dependency. Executing `poetry install --remove-untracked` will not
remove any of these reserved packages. However, `poetry add pip` and
`poetry remove pip` will trigger the update and removal of `pip`
respectively.

Relates-to: python-poetry#2826
Relates-to: python-poetry#3916
  • Loading branch information
abn committed Apr 30, 2021
1 parent 3dceee3 commit 02e2b3e
Show file tree
Hide file tree
Showing 11 changed files with 253 additions and 111 deletions.
13 changes: 11 additions & 2 deletions poetry/console/commands/debug/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,14 @@ def handle(self) -> Optional[int]:

pool = self.poetry.pool

solver = Solver(package, pool, Repository(), Repository(), self._io)
solver = Solver(
package,
pool,
Repository(),
Repository(),
self._io,
config=self.poetry.config,
)

ops = solver.solve()

Expand Down Expand Up @@ -121,7 +128,9 @@ def handle(self) -> Optional[int]:

pool.add_repository(locked_repository)

solver = Solver(package, pool, Repository(), Repository(), NullIO())
solver = Solver(
package, pool, Repository(), Repository(), NullIO(), poetry=self.poetry
)
with solver.use_environment(env):
ops = solver.solve()

Expand Down
1 change: 1 addition & 0 deletions poetry/console/commands/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ def handle(self) -> Optional[int]:
installed=Repository(),
locked=locked_repo,
io=NullIO(),
config=self.poetry.config,
)
solver.provider.load_deferred(False)
with solver.use_environment(self.env):
Expand Down
4 changes: 4 additions & 0 deletions poetry/installation/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ def __init__(
installed = self._get_installed()

self._installed_repository = installed
self._poetry_config = config

@property
def executor(self) -> Executor:
Expand Down Expand Up @@ -209,6 +210,7 @@ def _do_refresh(self) -> int:
locked_repository,
locked_repository,
self._io,
config=self._poetry_config,
)

ops = solver.solve(use_latest=[])
Expand Down Expand Up @@ -247,6 +249,7 @@ def _do_install(self, local_repo: Repository) -> int:
locked_repository,
self._io,
remove_untracked=self._remove_untracked,
config=self._poetry_config,
)

ops = solver.solve(use_latest=self._whitelist)
Expand Down Expand Up @@ -316,6 +319,7 @@ def _do_install(self, local_repo: Repository) -> int:
locked_repository,
NullIO(),
remove_untracked=self._remove_untracked,
config=self._poetry_config,
)
# Everything is resolved at this point, so we no longer need
# to load deferred dependencies (i.e. VCS, URL and path dependencies)
Expand Down
30 changes: 26 additions & 4 deletions poetry/puzzle/solver.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@


if TYPE_CHECKING:
from poetry.config.config import Config
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.directory_dependency import DirectoryDependency
from poetry.core.packages.file_dependency import FileDependency
Expand All @@ -49,6 +50,7 @@ def __init__(
io: IO,
remove_untracked: bool = False,
provider: Optional[Provider] = None,
config: Optional["Config"] = None,
):
self._package = package
self._pool = pool
Expand All @@ -63,10 +65,32 @@ def __init__(
self._overrides = []
self._remove_untracked = remove_untracked

self._poetry_config = config
self._preserved_package_names = None

@property
def provider(self) -> Provider:
return self._provider

@property
def preserved_package_names(self):
if self._preserved_package_names is None:
self._preserved_package_names = {
self._package.name,
*Provider.UNSAFE_PACKAGES,
}

deps = {package.name for package in self._locked.packages}

# preserve pip/setuptools/wheel when not managed by poetry, this is so
# to avoid externally managed virtual environments causing unnecessary
# removals.
for name in {"pip", "wheel", "setuptools"}:
if name not in deps:
self._preserved_package_names.add(name)

return self._preserved_package_names

@contextmanager
def use_environment(self, env: Env) -> None:
with self.provider.use_environment(env):
Expand Down Expand Up @@ -190,11 +214,9 @@ def solve(self, use_latest: List[str] = None) -> List["OperationTypes"]:
locked_names = {locked.name for locked in self._locked.packages}

for installed in self._installed.packages:
if installed.name == self._package.name:
continue
if installed.name in Provider.UNSAFE_PACKAGES:
# Never remove pip, setuptools etc.
if installed.name in self.preserved_package_names:
continue

if installed.name not in locked_names:
operations.append(Uninstall(installed))

Expand Down
107 changes: 61 additions & 46 deletions poetry/utils/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -877,13 +877,8 @@ def create_venv(
io.write_line(
"Creating virtualenv <c1>{}</> in {}".format(name, str(venv_path))
)

self.build_venv(
venv,
executable=executable,
flags=self._poetry.config.get("virtualenvs.options"),
)
else:
create_venv = False
if force:
if not env.is_sane():
io.write_line(
Expand All @@ -895,14 +890,23 @@ def create_venv(
"Recreating virtualenv <c1>{}</> in {}".format(name, str(venv))
)
self.remove_venv(venv)
self.build_venv(
venv,
executable=executable,
flags=self._poetry.config.get("virtualenvs.options"),
)
create_venv = True
elif io.is_very_verbose():
io.write_line(f"Virtualenv <c1>{name}</> already exists.")

if create_venv:
self.build_venv(
venv,
executable=executable,
flags=self._poetry.config.get("virtualenvs.options"),
# TODO: in a future version switch remove pip/setuptools/wheel
# poetry does not need them these exists today to not break developer
# environment assumptions
with_pip=True,
with_setuptools=True,
with_wheel=True,
)

# venv detection:
# stdlib venv may symlink sys.executable, so we can't use realpath.
# but others can symlink *to* the venv Python,
Expand All @@ -927,12 +931,29 @@ def build_venv(
path: Union[Path, str],
executable: Optional[Union[str, Path]] = None,
flags: Dict[str, bool] = None,
with_pip: bool = False,
with_pip: Optional[bool] = None,
with_wheel: Optional[bool] = None,
with_setuptools: Optional[bool] = None,
) -> virtualenv.run.session.Session:
flags = flags or {}

flags["no-pip"] = (
not with_pip if with_pip is not None else flags.pop("no-pip", True)
)

flags["no-setuptools"] = (
not with_setuptools
if with_setuptools is not None
else flags.pop("no-setuptools", True)
)

# we want wheels to be enabled when pip is required and it has not been explicitly disabled
flags["no-wheel"] = (
not with_wheel
if with_wheel is not None
else flags.pop("no-wheel", flags["no-pip"])
)

if isinstance(executable, Path):
executable = executable.resolve().as_posix()

Expand All @@ -943,20 +964,6 @@ def build_venv(
executable or sys.executable,
]

if not with_pip:
args.append("--no-pip")
else:
if with_wheel is None:
# we want wheels to be enabled when pip is required and it has
# not been explicitly disabled
with_wheel = True

if with_wheel is None or not with_wheel:
args.append("--no-wheel")

if with_setuptools is None or not with_setuptools:
args.append("--no-setuptools")

for flag, value in flags.items():
if value is True:
args.append(f"--{flag}")
Expand Down Expand Up @@ -1039,6 +1046,8 @@ def __init__(self, path: Path, base: Optional[Path] = None) -> None:
self._platlib = None
self._script_dirs = None

self._embedded_pip_path = None

@property
def path(self) -> Path:
return self._path
Expand Down Expand Up @@ -1074,6 +1083,12 @@ def get_embedded_wheel(self, distribution):
distribution, "{}.{}".format(self.version_info[0], self.version_info[1])
).path

@property
def pip_embedded(self) -> str:
if self._embedded_pip_path is None:
self._embedded_pip_path = str(self.get_embedded_wheel("pip") / "pip")
return self._embedded_pip_path

@property
def pip(self) -> str:
"""
Expand All @@ -1082,7 +1097,7 @@ def pip(self) -> str:
# we do not use as_posix() here due to issues with windows pathlib2 implementation
path = self._bin("pip")
if not Path(path).exists():
return str(self.get_embedded_wheel("pip") / "pip")
return str(self.pip_embedded)
return path

@property
Expand Down Expand Up @@ -1187,7 +1202,7 @@ def get_python_implementation(self) -> str:
def get_marker_env(self) -> Dict[str, Any]:
raise NotImplementedError()

def get_pip_command(self) -> List[str]:
def get_pip_command(self, embedded: bool = False) -> List[str]:
raise NotImplementedError()

def get_supported_tags(self) -> List[Tag]:
Expand All @@ -1208,16 +1223,20 @@ def is_sane(self) -> bool:
"""
return True

def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
def get_command_from_bin(self, bin: str) -> List[str]:
if bin == "pip":
return self.run_pip(*args, **kwargs)
# when pip is required we need to ensure that we fallback to
# embedded pip when pip is not available in the environment
return self.get_pip_command()

return [self._bin(bin)]

bin = self._bin(bin)
cmd = [bin] + list(args)
def run(self, bin: str, *args: str, **kwargs: Any) -> Union[str, int]:
cmd = self.get_command_from_bin(bin) + list(args)
return self._run(cmd, **kwargs)

def run_pip(self, *args: str, **kwargs: Any) -> Union[int, str]:
pip = self.get_pip_command()
pip = self.get_pip_command(embedded=True)
cmd = pip + list(args)
return self._run(cmd, **kwargs)

Expand Down Expand Up @@ -1260,17 +1279,13 @@ def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]:
return decode(output)

def execute(self, bin: str, *args: str, **kwargs: Any) -> Optional[int]:
if bin == "pip":
return self.run_pip(*args, **kwargs)

bin = self._bin(bin)
command = self.get_command_from_bin(bin) + list(args)
env = kwargs.pop("env", {k: v for k, v in os.environ.items()})

if not self._is_windows:
args = [bin] + list(args)
return os.execvpe(bin, args, env=env)
return os.execvpe(command[0], command, env=env)
else:
exe = subprocess.Popen([bin] + list(args), env=env, **kwargs)
exe = subprocess.Popen([command[0]] + command[1:], env=env, **kwargs)
exe.communicate()
return exe.returncode

Expand Down Expand Up @@ -1338,10 +1353,10 @@ def get_version_info(self) -> Tuple[int]:
def get_python_implementation(self) -> str:
return platform.python_implementation()

def get_pip_command(self) -> List[str]:
def get_pip_command(self, embedded: bool = False) -> List[str]:
# If we're not in a venv, assume the interpreter we're running on
# has a pip and use that
return [sys.executable, self.pip]
return [sys.executable, self.pip_embedded if embedded else self.pip]

def get_paths(self) -> Dict[str, str]:
# We can't use sysconfig.get_paths() because
Expand Down Expand Up @@ -1445,10 +1460,10 @@ def get_version_info(self) -> Tuple[int]:
def get_python_implementation(self) -> str:
return self.marker_env["platform_python_implementation"]

def get_pip_command(self) -> List[str]:
def get_pip_command(self, embedded: bool = False) -> List[str]:
# We're in a virtualenv that is known to be sane,
# so assume that we have a functional pip
return [self._bin("python"), self.pip]
return [self._bin("python"), self.pip_embedded if embedded else self.pip]

def get_supported_tags(self) -> List[Tag]:
file_path = Path(packaging.tags.__file__)
Expand Down Expand Up @@ -1560,8 +1575,8 @@ def __init__(
self._execute = execute
self.executed = []

def get_pip_command(self) -> List[str]:
return [self._bin("python"), self.pip]
def get_pip_command(self, embedded: bool = False) -> List[str]:
return [self._bin("python"), self.pip_embedded if embedded else self.pip]

def _run(self, cmd: List[str], **kwargs: Any) -> int:
self.executed.append(cmd)
Expand Down
2 changes: 1 addition & 1 deletion poetry/utils/pip.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def pip_install(
executable=environment.python, with_pip=True, with_setuptools=True
) as env:
return environment.run(
env._bin("pip"),
*env.get_pip_command(),
*args,
env={**os.environ, "PYTHONPATH": str(env.purelib)},
)
Expand Down
3 changes: 3 additions & 0 deletions tests/console/commands/env/test_use.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ def test_activate_activates_non_existing_virtualenv_no_envs_file(
venv_py37,
executable="python3.7",
flags={"always-copy": False, "system-site-packages": False},
with_pip=True,
with_setuptools=True,
with_wheel=True,
)

envs_file = TOMLFile(venv_cache / "envs.toml")
Expand Down
2 changes: 1 addition & 1 deletion tests/inspection/test_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def test_info_setup_missing_mandatory_should_trigger_pep517(
except PackageInfoError:
assert spy.call_count == 3
else:
assert spy.call_count == 1
assert spy.call_count == 2


def test_info_prefer_poetry_config_over_egg_info():
Expand Down
Loading

0 comments on commit 02e2b3e

Please sign in to comment.