From 6e4641d0d1ae96148e43cc629c215499bea4a392 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 2 May 2023 18:50:20 +0200 Subject: [PATCH] Ugly hack to link pip-tools on demand into the venv (#95) --- rye/src/bootstrap.rs | 24 +++++++++++++++++++++++- rye/src/lock.rs | 26 ++++++++++++-------------- rye/src/sync.rs | 12 +++++------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/rye/src/bootstrap.rs b/rye/src/bootstrap.rs index 093f4bb422..bd9cf88213 100644 --- a/rye/src/bootstrap.rs +++ b/rye/src/bootstrap.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::env::consts::{ARCH, OS}; use std::io::Write; +use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::atomic::{self, AtomicBool}; @@ -158,7 +159,6 @@ fn do_update(output: CommandOutput, venv_dir: &Path, app_dir: &Path) -> Result<( } #[cfg(not(target_os = "linux"))] { - use std::os::unix::fs::symlink; symlink(&this, shims.join("python")).context("tried to symlink python shim")?; symlink(&this, shims.join("python3")).context("tried to symlink python3 shim")?; } @@ -180,6 +180,28 @@ pub fn get_pip_module(venv: &Path) -> PathBuf { rv } +/// Links all of the pip-tools modules into a given folder. +/// +/// This is a very ugly workaround to us not having pip tools in the virtualenv. +pub fn link_pip_tools(venv: &Path, folder: &Path) -> Result<(), Error> { + let mut lib = venv.to_path_buf(); + lib.push("lib"); + lib.push(SELF_SITE_PACKAGES); + for module in [ + "pip", + "setuptools", + "wheel", + "build", + "click", + "piptools", + "pyproject_hooks", + ] { + symlink(lib.join(module), folder.join(module)) + .with_context(|| format!("tried to link '{}'", module))?; + } + Ok(()) +} + /// Fetches a version if missing. pub fn fetch( version: &PythonVersionRequest, diff --git a/rye/src/lock.rs b/rye/src/lock.rs index a43c4d0af3..95ddb0558a 100644 --- a/rye/src/lock.rs +++ b/rye/src/lock.rs @@ -1,6 +1,6 @@ use std::collections::{HashMap, HashSet}; use std::io::{BufWriter, Write}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use std::sync::Arc; use std::{fmt, fs}; @@ -12,7 +12,7 @@ use regex::Regex; use tempfile::NamedTempFile; use url::Url; -use crate::bootstrap::ensure_self_venv; +use crate::bootstrap::{ensure_self_venv, link_pip_tools}; use crate::pyproject::{normalize_package_name, DependencyKind, PyProject, Workspace}; use crate::utils::CommandOutput; @@ -52,13 +52,6 @@ pub struct LockOptions { pub pre: bool, } -fn get_pip_compile(output: CommandOutput) -> Result { - let mut pip_compile = ensure_self_venv(output)?; - pip_compile.push("bin"); - pip_compile.push("pip-compile"); - Ok(pip_compile) -} - /// Creates lockfiles for all projects in the workspace. pub fn update_workspace_lockfile( workspace: &Arc, @@ -220,17 +213,21 @@ fn generate_lockfile( exclusions: &HashSet, extra_args: &[&str], ) -> Result<(), Error> { + let self_venv = ensure_self_venv(output)?; let scratch = tempfile::tempdir()?; let requirements_file = scratch.path().join("requirements.txt"); if lockfile.is_file() { fs::copy(lockfile, &requirements_file)?; } else { - fs::write(lockfile, b"")?; + fs::write(&requirements_file, b"")?; } - let pip_compile_path = get_pip_compile(output)?; - let mut cmd = Command::new(pip_compile_path); - cmd.arg("--resolver=backtracking") + link_pip_tools(&self_venv, scratch.path()).context("failed to link pip-tools internals")?; + + let mut cmd = Command::new(workspace_path.join(".venv/bin/python")); + cmd.arg("-c") + .arg("from piptools.scripts.compile import cli; cli()") + .arg("--resolver=backtracking") .arg("--no-annotate") .arg("--strip-extras") .arg("--allow-unsafe") @@ -238,7 +235,8 @@ fn generate_lockfile( .arg("-o") .arg(&requirements_file) .arg(requirements_file_in) - .env("PYTHONWARNINGS", "ignore"); + .env("PYTHONWARNINGS", "ignore") + .env("PYTHONPATH", scratch.path()); if output == CommandOutput::Verbose { cmd.arg("--verbose"); } else { diff --git a/rye/src/sync.rs b/rye/src/sync.rs index 230984a9fa..4728845c45 100644 --- a/rye/src/sync.rs +++ b/rye/src/sync.rs @@ -1,4 +1,3 @@ -use std::os::unix::fs::symlink; use std::path::Path; use std::process::Command; use std::{env, fs}; @@ -8,7 +7,7 @@ use console::style; use serde::{Deserialize, Serialize}; use tempfile::TempDir; -use crate::bootstrap::{ensure_self_venv, fetch, get_pip_module}; +use crate::bootstrap::{ensure_self_venv, fetch, link_pip_tools}; use crate::config::{get_py_bin, load_python_version}; use crate::lock::{ update_single_project_lockfile, update_workspace_lockfile, LockMode, LockOptions, @@ -137,8 +136,6 @@ pub fn sync(cmd: SyncOptions) -> Result<(), Error> { // can pass to pip-sync to install the local package. if recreate || cmd.mode != SyncMode::PythonOnly { let dir = TempDir::new()?; - symlink(get_pip_module(&self_venv), dir.path().join("pip")) - .context("failed linking pip module into for pip-sync")?; if cmd.no_lock { let lockfile = if cmd.dev { &dev_lockfile } else { &lockfile }; @@ -191,17 +188,18 @@ pub fn sync(cmd: SyncOptions) -> Result<(), Error> { if output != CommandOutput::Quiet { eprintln!("Installing dependencies"); } - let mut pip_sync_cmd = Command::new(self_venv.join("bin/pip-sync")); + link_pip_tools(&self_venv, dir.path()).context("failed to link pip-tools internals")?; + let mut pip_sync_cmd = Command::new(pyproject.venv_bin_path().join("python")); let root = pyproject.workspace_path(); pip_sync_cmd + .arg("-c") + .arg("from piptools.scripts.sync import cli; cli()") .env("PYTHONPATH", dir.path()) // XXX: ${PROJECT_ROOT} is supposed to be used in the context of file:/// // so let's make sure it is url escaped. This is pretty hacky but // good enough for now. .env("PROJECT_ROOT", root.to_string_lossy().replace(' ', "%2F")) .current_dir(&root) - .arg("--python-executable") - .arg(venv.join("bin/python")) .arg("--pip-args") // note that the double quotes are necessary to properly handle // spaces in paths