-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
The Python package manager Poetry is now supported for installing app dependencies: https://python-poetry.org To use Poetry apps must have a `poetry.lock` lockfile, which can be created by running `poetry lock` locally, after adding Poetry config to `pyproject.toml` (which can be done either manually or by using `poetry init`). Apps must only have one package manager file (either `requirements.txt` or `poetry.lock`, but not both) otherwise the buildpack will abort the build with an error (which will help prevent some of the types of support tickets we see in the classic buildpack). Poetry is installed into a build-only layer, so is not available at run-time to reduce image size. The app dependencies are installed into a virtual environment (the same as for pip, after #257), which is on `PATH` so does not need explicit activation when using the app image. As such, use of `poetry run` or `poetry shell` is not required at run-time to use dependencies in the environment. When using Poetry, pip is not explicitly installed, since Poetry includes its own bundled copy that it will use instead (for the small number of Poetry operations for which it still calls out to pip, such as package uninstalls). Both the Poetry and app dependencies layers are cached, however, the Poetry download/wheel cache is not cached, since using it is slower than caching the dependencies layer (for more details see the comments on `poetry_dependencies::install_dependencies`). The `poetry install --sync` command is run using `--only main` so as to only install the main dependencies group and not any other groups (such as test/dev/... groups). Relevant Poetry docs: - https://python-poetry.org/docs/cli/#install - https://python-poetry.org/docs/configuration/ - https://python-poetry.org/docs/managing-dependencies/#dependency-groups Work that will be handled later: - Support for selecting Python version via `tool.poetry.dependencies.python`: #260 - Build output and error messages polish/CX review (this will be performed when switching the buildpack to the new logging style). - More detailed user-facing docs: #11 Closes #7. GUS-W-9607867. GUS-W-9608286. GUS-W-9608295.
- Loading branch information
Showing
26 changed files
with
963 additions
and
70 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
poetry==1.8.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,6 @@ | ||
pub(crate) mod pip; | ||
pub(crate) mod pip_cache; | ||
pub(crate) mod pip_dependencies; | ||
pub(crate) mod poetry; | ||
pub(crate) mod poetry_dependencies; | ||
pub(crate) mod python; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
use crate::packaging_tool_versions::POETRY_VERSION; | ||
use crate::python_version::PythonVersion; | ||
use crate::utils::StreamedCommandError; | ||
use crate::{utils, BuildpackError, PythonBuildpack}; | ||
use libcnb::build::BuildContext; | ||
use libcnb::data::layer_name; | ||
use libcnb::layer::{ | ||
CachedLayerDefinition, EmptyLayerCause, InvalidMetadataAction, LayerState, RestoredLayerAction, | ||
}; | ||
use libcnb::layer_env::{LayerEnv, ModificationBehavior, Scope}; | ||
use libcnb::Env; | ||
use libherokubuildpack::log::log_info; | ||
use serde::{Deserialize, Serialize}; | ||
use std::io; | ||
use std::path::Path; | ||
use std::process::Command; | ||
|
||
/// Creates a build-only layer containing Poetry. | ||
pub(crate) fn install_poetry( | ||
context: &BuildContext<PythonBuildpack>, | ||
env: &mut Env, | ||
python_version: &PythonVersion, | ||
python_layer_path: &Path, | ||
) -> Result<(), libcnb::Error<BuildpackError>> { | ||
let new_metadata = PoetryLayerMetadata { | ||
arch: context.target.arch.clone(), | ||
distro_name: context.target.distro_name.clone(), | ||
distro_version: context.target.distro_version.clone(), | ||
python_version: python_version.to_string(), | ||
poetry_version: POETRY_VERSION.to_string(), | ||
}; | ||
|
||
let layer = context.cached_layer( | ||
layer_name!("poetry"), | ||
CachedLayerDefinition { | ||
build: true, | ||
launch: false, | ||
invalid_metadata_action: &|_| InvalidMetadataAction::DeleteLayer, | ||
restored_layer_action: &|cached_metadata: &PoetryLayerMetadata, _| { | ||
let cached_poetry_version = cached_metadata.poetry_version.clone(); | ||
if cached_metadata == &new_metadata { | ||
(RestoredLayerAction::KeepLayer, cached_poetry_version) | ||
} else { | ||
(RestoredLayerAction::DeleteLayer, cached_poetry_version) | ||
} | ||
}, | ||
}, | ||
)?; | ||
|
||
// Move the Python user base directory to this layer instead of under HOME: | ||
// https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE | ||
let mut layer_env = LayerEnv::new().chainable_insert( | ||
Scope::Build, | ||
ModificationBehavior::Override, | ||
"PYTHONUSERBASE", | ||
layer.path(), | ||
); | ||
|
||
match layer.state { | ||
LayerState::Restored { | ||
cause: ref cached_poetry_version, | ||
} => { | ||
log_info(format!("Using cached Poetry {cached_poetry_version}")); | ||
} | ||
LayerState::Empty { ref cause } => { | ||
match cause { | ||
EmptyLayerCause::InvalidMetadataAction { .. } => { | ||
log_info("Discarding cached Poetry since its layer metadata can't be parsed"); | ||
} | ||
EmptyLayerCause::RestoredLayerAction { | ||
cause: cached_poetry_version, | ||
} => { | ||
log_info(format!("Discarding cached Poetry {cached_poetry_version}")); | ||
} | ||
EmptyLayerCause::NewlyCreated => {} | ||
} | ||
|
||
log_info(format!("Installing Poetry {POETRY_VERSION}")); | ||
|
||
// We use the pip wheel bundled within Python's standard library to install Poetry. | ||
// Whilst Poetry does still require pip for some tasks (such as package uninstalls), | ||
// it bundles its own copy for use as a fallback. As such we don't need to install pip | ||
// into the user site-packages (and in fact, Poetry wouldn't use this install anyway, | ||
// since it only finds an external pip if it exists in the target venv). | ||
let bundled_pip_module_path = | ||
utils::bundled_pip_module_path(python_layer_path, python_version) | ||
.map_err(PoetryLayerError::LocateBundledPip)?; | ||
|
||
utils::run_command_and_stream_output( | ||
Command::new("python") | ||
.args([ | ||
&bundled_pip_module_path.to_string_lossy(), | ||
"install", | ||
// There is no point using pip's cache here, since the layer itself will be cached. | ||
"--no-cache-dir", | ||
"--no-input", | ||
"--no-warn-script-location", | ||
"--quiet", | ||
"--user", | ||
format!("poetry=={POETRY_VERSION}").as_str(), | ||
]) | ||
.env_clear() | ||
.envs(&layer_env.apply(Scope::Build, env)), | ||
) | ||
.map_err(PoetryLayerError::InstallPoetryCommand)?; | ||
|
||
layer.write_metadata(new_metadata)?; | ||
} | ||
} | ||
|
||
layer.write_env(&layer_env)?; | ||
// Required to pick up the automatic PATH env var. See: https://github.com/heroku/libcnb.rs/issues/842 | ||
layer_env = layer.read_env()?; | ||
env.clone_from(&layer_env.apply(Scope::Build, env)); | ||
|
||
Ok(()) | ||
} | ||
|
||
// Some of Poetry's dependencies contain compiled components so are platform-specific (unlike pure | ||
// Python packages). As such we have to take arch and distro into account for cache invalidation. | ||
#[derive(Deserialize, PartialEq, Serialize)] | ||
#[serde(deny_unknown_fields)] | ||
struct PoetryLayerMetadata { | ||
arch: String, | ||
distro_name: String, | ||
distro_version: String, | ||
python_version: String, | ||
poetry_version: String, | ||
} | ||
|
||
/// Errors that can occur when installing Poetry into a layer. | ||
#[derive(Debug)] | ||
pub(crate) enum PoetryLayerError { | ||
InstallPoetryCommand(StreamedCommandError), | ||
LocateBundledPip(io::Error), | ||
} | ||
|
||
impl From<PoetryLayerError> for libcnb::Error<BuildpackError> { | ||
fn from(error: PoetryLayerError) -> Self { | ||
Self::BuildpackError(BuildpackError::PoetryLayer(error)) | ||
} | ||
} |
Oops, something went wrong.