heroku · edmorley · Nov 6, 2024 · Oct 31, 2024
@@ -2,6 +2,7 @@
 
 ## [Unreleased]
 
+- Added support for the package manager Poetry. Apps must have a `pyproject.toml` + `poetry.lock` and no other package manager files (otherwise pip/Pipenv will take precedence for backwards compatibility). ([#1682](https://github.com/heroku/heroku-buildpack-python/pull/1682))
 
 ## [v263] - 2024-10-31
 

@@ -14,7 +14,7 @@ See the [Getting Started on Heroku with Python](https://devcenter.heroku.com/art
 
 ## Application Requirements
 
-A `requirements.txt` or `Pipfile` file must be present in the root (top-level) directory of your app's source code.
+A `requirements.txt`, `Pipfile` or `poetry.lock` file must be present in the root (top-level) directory of your app's source code.
 
 ## Configuration
 

@@ -28,6 +28,7 @@ source "${BUILDPACK_DIR}/lib/package_manager.sh"
 source "${BUILDPACK_DIR}/lib/pip.sh"
 source "${BUILDPACK_DIR}/lib/pipenv.sh"
 source "${BUILDPACK_DIR}/lib/python_version.sh"
+source "${BUILDPACK_DIR}/lib/poetry.sh"
 
 compile_start_time=$(nowms)
 
@@ -166,6 +167,9 @@ case "${package_manager}" in
 		pip::install_pip_setuptools_wheel "${python_home}" "${python_major_version}"
 		pipenv::install_pipenv
 		;;
+	poetry)
+		poetry::install_poetry "${python_home}" "${CACHE_DIR}" "${EXPORT_PATH}"
+		;;
 	*)
 		utils::abort_internal_error "Unhandled package manager: ${package_manager}"
 		;;
@@ -175,8 +179,8 @@ meta_time "package_manager_install_duration" "${package_manager_install_start_ti
 # SQLite3 support.
 # Installs the sqlite3 dev headers and sqlite3 binary but not the
 # libsqlite3-0 library since that exists in the base image.
-# We skip this step on Python 3.13, as a first step towards removing this feature.
-if [[ "${python_major_version}" == +(3.8|3.9|3.10|3.11|3.12) ]]; then
+# We skip this step on Python 3.13 or when using Poetry, as a first step towards removing this feature.
+if [[ "${python_major_version}" == +(3.8|3.9|3.10|3.11|3.12) && "${package_manager}" != "poetry" ]]; then
 	install_sqlite_start_time=$(nowms)
 	source "${BUILDPACK_DIR}/bin/steps/sqlite3"
 	buildpack_sqlite3_install
@@ -192,6 +196,9 @@ case "${package_manager}" in
 	pipenv)
 		pipenv::install_dependencies
 		;;
+	poetry)
+		poetry::install_dependencies
+		;;
 	*)
 		utils::abort_internal_error "Unhandled package manager: ${package_manager}"
 		;;

@@ -49,9 +49,9 @@ output::error <<EOF
 Error: Your app is configured to use the Python buildpack,
 but we couldn't find any supported Python project files.
 
-A Python app on Heroku must have either a 'requirements.txt' or
-'Pipfile' package manager file in the root directory of its
-source code.
+A Python app on Heroku must have either a 'requirements.txt',
+'Pipfile' or 'poetry.lock' package manager file in the root
+directory of its source code.
 
 Currently the root directory of your app contains:
 

@@ -68,6 +68,7 @@ STRING_FIELDS=(
 	package_manager_multiple_found
 	pip_version
 	pipenv_version
+	poetry_version
 	python_version_major
 	python_version_reason
 	python_version

@@ -5,7 +5,7 @@
 # automatically be executed as part of the build process. If collectstatic
 # fails, your build fails.
 
-# This functionality will only activate if Django is in requirements.txt.
+# This functionality will only activate if Django is installed.
 
 # Runtime arguments:
 #   - $DISABLE_COLLECTSTATIC: disables this functionality.

@@ -104,6 +104,14 @@ function cache::restore() {
 					cache_invalidation_reasons+=("The Pipenv version has changed from ${cached_pipenv_version} to ${PIPENV_VERSION}")
 				fi
 				;;
+			poetry)
+				local cached_poetry_version
+				cached_poetry_version="$(meta_prev_get "poetry_version")"
+				# Poetry support was added after the metadata store, so we'll always have the version here.
+				if [[ "${cached_poetry_version}" != "${POETRY_VERSION:?}" ]]; then
+					cache_invalidation_reasons+=("The Poetry version has changed from ${cached_poetry_version:-"unknown"} to ${POETRY_VERSION}")
+				fi
+				;;
 			*)
 				utils::abort_internal_error "Unhandled package manager: ${package_manager}"
 				;;
@@ -119,6 +127,7 @@ function cache::restore() {
 
 		rm -rf \
 			"${cache_dir}/.heroku/python" \
+			"${cache_dir}/.heroku/python-poetry" \
 			"${cache_dir}/.heroku/python-stack" \
 			"${cache_dir}/.heroku/python-version" \
 			"${cache_dir}/.heroku/src" \

@@ -27,6 +27,15 @@ function package_manager::determine_package_manager() {
 		package_managers_found+=(pip)
 	fi
 
+	# This must be after the requirements.txt check, so that the requirements.txt exported by
+	# `python-poetry-buildpack` takes precedence over poetry.lock, for consistency with the
+	# behaviour prior to this buildpack supporting Poetry natively. In the future the presence
+	# of multiple package manager files will be turned into an error, at which point the
+	# ordering here won't matter.
+	if [[ -f "${build_dir}/poetry.lock" ]]; then
+		package_managers_found+=(poetry)
+	fi
+
 	# TODO: Deprecate/sunset this fallback, since using setup.py declared dependencies is
 	# not a best practice, and we can only guess as to which package manager to use.
 	if ((${#package_managers_found[@]} == 0)) && [[ -f "${build_dir}/setup.py" ]]; then
@@ -47,9 +56,9 @@ function package_manager::determine_package_manager() {
 			output::error <<-EOF
 				Error: Couldn't find any supported Python package manager files.
 
-				A Python app on Heroku must have either a 'requirements.txt' or
-				'Pipfile' package manager file in the root directory of its
-				source code.
+				A Python app on Heroku must have either a 'requirements.txt',
+				'Pipfile' or 'poetry.lock' package manager file in the root
+				directory of its source code.
 
 				Currently the root directory of your app contains:
 
@@ -76,8 +85,7 @@ function package_manager::determine_package_manager() {
 			# TODO: Turn this case into an error since it results in support tickets from users
 			# who don't realise they have multiple package manager files and think their changes
 			# aren't taking effect. (We'll need to wait until after Poetry support has landed,
-			# and people have had a chance to migrate from the third-party Poetry buildpack,
-			# since using it results in both a requirements.txt and a poetry.lock.)
+			# and people have had a chance to migrate from the Poetry buildpack mentioned above.)
 			echo "${package_managers_found[0]}"
 			meta_set "package_manager_multiple_found" "$(
 				IFS=,

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+
+# This is technically redundant, since all consumers of this lib will have enabled these,
+# however, it helps Shellcheck realise the options under which these functions will run.
+set -euo pipefail
+
+POETRY_VERSION=$(utils::get_requirement_version 'poetry')
+
+function poetry::install_poetry() {
+	local python_home="${1}"
+	local cache_dir="${2}"
+	local export_file="${3}"
+
+	# We store Poetry in the build cache, since we only need it during the build.
+	local poetry_root="${cache_dir}/.heroku/python-poetry"
+
+	# We nest the venv and then symlink the `poetry` script to prevent the rest of `venv/bin/`
+	# (such as entrypoint scripts from Poetry's dependencies, or the venv's activation scripts)
+	# from being added to PATH and exposed to the app.
+	local poetry_bin_dir="${poetry_root}/bin"
+	local poetry_venv_dir="${poetry_root}/venv"
+
+	meta_set "poetry_version" "${POETRY_VERSION}"
+
+	# The earlier buildpack cache invalidation step will have already handled the case where the
+	# Poetry version has changed, so here we only need to check that a valid Poetry install exists.
+	# venvs are not relocatable, so if the cache directory were ever to change location, the cached
+	# Poetry installation would stop working. To save having to track the cache location via build
+	# metadata, we instead rely on the fact that relocating the venv would also break the absolute
+	# path `poetry` symlink created below, and that the `-e` condition not only checks that the
+	# `poetry` symlink exists, but that its target is also valid.
+	# Note: Whilst the Codon cache location remains stable from build to build, for Heroku CI the
+	# cache directory currently does not, so the cached Poetry will always be invalidated there.
+	if [[ -e "${poetry_bin_dir}/poetry" ]]; then
+		output::step "Using cached Poetry ${POETRY_VERSION}"
+	else
+		output::step "Installing Poetry ${POETRY_VERSION}"
+
+		# The Poetry directory will already exist in the relocated cache case mentioned above.
+		rm -rf "${poetry_root}"
+
+		python -m venv --without-pip "${poetry_venv_dir}"
+
+		# We use the pip wheel bundled within Python's standard library to install Poetry.
+		# Whilst Poetry does still require pip for some tasks (such as package uninstalls),
+		# it bundles its own copy for use as a fallback. As such we don't need to install pip
+		# into the Poetry venv (and in fact, Poetry wouldn't use this install anyway, since
+		# it only finds an external pip if it exists in the target venv).
+		local bundled_pip_module_path
+		bundled_pip_module_path="$(utils::bundled_pip_module_path "${python_home}")"
+
+		if ! {
+			python "${bundled_pip_module_path}" \
+				--python "${poetry_venv_dir}" \
+				install \
+				--disable-pip-version-check \
+				--no-cache-dir \
+				--no-input \
+				--quiet \
+				"poetry==${POETRY_VERSION}"
+		}; then
+			output::error <<-EOF
+				Error: Unable to install Poetry.
+
+				Try building again to see if the error resolves itself.
+
+				If that does not help, check the status of PyPI (the Python
+				package repository service), here:
+				https://status.python.org
+			EOF
+			meta_set "failure_reason" "install-poetry"
+			return 1
+		fi
+
+		mkdir -p "${poetry_bin_dir}"
+		# NB: This symlink must not use `--relative`, since we want the symlink to break if the cache
+		# (and thus venv) were ever relocated - so that it triggers a reinstall (see above).
+		ln --symbolic --no-target-directory "${poetry_venv_dir}/bin/poetry" "${poetry_bin_dir}/poetry"
+	fi
+
+	export PATH="${poetry_bin_dir}:${PATH}"
+	echo "export PATH=\"${poetry_bin_dir}:\${PATH}\"" >>"${export_file}"
+	# Force Poetry to manage the system Python site-packages instead of using venvs.
+	export POETRY_VIRTUALENVS_CREATE="false"
+	echo 'export POETRY_VIRTUALENVS_CREATE="false"' >>"${export_file}"
+}
+
+# Note: We cache site-packages since:
+# - It results in faster builds than only caching Poetry's download/wheel cache.
+# - It's safe to do so, since `poetry install --sync` fully manages the environment
+#   (including e.g. uninstalling packages when they are removed from the lockfile).
+#
+# With site-packages cached there is no need to persist Poetry's download/wheel cache in the build
+# cache, so we let Poetry write it to the home directory where it will be discarded at the end of
+# the build. We don't use `--no-cache` since the cache still offers benefits (such as avoiding
+# repeat downloads of PEP-517/518 build requirements).
+function poetry::install_dependencies() {
+	local poetry_install_command=(
+		poetry
+		install
+		--sync
+	)
+
+	# On Heroku CI, all default Poetry dependency groups are installed (i.e. all groups minus those
+	# marked as `optional = true`). Otherwise, only the 'main' Poetry dependency group is installed.
+	if [[ ! -v INSTALL_TEST ]]; then
+		poetry_install_command+=(--only main)
+	fi
+
+	# We only display the most relevant command args here, to improve the signal to noise ratio.
+	output::step "Installing dependencies using '${poetry_install_command[*]}'"
+
+	# `--compile`: Compiles Python bytecode, to improve app boot times (pip does this by default).
+	# `--no-ansi`: Whilst we'd prefer to enable colour if possible, Poetry also emits ANSI escape
+	#              codes for redrawing lines, which renders badly in persisted build logs.
+	# shellcheck disable=SC2310 # This function is invoked in an 'if' condition so set -e will be disabled.
+	if ! {
+		"${poetry_install_command[@]}" --compile --no-ansi --no-interaction \
+			|& tee "${WARNINGS_LOG:?}" \
+			|& grep --invert-match 'Skipping virtualenv creation' \
+			|& output::indent
+	}; then
+		show-warnings
+
+		output::error <<-EOF
+			Error: Unable to install dependencies using Poetry.
+
+			See the log output above for more information.
+		EOF
+		meta_set "failure_reason" "install-dependencies::poetry"
+		return 1
+	fi
+}
@@ -0,0 +1 @@
+poetry==1.8.4
@@ -0,0 +1 @@
+3.12
@@ -0,0 +1,9 @@
+{
+  "environments": {
+    "test": {
+      "scripts": {
+        "test": "./bin/print-env-vars.sh && pytest --version"
+      }
+    }
+  }
+}
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# This file is run by the inline buildpack, and tests that the environment is
+# configured as expected for buildpacks that run after the Python buildpack.
+
+set -euo pipefail
+
+BUILD_DIR="${1}"
+
+cd "${BUILD_DIR}"
+
+exec bin/print-env-vars.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# This file is run by the inline buildpack.
+
+set -euo pipefail
+
+echo "Inline"
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+exec bin/print-env-vars.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+printenv | sort | grep -vE '^(_|BUILDPACK_LOG_FILE|BUILD_DIR|CACHE_DIR|CI_NODE_.+|DYNO|ENV_DIR|HEROKU_TEST_RUN_.+|HOME|OLDPWD|PORT|PWD|SHLVL|STACK|TERM)='
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,7 @@

		## [Unreleased]

		- Added support for the package manager Poetry. Apps must have a `pyproject.toml` + `poetry.lock` and no other package manager files (otherwise pip/Pipenv will take precedence for backwards compatibility). ([#1682](https://github.com/heroku/heroku-buildpack-python/pull/1682))

		## [v263] - 2024-10-31

Expand Down