diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers index a4a8c88a29c..70815c16da1 100644 --- a/build/bin/sage-dist-helpers +++ b/build/bin/sage-dist-helpers @@ -1,4 +1,4 @@ -# Shell functions for making spkg-install scripts a little easier to write, +# -*- shell-script -*- functions for making spkg-install scripts a little easier to write, # eliminating duplication. All Sage helper functions begin with sdh_ (for # Sage-distribution helper). Consult the below documentation for the list of # available helper functions. @@ -205,15 +205,51 @@ sdh_pip_install() { echo "Installing $PKG_NAME" if [ -n "$SAGE_DESTDIR" ]; then local sudo="" - local root="--root=$SAGE_DESTDIR" else local sudo="$SAGE_SUDO" - local root="" fi - $sudo sage-pip-install $root "$@" || \ - sdh_die "Error installing $PKG_NAME" + $sudo sage-pip-uninstall "$@" || \ + sdh_die "Error uninstalling a previous version of $PKG_NAME" + + mkdir -p dist + rm -f dist/*.whl + sage-python23 -m pip wheel --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \ + sdh_die "Error building a wheel for $PKG_NAME" + + sdh_store_and_pip_install_wheel . } +sdh_store_and_pip_install_wheel() { + if [ -n "$SAGE_DESTDIR" ]; then + local sudo="" + # --no-warn-script-location: Suppress a warning caused by --root + local root="--root=$SAGE_DESTDIR --no-warn-script-location" + else + local sudo="$SAGE_SUDO" + local root="" + fi + if [ "$*" != "." ]; then + sdh_die "Error: sdh_store_and_pip_install_wheel requires . as only argument" + fi + wheel="" + for w in dist/*.whl; do + if [ -n "$wheel" ]; then + sdh_die "Error: more than one wheel found after building" + fi + if [ -f "$w" ]; then + wheel="$w" + fi + done + if [ -z "$wheel" ]; then + sdh_die "Error: no wheel found after building" + fi + + $sudo sage-pip-install $root "$wheel" || \ + sdh_die "Error installing $wheel" + mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \ + $sudo mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \ + sdh_die "Error storing $wheel" +} sdh_cmake() { echo "Configuring $PKG_NAME with cmake" diff --git a/build/bin/sage-pip-install b/build/bin/sage-pip-install index 2ff9bd43d3e..077d8e0f58c 100755 --- a/build/bin/sage-pip-install +++ b/build/bin/sage-pip-install @@ -1,11 +1,6 @@ #!/usr/bin/env bash -# This command is specifically for pip-installing from a local -# source directory, as opposed to from a package index via package -# name. That is, it is for pip-installing Sage spkgs from their -# extracted upstream sources. -# -# This ensures that any previous installations of the same package -# are uninstalled first. +# This command is specifically for pip-installing a previously +# built wheel. # Default arguments for all packages installed with `pip install` # --ignore-installed : Force pip to re-install package even if it thinks it's @@ -17,81 +12,25 @@ # This also disables pip's version self-check. # --isolated : Don't read configuration files such as # ~/.pydistutils.cfg -# --no-build-isolation:Build the package in the usual Python environment -# (containing the dependencies) instead of an -# "isolated" environment -pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated --no-build-isolation" - -# Consume any additional pip install arguments except the last one -while [ $# -gt 1 ]; do - pip_install_flags="$pip_install_flags $1" - shift -done - -# Last argument must be "." and will be ignored -if [ "$1" != "." ]; then - echo >&2 "$0 requires . as final argument" - exit 1 -fi - +pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated" # Note: We need to take care to specify the full path to Sage's Python here -# to emphasize that this command hould use it, and not the system Python; +# to emphasize that this command should use it, and not the system Python; # see https://trac.sagemath.org/ticket/18438 # But now we delegate this to sage-python23. PYTHON=sage-python23 # The PIP variable is only used to determine the name of the lock file. -if [ "$SAGE_PYTHON3" = yes ]; then - PIP=pip3 -else - PIP=pip2 -fi +PIP=pip3 - -# Find out the name of the package that we are installing -name="$($PYTHON setup.py --name)" - -if [ $? -ne 0 ]; then - echo >&2 "Error: could not determine package name" - exit 1 -fi - -if [ $(echo "$name" | wc -l) -gt 1 ]; then - name="$(echo "$name" | tail -1)" - echo >&2 "Warning: This package has a badly-behaved setup.py which outputs" - echo >&2 "more than the package name for 'setup.py --name'; using the last" - echo >&2 "line as the package name: $name" -fi - - -# We should avoid running pip2/3 while uninstalling a package because that +# We should avoid running pip while installing a package because that # is prone to race conditions. Therefore, we use a lockfile while # running pip. This is implemented in the Python script sage-flock LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" -# Keep uninstalling as long as it succeeds -while true; do - out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1) - if [ $? -ne 0 ]; then - # Uninstall failed - echo >&2 "$out" - exit 1 - fi - - # Uninstall succeeded, which may mean that the package was not - # installed to begin with. - if [[ "$out" != *"not installed" ]]; then - break - fi -done - - # Finally actually do the installation (the "SHARED" tells pip2/3-lock # to apply a shared lock) -echo "Installing package $name using $PIP" - -sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags . +sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags "$@" if [ $? -ne 0 ]; then echo >&2 "Error: installing with $PIP failed" exit 3 diff --git a/build/bin/sage-pip-uninstall b/build/bin/sage-pip-uninstall new file mode 100755 index 00000000000..3326880cf78 --- /dev/null +++ b/build/bin/sage-pip-uninstall @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# This command ensures that any previous installations of the same package +# are uninstalled. + +# Only argument must be "." and will be ignored. +if [ $# -gt 1 ]; then + echo >&2 "$0 requires . as only argument" + exit 1 +fi +if [ "$1" != "." ]; then + echo >&2 "$0 requires . as final argument" + exit 1 +fi + +# Note: We need to take care to specify the full path to Sage's Python here +# to emphasize that this command should use it, and not the system Python; +# see https://trac.sagemath.org/ticket/18438 +# But now we delegate this to sage-python23. +PYTHON=sage-python23 + +# The PIP variable is only used to determine the name of the lock file. +PIP=pip3 + +# Find out the name of the package that we are installing +name="$($PYTHON setup.py --name)" + +if [ $? -ne 0 ]; then + echo >&2 "Error: could not determine package name" + exit 1 +fi + +if [ $(echo "$name" | wc -l) -gt 1 ]; then + name="$(echo "$name" | tail -1)" + echo >&2 "Warning: This package has a badly-behaved setup.py which outputs" + echo >&2 "more than the package name for 'setup.py --name'; using the last" + echo >&2 "line as the package name: $name" +fi + +# We should avoid running pip while uninstalling a package because that +# is prone to race conditions. Therefore, we use a lockfile while +# running pip. This is implemented in the Python script sage-flock +LOCK="$SAGE_LOCAL/var/lock/$PIP.lock" + +# Keep uninstalling as long as it succeeds +while true; do + out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1) + if [ $? -ne 0 ]; then + # Uninstall failed + echo >&2 "$out" + exit 1 + fi + + # Uninstall succeeded, which may mean that the package was not + # installed to begin with. + if [[ "$out" != *"not installed" ]]; then + break + fi +done diff --git a/build/bin/sage-spkg b/build/bin/sage-spkg index f66246ac151..26856940d4b 100755 --- a/build/bin/sage-spkg +++ b/build/bin/sage-spkg @@ -181,6 +181,9 @@ if [ -z "$SAGE_BUILD_DIR" ]; then export SAGE_BUILD_DIR="$SAGE_LOCAL/var/tmp/sage/build" fi +export SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed" +export SAGE_SPKG_SCRIPTS="$SAGE_LOCAL/var/lib/sage/scripts" +export SAGE_SPKG_WHEELS="$SAGE_LOCAL/var/lib/sage/wheels" # Remove '.' from PYTHONPATH, to avoid trouble with setuptools / easy_install # (cf. #10192, #10176): diff --git a/build/make/install b/build/make/install index 0d18185295f..93bf676fdb3 100755 --- a/build/make/install +++ b/build/make/install @@ -19,7 +19,6 @@ export SAGE_SHARE="$SAGE_LOCAL/share" export SAGE_PKGCONFIG="$SAGE_LOCAL/lib/pkgconfig" export SAGE_LOGS="$SAGE_ROOT/logs/pkgs" export SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed" -export SAGE_SPKG_SCRIPTS="$SAGE_LOCAL/var/lib/sage/scripts" if [ -z "${SAGE_ORIG_PATH_SET}" ]; then SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH diff --git a/build/pkgs/gambit/spkg-install.in b/build/pkgs/gambit/spkg-install.in index b09d901dbd1..9e2f7964129 100644 --- a/build/pkgs/gambit/spkg-install.in +++ b/build/pkgs/gambit/spkg-install.in @@ -11,8 +11,9 @@ cd src/python rm gambit/lib/libgambit.cpp # pip doesn't work (https://github.com/gambitproject/gambit/issues/207) -sage-python23 setup.py --no-user-cfg build install +sage-python23 setup.py --no-user-cfg bdist_wheel if [ $? -ne 0 ]; then echo "Error installing Python API" exit 1 fi +sdh_store_and_pip_install_wheel . diff --git a/build/pkgs/numpy/spkg-install.in b/build/pkgs/numpy/spkg-install.in index 8080a3115e0..c4343132954 100644 --- a/build/pkgs/numpy/spkg-install.in +++ b/build/pkgs/numpy/spkg-install.in @@ -23,7 +23,7 @@ export NUMPY_FCONFIG="config_fc --noopt --noarch" sage-python23 setup.py \ --no-user-cfg \ - install \ - --single-version-externally-managed \ - --root "$SAGE_DESTDIR" \ - ${NUMPY_FCONFIG} || sdh_die "Error building / installing numpy" + bdist_wheel \ + ${NUMPY_FCONFIG} || sdh_die "Error building wheel for numpy" + +sdh_store_and_pip_install_wheel . diff --git a/build/pkgs/pillow/spkg-install.in b/build/pkgs/pillow/spkg-install.in index 563c5869bd6..e01a3aee112 100644 --- a/build/pkgs/pillow/spkg-install.in +++ b/build/pkgs/pillow/spkg-install.in @@ -24,6 +24,6 @@ sage-python23 setup.py \ --debug \ --disable-jpeg \ $extra_build_ext \ - install \ - --single-version-externally-managed \ - --root "$SAGE_DESTDIR" || sdh_die "Error building/installing Pillow" + bdist_wheel || sdh_die "Error building/installing Pillow" + +sdh_store_and_pip_install_wheel . diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst index f3aaaea35de..dbde06c3f3b 100644 --- a/src/doc/en/developer/packaging.rst +++ b/src/doc/en/developer/packaging.rst @@ -372,12 +372,26 @@ begin with ``sdh_``, which stands for "Sage-distribution helper". arguments. If ``$SAGE_DESTDIR`` is not set then the command is run with ``$SAGE_SUDO``, if set. -- ``sdh_pip_install [...]``: Runs ``pip install`` with the given - arguments, as well as additional default arguments used for - installing packages into Sage with pip. Currently this is just a - wrapper around the ``sage-pip-install`` command. If - ``$SAGE_DESTDIR`` is not set then the command is run with - ``$SAGE_SUDO``, if set. +- ``sdh_pip_install [...]``: The equivalent of running ``pip install`` + with the given arguments, as well as additional default arguments used for + installing packages into Sage with pip. The last argument must be + ``.`` to indicate installation from the current directory. + + ``sdh_pip_install`` actually does the installation via ``pip wheel``, + creating a wheel file in ``dist/``, followed by + ``sdh_store_and_pip_install_wheel`` (see below). + +- ``sdh_store_and_pip_install_wheel .``: The current directory, + indicated by the required argument ``.``, must have a subdirectory + ``dist`` containing a unique wheel file (``*.whl``). + + This command (1) moves this wheel file to the + directory ``$SAGE_SPKG_WHEELS`` (``$SAGE_LOCAL/var/lib/sage/wheels``) + and then (2) installs the wheel in ``$SAGE_LOCAL``. + + Both of these steps, instead of writing directly into ``$SAGE_LOCAL``, + use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they + use ``$SAGE_SUDO`` (if set). - ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or directories given as ``SRC`` (recursively in the case of