Trac #26018: Race conditions because of DESTDIR

Using a staging directory for package install can introduce race
conditions. For example, when installing a Python package, the package
is installed first and then the `egg-info` directory is installed. The
latter is what `pkg_resources` uses to consider a package "installed".

However, when a `DESTDIR` install is used, the order that files appear
in the final installation directory might be different.

If the `egg-info` directory is copied to `site-packages` before the
actual Python package, then this can happen for example:
{{{
Installing alabaster-0.7.10
Installing package alabaster using pip2
Ignoring indexes: https://pypi.python.org/simple
Processing /home/jdemeyer/sage-
test/local/var/tmp/sage/build/alabaster-0.7.10/src
  Running setup.py (path:/tmp/pip-0ac_4C-build/setup.py) egg_info for
package from file:///home/jdemeyer/sage-
test/local/var/tmp/sage/build/alabaster-0.7.10/src
    Running command python setup.py egg_info
    running egg_info
    creating pip-egg-info/alabaster.egg-info
    writing pip-egg-info/alabaster.egg-info/PKG-INFO
    writing top-level names to pip-egg-info/alabaster.egg-
info/top_level.txt
    writing dependency_links to pip-egg-info/alabaster.egg-
info/dependency_links.txt
    writing manifest file 'pip-egg-info/alabaster.egg-info/SOURCES.txt'
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/tmp/pip-0ac_4C-build/setup.py", line 37, in <module>
        'Programming Language :: Python :: 3.4',
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/__init__.py", line 129, in setup
        return distutils.core.setup(**attrs)
      File "/home/jdemeyer/sage-
test/local/lib/python2.7/distutils/core.py", line 151, in setup
        dist.run_commands()
      File "/home/jdemeyer/sage-
test/local/lib/python2.7/distutils/dist.py", line 953, in run_commands
        self.run_command(cmd)
      File "/home/jdemeyer/sage-
test/local/lib/python2.7/distutils/dist.py", line 972, in run_command
        cmd_obj.run()
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/command/egg_info.py", line 278, in run
        self.find_sources()
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/command/egg_info.py", line 293, in find_sources
        mm.run()
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/command/egg_info.py", line 524, in run
        self.add_defaults()
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/command/egg_info.py", line 563, in add_defaults
        rcfiles = list(walk_revctrl())
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools/command/sdist.py", line 20, in walk_revctrl
        for item in ep.load()(dirname):
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/pkg_resources/__init__.py", line 2405, in load
        return self.resolve()
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/pkg_resources/__init__.py", line 2411, in resolve
        module = __import__(self.module_name, fromlist=['__name__'],
level=0)
      File "/home/jdemeyer/sage-test/local/lib/python2.7/site-
packages/setuptools_scm/__init__.py", line 9, in <module>
        from .utils import trace
    ImportError: No module named utils
}}}
What you see here is that `setuptools_scm` is used by `pkg_resources`
even though the package `setuptools_scm` has not fully been installed.

See also: #26160 (py3: race condition with .pth files)

URL: https://trac.sagemath.org/26018
Reported by: jdemeyer
Ticket author(s): Erik Bray, John Palmieri
Reviewer(s): John Palmieri, Matthias Koeppe

build/make/deps

@@ -103,6 +103,11 @@ toolchain-deps:
 all-toolchain: base-toolchain
 	$(MAKE) toolchain-deps
 
+# All packages needed as a prerequisite to install other Python packages with
+# pip or which are otherwise used by the Python build tools; these should be
+# given as a prerequisite to any pip-installed packages
+PYTHON_TOOLCHAIN = setuptools pip setuptools_scm future
+
 # Everything needed to start up Sage using "./sage".  Of course, not
 # every part of Sage will work.  It does not include Maxima for example.
 SAGERUNTIME = sagelib $(SCRIPTS) $(inst_ipython) $(inst_pexpect) \

build/pkgs/alabaster/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/appnope/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/babel/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip pytz
+$(PYTHON) | $(PYTHON_TOOLCHAIN) pytz
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/backports_abc/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/backports_functools_lru_cache/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools_scm pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/backports_shutil_get_terminal_size/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/backports_ssl_match_hostname/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/bleach/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/certifi/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/configparser/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/cvxopt/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) numpy $(BLAS) gsl glpk suitesparse | pkgconfig pip nose matplotlib
+$(PYTHON) numpy $(BLAS) gsl glpk suitesparse | $(PYTHON_TOOLCHAIN) pkgconfig nose matplotlib
 
 matplotlib is needed to test cvxopt (i.e., if SAGE_CHECK=yes). See #12742.
 

build/pkgs/cycler/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) six | setuptools pip
+$(PYTHON) six | $(PYTHON_TOOLCHAIN)
 ----------
 All lines of this file are ignored except the first.
 It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/cypari/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cython pari cysignals | pip
+$(PYTHON) cython pari cysignals | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/cysignals/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cython pari | pip
+$(PYTHON) cython pari | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/cython/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/dateutil/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) six | setuptools setuptools_scm pip
+$(PYTHON) six | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/decorator/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/defusedxml/dependencies

@@ -1 +1,5 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/docutils/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/dot2tex/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip pyparsing
+$(PYTHON) | $(PYTHON_TOOLCHAIN) pyparsing
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/entrypoints/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip configparser
+$(PYTHON) | $(PYTHON_TOOLCHAIN) configparser
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/enum34/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip werkzeug jinja2 itsdangerous
+$(PYTHON) | $(PYTHON_TOOLCHAIN) werkzeug jinja2 itsdangerous
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask_autoindex/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip flask_silk future
+$(PYTHON) | $(PYTHON_TOOLCHAIN) flask_silk
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask_babel/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip flask speaklater babel
+$(PYTHON) | $(PYTHON_TOOLCHAIN) flask speaklater babel
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask_oldsessions/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask_openid/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip flask python_openid
+$(PYTHON) | $(PYTHON_TOOLCHAIN) flask python_openid
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/flask_silk/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip flask
+$(PYTHON) | $(PYTHON_TOOLCHAIN) flask
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/functools32/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/future/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | setuptools pip setuptools_scm
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/gambit/dependencies

@@ -1,4 +1,4 @@
-cython | setuptools
+cython | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/gap_jupyter/dependencies

@@ -1 +1 @@
-$(PYTHON) | pip ipython gap
+$(PYTHON) | $(PYTHON_TOOLCHAIN) ipython gap

build/pkgs/giacpy_sage/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cython giac cysignals | $(SAGERUNTIME) pip sympy maxima
+$(PYTHON) cython giac cysignals | $(SAGERUNTIME) $(PYTHON_TOOLCHAIN) sympy maxima
 
 spkg-check uses the sage command. The doctest suite (SAGE_CHECK=yes)  also uses Sympy and Maxima
 ----------

build/pkgs/git_trac/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/gmpy2/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) $(MP_LIBRARY) mpfr mpc | pip
+$(PYTHON) $(MP_LIBRARY) mpfr mpc | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/html5lib/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/imagesize/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/ipaddress/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/ipykernel/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | ipython jupyter_client pip scandir
+$(PYTHON) | ipython jupyter_client $(PYTHON_TOOLCHAIN) scandir
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/ipython/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) jinja2 tornado pyzmq pickleshare simplegeneric traitlets | pip backports_shutil_get_terminal_size wcwidth prompt_toolkit pygments pexpect appnope
+$(PYTHON) jinja2 tornado pyzmq pickleshare simplegeneric traitlets | $(PYTHON_TOOLCHAIN) backports_shutil_get_terminal_size wcwidth prompt_toolkit pygments pexpect appnope
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/ipython_genutils/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/ipywidgets/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) widgetsnbextension | pip
+$(PYTHON) widgetsnbextension | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/itsdangerous/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/jinja2/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) markupsafe docutils | setuptools pip
+$(PYTHON) markupsafe docutils | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/jsonschema/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) vcversioner functools32 | setuptools pip
+$(PYTHON) vcversioner functools32 | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/jupymake/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) polymake | pip
+$(PYTHON) polymake | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/jupyter_client/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) jupyter_core | pip pyzmq dateutil
+$(PYTHON) jupyter_core | $(PYTHON_TOOLCHAIN) pyzmq dateutil
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/jupyter_core/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip traitlets
+$(PYTHON) | $(PYTHON_TOOLCHAIN) traitlets
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/kiwisolver/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/markupsafe/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | setuptools pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/matplotlib/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) numpy freetype libpng dateutil pyparsing tornado six cycler | setuptools pip pytz functools32 backports_functools_lru_cache subprocess32 kiwisolver
+$(PYTHON) numpy freetype libpng dateutil pyparsing tornado six cycler | $(PYTHON_TOOLCHAIN) pytz functools32 backports_functools_lru_cache subprocess32 kiwisolver
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/mistune/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) cython | setuptools pip
+$(PYTHON) cython | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/mpmath/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/nbconvert/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip mistune jinja2 pygments traitlets jupyter_core nbformat entrypoints bleach pandocfilters testpath defusedxml
+$(PYTHON) | $(PYTHON_TOOLCHAIN) mistune jinja2 pygments traitlets jupyter_core nbformat entrypoints bleach pandocfilters testpath defusedxml
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/nbformat/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip jsonschema
+$(PYTHON) | $(PYTHON_TOOLCHAIN) jsonschema
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/networkx/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) decorator | pip nose scipy
+$(PYTHON) decorator | $(PYTHON_TOOLCHAIN) nose scipy
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/nose/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/notebook/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip ipython jupyter_client ipykernel nbconvert nbformat jinja2 tornado terminado send2trash ipaddress prometheus_client
+$(PYTHON) | $(PYTHON_TOOLCHAIN) ipython jupyter_client ipykernel nbconvert nbformat jinja2 tornado terminado send2trash ipaddress prometheus_client
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/notedown/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) setuptools | pip nbformat nbconvert six pandoc_attributes
+$(PYTHON) $(PYTHON_TOOLCHAIN) | pip nbformat nbconvert six pandoc_attributes
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/numpy/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) $(BLAS) gfortran | setuptools pip pkgconfig cython
+$(PYTHON) $(BLAS) gfortran | $(PYTHON_TOOLCHAIN) pkgconfig cython
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/p_group_cohomology/dependencies

@@ -1 +1 @@
-$(PYTHON) cython cysignals singular meataxe $(SAGE_SRC)/sage/matrix/matrix_gfpn_dense.pxd $(SAGE_SRC)/sage/structure/element.pxd $(SAGE_SRC)/sage/matrix/matrix_gfpn_dense.pxd $(SAGE_SRC)/sage/matrix/matrix0.pxd $(SAGE_SRC)/sage/libs/meataxe.pxd $(SAGE_SRC)/sage/rings/morphism.pxd | pip matplotlib gap xz $(SAGERUNTIME)
+$(PYTHON) cython cysignals singular meataxe $(SAGE_SRC)/sage/matrix/matrix_gfpn_dense.pxd $(SAGE_SRC)/sage/structure/element.pxd $(SAGE_SRC)/sage/matrix/matrix_gfpn_dense.pxd $(SAGE_SRC)/sage/matrix/matrix0.pxd $(SAGE_SRC)/sage/libs/meataxe.pxd $(SAGE_SRC)/sage/rings/morphism.pxd | $(PYTHON_TOOLCHAIN) matplotlib gap xz $(SAGERUNTIME)

build/pkgs/packaging/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip pyparsing six
+$(PYTHON) | $(PYTHON_TOOLCHAIN) pyparsing six
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/pandoc_attributes/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) setuptools | pip pandocfilters
+$(PYTHON) $(PYTHON_TOOLCHAIN) | pip pandocfilters
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/pandocfilters/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) | pip
+$(PYTHON) | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/pari_jupyter/dependencies

@@ -1 +1 @@
-$(PYTHON) pari | pip cython notebook jupyter_core
+$(PYTHON) pari | $(PYTHON_TOOLCHAIN) cython notebook jupyter_core