Skip to content

Commit

Permalink
Trac #21535: Make src/setup.py respect --build-base and --inplace, in…
Browse files Browse the repository at this point in the history 
…dependent of SAGE_CYTHONIZED

This is a follow-up on #21480, which put `src/setup.py` in charge of all
`sagelib`building and removed dependencies on various environment
variables, as a step towards the goal of making `sagelib` an ordinary
Python package (#21507).

However, there is still a dependency on `$SAGE_CYTHONIZED`, which needs
to be set in a way that matches the build-base (set by `setup.py build
--build-base`).

This dependency should be removed.

Also, `--inplace` should be supported.
(See also: #12659: build the sage library in place)

See also #21508 on other cleanup issues of `src/setup.py`.

URL: https://trac.sagemath.org/21535
Reported by: mkoeppe
Ticket author(s): Jeroen Demeyer
Reviewer(s): Matthias Koeppe
  • Loading branch information
@vbraun
Release Manager authored and vbraun committed Sep 10, 2017
2 parents 99a0951 + bd91b43 commit c7b2d40
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 84 deletions.
2 changes: 0 additions & 2 deletions src/Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ all: sage
## In this way we make sure that all of the sagelib build's source paths are communicated through
## the current directory (for the source tree).
## Building takes places in the build/ subdirectory.
## The environment variable SAGE_CYTHONIZED needs to point to build/cythonized.
##
## As a special exception, we feed SAGE_PKGS. This is needed by src/sage_setup/optional_extension.py
## via src/sage/misc/package.py. Hoping that #20382 will make this unnecessary.
Expand All @@ -38,7 +37,6 @@ sage:
SAGE_DOC_SRC=/doesnotexist \
SAGE_BUILD_DIR=/doesnotexist \
SAGE_PKGS=$(abs_top_srcdir)/build/pkgs \
SAGE_CYTHONIZED=$(abs_builddir)/build/cythonized \
&& sage-python23 -u setup.py --no-user-cfg build install
if [ "$$UNAME" = "CYGWIN" ]; then \
sage-rebase.sh "$$SAGE_LOCAL" 2>/dev/null; \
Expand Down
2 changes: 0 additions & 2 deletions src/sage/env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,6 @@ def _add_variable_or_fallback(key, fallback, force=False):

_add_variable_or_fallback('SAGE_LIB', SITE_PACKAGES[0])

_add_variable_or_fallback('SAGE_CYTHONIZED', opj('$SAGE_ROOT', 'src', 'build', 'cythonized'))

# Used by sage/misc/package.py. Should be SAGE_SRC_ROOT in VPATH.
_add_variable_or_fallback('SAGE_PKGS', opj('$SAGE_ROOT', 'build', 'pkgs'))

Expand Down
5 changes: 3 additions & 2 deletions src/sage_setup/clean.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,13 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module
course. We check that when the doctest is being run, that is,
after installation, there are no stale files::
sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_CYTHONIZED
sage: from sage.env import SAGE_SRC, SAGE_LIB
sage: cythonized_dir = os.path.join(SAGE_SRC, "build", "cythonized")
sage: from sage_setup.find import find_python_sources, find_extra_files
sage: python_packages, python_modules = find_python_sources(
....: SAGE_SRC, ['sage', 'sage_setup'])
sage: extra_files = find_extra_files(python_packages, SAGE_SRC,
....: SAGE_CYTHONIZED, ["ntlwrap.cpp"])
....: cythonized_dir, ["ntlwrap.cpp"])
sage: from sage_setup.clean import _find_stale_files
TODO: move ``module_list.py`` into ``sage_setup`` and also check
Expand Down
5 changes: 3 additions & 2 deletions src/sage_setup/find.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ def find_extra_files(packages, src_dir, cythonized_dir, special_filenames=[]):
EXAMPLES::
sage: from sage_setup.find import find_extra_files
sage: from sage.env import SAGE_SRC, SAGE_CYTHONIZED
sage: find_extra_files(["sage.ext.interpreters"], SAGE_SRC, SAGE_CYTHONIZED)
sage: from sage.env import SAGE_SRC
sage: cythonized_dir = os.path.join(SAGE_SRC, "build", "cythonized")
sage: find_extra_files(["sage.ext.interpreters"], SAGE_SRC, cythonized_dir)
[('sage/ext/interpreters',
['.../src/sage/ext/interpreters/wrapper_cdf.pxd', ...wrapper_cdf.h...])]
"""
Expand Down
148 changes: 72 additions & 76 deletions src/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
from distutils.errors import (DistutilsSetupError, DistutilsModuleError,
DistutilsOptionError)

# This import allows instancemethods to be pickable
import fpickle_setup


def excepthook(*exc):
"""
Expand Down Expand Up @@ -46,18 +49,13 @@ def excepthook(*exc):
sys.excepthook = excepthook


#########################################################
### Check build-base
#########################################################

build_base = 'build' # the distutils default. Changing it is not supported by this setup.sh.

#########################################################
### Set source directory
#########################################################

import sage.env
sage.env.SAGE_SRC = os.getcwd()
from sage.env import *

#########################################################
### List of Extensions
Expand All @@ -67,7 +65,6 @@ def excepthook(*exc):
#########################################################

from module_list import ext_modules, library_order, aliases
from sage.env import *
from sage_setup.find import find_extra_files

#########################################################
Expand All @@ -88,7 +85,6 @@ def excepthook(*exc):

# search for dependencies and add to gcc -I<path>
include_dirs = sage_include_directories(use_sources=True)
include_dirs.append(SAGE_CYTHONIZED)

# Look for libraries in $SAGE_LOCAL/lib
library_dirs = [os.path.join(SAGE_LOCAL, "lib")]
Expand Down Expand Up @@ -196,67 +192,6 @@ def __call__(self, *args):
}


def sage_create_extension(template, kwds):
"""
Create a distutils Extension given data from Cython
This adjust the ``kwds`` in the following ways:
- Make everything depend on *this* setup.py file
- Add dependencies on header files for certain libraries
- Ensure that C++ extensions link with -lstdc++
- Sort the libraries according to the library order
- Add some default compile/link args and directories
- Drop -std=c99 and similar from C++ extensions
- Ensure that each flag, library, ... is listed at most once
"""
lang = kwds.get('language', 'c')

# Libraries: add stdc++ if needed and sort them
libs = kwds.get('libraries', [])
if lang == 'c++':
libs = libs + ['stdc++']
kwds['libraries'] = sorted(set(libs),
key=lambda lib: library_order.get(lib, 0))

# Dependencies: add setup.py and lib_headers
depends = kwds.get('depends', []) + [__file__]
for lib, headers in lib_headers.items():
if lib in libs:
depends += headers
kwds['depends'] = depends # These are sorted and uniq'ed by Cython

# Process extra_compile_args
cflags = []
for flag in kwds.get('extra_compile_args', []):
if lang == "c++":
if flag.startswith("-std=") and "++" not in flag:
continue # Skip -std=c99 and similar for C++
cflags.append(flag)
cflags = extra_compile_args + cflags
kwds['extra_compile_args'] = stable_uniq(cflags)

# Process extra_link_args
ldflags = kwds.get('extra_link_args', []) + extra_link_args
kwds['extra_link_args'] = stable_uniq(ldflags)

# Process library_dirs
lib_dirs = kwds.get('library_dirs', []) + library_dirs
kwds['library_dirs'] = stable_uniq(lib_dirs)

# Process include_dirs
inc_dirs = kwds.get('include_dirs', []) + include_dirs
kwds['include_dirs'] = stable_uniq(inc_dirs)

return default_create_extension(template, kwds)


class sage_build_cython(Command):
name = 'build_cython'
description = "compile Cython extensions into C/C++ extensions"
Expand All @@ -278,6 +213,7 @@ class sage_build_cython(Command):

def initialize_options(self):
self.extensions = None
self.build_base = None
self.build_dir = None

# Always have Cython produce debugging info by default, unless
Expand All @@ -295,9 +231,10 @@ def initialize_options(self):
def finalize_options(self):
self.extensions = self.distribution.ext_modules

# TODO: Could get source path for Cythonized files from the build
# command, rather than relying solely on SAGE_CYTHONIZED
self.build_dir = SAGE_CYTHONIZED
# Let Cython generate its files in the "cythonized"
# subdirectory of the build_base directory.
self.set_undefined_options('build', ('build_base', 'build_base'))
self.build_dir = os.path.join(self.build_base, "cythonized")

# Inherit some options from the 'build_ext' command if possible
# (this in turn implies inheritance from the 'build' command)
Expand Down Expand Up @@ -398,7 +335,7 @@ def get_cythonized_package_files(self):

dist = self.distribution
self.cythonized_files = find_extra_files(dist.packages,
".", SAGE_CYTHONIZED, ["ntlwrap.cpp"])
".", self.build_dir, ["ntlwrap.cpp"])

return self.cythonized_files

Expand All @@ -424,7 +361,7 @@ def run(self):
aliases=aliases,
compiler_directives=self.cython_directives,
compile_time_env={'PY_VERSION_HEX':sys.hexversion},
create_extension=sage_create_extension,
create_extension=self.create_extension,
# Debugging
gdb_debug=self.debug,
output_dir=self.build_dir,
Expand All @@ -438,14 +375,74 @@ def run(self):
with open(self._version_file, 'w') as f:
f.write(self._version_stamp)

# Finally, copy relevant cythonized files from the SAGE_CYTHONIZED
# Finally, copy relevant cythonized files from build/cythonized
# tree into the build-lib tree
for (dst_dir, src_files) in self.get_cythonized_package_files():
dst = os.path.join(self.build_lib, dst_dir)
self.mkpath(dst)
for src in src_files:
self.copy_file(src, dst, preserve_mode=False)

def create_extension(self, template, kwds):
"""
Create a distutils Extension given data from Cython.
This adjust the ``kwds`` in the following ways:
- Make everything depend on *this* setup.py file
- Add dependencies on header files for certain libraries
- Ensure that C++ extensions link with -lstdc++
- Sort the libraries according to the library order
- Add some default compile/link args and directories
- Drop -std=c99 and similar from C++ extensions
- Ensure that each flag, library, ... is listed at most once
"""
lang = kwds.get('language', 'c')

# Libraries: add stdc++ if needed and sort them
libs = kwds.get('libraries', [])
if lang == 'c++':
libs = libs + ['stdc++']
kwds['libraries'] = sorted(set(libs),
key=lambda lib: library_order.get(lib, 0))

# Dependencies: add setup.py and lib_headers
depends = kwds.get('depends', []) + [__file__]
for lib, headers in lib_headers.items():
if lib in libs:
depends += headers
kwds['depends'] = depends # These are sorted and uniq'ed by Cython

# Process extra_compile_args
cflags = []
for flag in kwds.get('extra_compile_args', []):
if lang == "c++":
if flag.startswith("-std=") and "++" not in flag:
continue # Skip -std=c99 and similar for C++
cflags.append(flag)
cflags = extra_compile_args + cflags
kwds['extra_compile_args'] = stable_uniq(cflags)

# Process extra_link_args
ldflags = kwds.get('extra_link_args', []) + extra_link_args
kwds['extra_link_args'] = stable_uniq(ldflags)

# Process library_dirs
lib_dirs = kwds.get('library_dirs', []) + library_dirs
kwds['library_dirs'] = stable_uniq(lib_dirs)

# Process include_dirs
inc_dirs = kwds.get('include_dirs', []) + include_dirs + [self.build_dir]
kwds['include_dirs'] = stable_uniq(inc_dirs)

return default_create_extension(template, kwds)


########################################################################
##
Expand Down Expand Up @@ -517,7 +514,6 @@ def execute_list_of_commands_in_parallel(command_list, nthreads):
command_list[i] = command_list[i] + (progress,)

from multiprocessing import Pool
import fpickle_setup #doing this import will allow instancemethods to be pickable
# map_async handles KeyboardInterrupt correctly if an argument is
# given to get(). Plain map() and apply_async() do not work
# correctly, see Trac #16113.
Expand Down

0 comments on commit c7b2d40

Please sign in to comment.