Merge pull request pypa#110 from ehashman/issue-107

Support non-extension wheels with binary dependencies

.gitignore

@@ -62,3 +62,4 @@ target/
 # Generated by test script
 *.zip
 wheelhoust-*
+tests/testpackage/testpackage/testprogram

auditwheel/repair.py

@@ -41,6 +41,11 @@ def repair_wheel(wheel_path: str, abi: str, lib_sdir: str, out_dir: str,
                  update_tags: bool) -> Optional[str]:
 
     external_refs_by_fn = get_wheel_elfdata(wheel_path)[1]
+
+    # Do not repair a pure wheel, i.e. has no external refs
+    if not external_refs_by_fn:
+        return
+
     soname_map = {}  # type: Dict[str, str]
     if not isabs(out_dir):
         out_dir = abspath(out_dir)

auditwheel/wheel_abi.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 import logging
 import functools
@@ -25,6 +26,7 @@
 @functools.lru_cache()
 def get_wheel_elfdata(wheel_fn: str):
     full_elftree = {}
+    nonpy_elftree = {}
     full_external_refs = {}
     versioned_symbols = defaultdict(lambda: set())  # type: Dict[str, Set[str]]
     uses_ucs2_symbols = False
@@ -43,19 +45,44 @@ def get_wheel_elfdata(wheel_fn: str):
 
             log.info('processing: %s', fn)
             elftree = lddtree(fn)
-            full_elftree[fn] = elftree
-            if is_py_ext:
-                uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf)
+
             for key, value in elf_find_versioned_symbols(elf):
+                log.debug('key %s, value %s', key, value)
                 versioned_symbols[key].add(value)
 
+            # If the ELF is a Python extention, we definitely need to include
+            # its external dependencies.
             if is_py_ext:
+                full_elftree[fn] = elftree
+                uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf)
                 if py_ver == 2:
                     uses_ucs2_symbols |= any(
                         True for _ in elf_find_ucs2_symbols(elf))
-            full_external_refs[fn] = lddtree_external_references(elftree,
-                                                                 ctx.path)
-
+                full_external_refs[fn] = lddtree_external_references(elftree,
+                                                                     ctx.path)
+            else:
+                # If the ELF is not a Python extension, it might be included in
+                # the wheel already because auditwheel repair vendored it, so
+                # we will check whether we should include its internal
+                # references later.
+                nonpy_elftree[fn] = elftree
+
+        # Get a list of all external libraries needed by ELFs in the wheel.
+        needed_libs = {
+            lib
+            for elf in itertools.chain(full_elftree.values(),
+                                       nonpy_elftree.values())
+            for lib in elf['needed']
+        }
+
+        for fn in nonpy_elftree.keys():
+            # If a non-pyextension ELF file is not needed by something else
+            # inside the wheel, then it was not checked by the logic above and
+            # we should walk its elftree.
+            if basename(fn) not in needed_libs:
+                full_elftree[fn] = nonpy_elftree[fn]
+                full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn],
+                                                                     ctx.path)
 
     log.debug(json.dumps(full_elftree, indent=4))
 

auditwheel/wheeltools.py

@@ -185,11 +185,10 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
         platform tags to remove to the wheel filename and WHEEL tags, e.g.
         ``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added
     """
+    definitely_not_purelib = False
+
     info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL')
     info = read_pkg_info(info_fname)
-    if info['Root-Is-Purelib'] == 'true':
-        print('No need to add platforms to pure wheel - Skipping {}'.format(wheel_ctx.in_wheel))
-        return
 
     # Check what tags we have
     if wheel_ctx.out_wheel is not None:
@@ -203,11 +202,16 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
     fparts = parsed_fname.groupdict()
     original_fname_tags = fparts['plat'].split('.')
     print('Previous filename tags:', ', '.join(original_fname_tags))
-    fname_tags = [tag for tag in original_fname_tags
-                  if tag not in remove_platforms]
-    for platform in platforms:
-        if platform not in fname_tags:
-            fname_tags.append(platform)
+    fname_tags = {tag for tag in original_fname_tags
+                  if tag not in remove_platforms}
+    fname_tags |= set(platforms)
+
+    # Can't be 'any' and another platform
+    if 'any' in fname_tags and len(fname_tags) > 1:
+        fname_tags.remove('any')
+        remove_platforms.append('any')
+        definitely_not_purelib = True
+
     if fname_tags != original_fname_tags:
         print('New filename tags:', ', '.join(fname_tags))
     else:
@@ -232,11 +236,15 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()):
                      for tup in product(pyc_apis, remove_platforms)]
     updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags]
     updated_tags += new_tags
-    needs_write = updated_tags != in_info_tags
-    if needs_write:
+    if updated_tags != in_info_tags:
         del info['Tag']
         for tag in updated_tags:
             info.add_header('Tag', tag)
+
+        if definitely_not_purelib:
+            info['Root-Is-Purelib'] = 'False'
+            print('Changed wheel type to Platlib')
+
         print('New WHEEL info tags:', ', '.join(info.get_all('Tag')))
         write_pkg_info(info_fname, info)
     else:

tests/test_manylinux.py

@@ -16,7 +16,7 @@
         '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin')
 WHEEL_CACHE_FOLDER = op.expanduser('~/.cache/auditwheel_tests')
 ORIGINAL_NUMPY_WHEEL = 'numpy-1.11.0-cp35-cp35m-linux_x86_64.whl'
-ORIGINAL_TESTPACKAGE_WHEEL = 'testpackage-0.0.1-cp35-cp35m-linux_x86_64.whl'
+ORIGINAL_SIX_WHEEL = 'six-1.11.0-py2.py3-none-any.whl'
 
 
 def find_src_folder():
@@ -85,7 +85,7 @@ def docker_container():
             volumes={'/io': io_folder, '/auditwheel_src': src_folder},
             env_variables={'PATH': PATH})
         # Install the development version of auditwheel from source:
-        docker_exec(manylinux_id, 'pip install -U pip setuptools wheel')
+        docker_exec(manylinux_id, 'pip install -U pip setuptools')
         docker_exec(manylinux_id, 'pip install -U /auditwheel_src')
 
         # Launch a docker container with a more recent userland to check that
@@ -170,10 +170,10 @@ def test_build_wheel_with_binary_executable(docker_container):
     manylinux_id, python_id, io_folder = docker_container
     docker_exec(manylinux_id, 'yum install -y gsl-devel')
 
-    docker_exec(manylinux_id, 'cd /auditwheel_src/test/testpackage && python setup.py bdist_wheel -d /io')
+    docker_exec(manylinux_id, ['bash', '-c', 'cd /auditwheel_src/tests/testpackage && python setup.py bdist_wheel -d /io'])
 
     filenames = os.listdir(io_folder)
-    assert filenames == [ORIGINAL_TESTPACKAGE_WHEEL]
+    assert filenames == ['testpackage-0.0.1-py3-none-any.whl']
     orig_wheel = filenames[0]
     assert 'manylinux' not in orig_wheel
 
@@ -182,17 +182,52 @@ def test_build_wheel_with_binary_executable(docker_container):
     filenames = os.listdir(io_folder)
     assert len(filenames) == 2
     repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn]
-    assert repaired_wheels == ['testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl']
+    assert repaired_wheels == ['testpackage-0.0.1-py3-none-manylinux1_x86_64.whl']
     repaired_wheel = repaired_wheels[0]
     output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel)
     assert (
-        'testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl is consistent'
+        'testpackage-0.0.1-py3-none-manylinux1_x86_64.whl is consistent'
         ' with the following platform tag: "manylinux1_x86_64"'
     ) in output.replace('\n', ' ')
 
     # Check that the repaired numpy wheel can be installed and executed
     # on a modern linux image.
     docker_exec(python_id, 'pip install /io/' + repaired_wheel)
     output = docker_exec(
-        python_id, 'python -c "from testpackage import runit; print(runit(1.5))"').strip()
+        python_id, ['python', '-c', 'from testpackage import runit; print(runit(1.5))']).strip()
     assert output.strip() == '2.25'
+
+
+def test_build_repair_pure_wheel(docker_container):
+    manylinux_id, python_id, io_folder = docker_container
+
+    if op.exists(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)):
+        # If six has already been built and put in cache, let's reuse this.
+        shutil.copy2(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL),
+                     op.join(io_folder, ORIGINAL_SIX_WHEEL))
+    else:
+        docker_exec(manylinux_id,
+                    'pip wheel -w /io --no-binary=:all: six==1.11.0')
+        shutil.copy2(op.join(io_folder, ORIGINAL_SIX_WHEEL),
+                     op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL))
+
+    filenames = os.listdir(io_folder)
+    assert filenames == [ORIGINAL_SIX_WHEEL]
+    orig_wheel = filenames[0]
+    assert 'manylinux' not in orig_wheel
+
+    # Repair the wheel using the manylinux1 container
+    docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel)
+    filenames = os.listdir(io_folder)
+    assert len(filenames) == 1  # no new wheels
+    assert filenames == [ORIGINAL_SIX_WHEEL]
+
+    output = docker_exec(manylinux_id, 'auditwheel show /io/' + filenames[0])
+    assert ''.join([
+        ORIGINAL_SIX_WHEEL,
+        ' is consistent with the following platform tag: ',
+        '"manylinux1_x86_64".  ',
+        'The wheel references no external versioned symbols from system- ',
+        'provided shared libraries.  ',
+        'The wheel requires no external shared libraries! :)',
+    ]) in output.replace('\n', ' ')

tests/testpackage/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 import subprocess
 
-cmd = 'gcc testpackage/testprogram.c -lgsl -o testpackage/testprogram'
+cmd = 'gcc testpackage/testprogram.c -lgsl -lgslcblas -o testpackage/testprogram'
 subprocess.check_call(cmd.split())
 
 setup(

tests/testpackage/testpackage/__init__.py

@@ -5,4 +5,4 @@
 def runit(x):
     filename = pkg_resources.resource_filename(__name__, 'testprogram')
     output = subprocess.check_output([filename, str(x)])
-    return float(x)
+    return float(output)