diff --git a/tests/pr134/hello_module/.gitignore b/tests/pr134/hello_module/.gitignore new file mode 100644 index 00000000..414521ef --- /dev/null +++ b/tests/pr134/hello_module/.gitignore @@ -0,0 +1,168 @@ + +# Created by https://www.gitignore.io/api/python,linux,macos +# Edit at https://www.gitignore.io/?templates=python,linux,macos + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +### Python Patch ### +.venv/ + +# End of https://www.gitignore.io/api/python,linux,macos diff --git a/tests/pr134/hello_module/README.md b/tests/pr134/hello_module/README.md new file mode 100755 index 00000000..c9c1c965 --- /dev/null +++ b/tests/pr134/hello_module/README.md @@ -0,0 +1,4 @@ +# Python 3 extension example + +This example was inspired from https://gist.github.com/physacco/2e1b52415f3a964ad2a542a99bebed8f + diff --git a/tests/pr134/hello_module/extensions/.gitignore b/tests/pr134/hello_module/extensions/.gitignore new file mode 100644 index 00000000..451dfd06 --- /dev/null +++ b/tests/pr134/hello_module/extensions/.gitignore @@ -0,0 +1 @@ +testzlib diff --git a/tests/pr134/hello_module/extensions/testzlib.cpp b/tests/pr134/hello_module/extensions/testzlib.cpp new file mode 100644 index 00000000..88d91df1 --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.cpp @@ -0,0 +1,137 @@ +// Copyright 2007 Timo Bingmann +// Distributed under the Boost Software License, Version 1.0. +// (See http://www.boost.org/LICENSE_1_0.txt) +// Taken from https://panthema.net/2007/0328-ZLibString.html + +#include +#include +#include +#include +#include + +#include +#include "testzlib.h" + +/** Compress a STL string using zlib with given compression level and return + * the binary data. */ +std::string compress_string(const std::string& str, + int compressionlevel) +{ + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + if (deflateInit(&zs, compressionlevel) != Z_OK) + throw(std::runtime_error("deflateInit failed while compressing.")); + + zs.next_in = (Bytef*)str.data(); + zs.avail_in = str.size(); // set the z_stream's input + + int ret; + char outbuffer[32768]; + std::string outstring; + + // retrieve the compressed bytes blockwise + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = deflate(&zs, Z_FINISH); + + if (outstring.size() < zs.total_out) { + // append the block to the output string + outstring.append(outbuffer, + zs.total_out - outstring.size()); + } + } while (ret == Z_OK); + + deflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream oss; + oss << "Exception during zlib compression: (" << ret << ") " << zs.msg; + throw(std::runtime_error(oss.str())); + } + + return outstring; +} + +/** Decompress an STL string using zlib and return the original data. */ +std::string decompress_string(const std::string& str) +{ + z_stream zs; // z_stream is zlib's control structure + memset(&zs, 0, sizeof(zs)); + + if (inflateInit(&zs) != Z_OK) + throw(std::runtime_error("inflateInit failed while decompressing.")); + + zs.next_in = (Bytef*)str.data(); + zs.avail_in = str.size(); + + int ret; + char outbuffer[32768]; + std::string outstring; + + // get the decompressed bytes blockwise using repeated calls to inflate + do { + zs.next_out = reinterpret_cast(outbuffer); + zs.avail_out = sizeof(outbuffer); + + ret = inflate(&zs, 0); + + if (outstring.size() < zs.total_out) { + outstring.append(outbuffer, + zs.total_out - outstring.size()); + } + + } while (ret == Z_OK); + + inflateEnd(&zs); + + if (ret != Z_STREAM_END) { // an error occurred that was not EOF + std::ostringstream oss; + oss << "Exception during zlib decompression: (" << ret << ") " + << zs.msg; + throw(std::runtime_error(oss.str())); + } + + return outstring; +} + +/** Small dumb tool (de)compressing cin to cout. It holds all input in memory, + * so don't use it for huge files. */ +int main(int argc, char* argv[]) +{ + std::string allinput; + + while (std::cin.good()) // read all input from cin + { + char inbuffer[32768]; + std::cin.read(inbuffer, sizeof(inbuffer)); + allinput.append(inbuffer, std::cin.gcount()); + } + + if (argc >= 2 && strcmp(argv[1], "-d") == 0) + { + std::string cstr = decompress_string( allinput ); + + std::cerr << "Inflated data: " + << allinput.size() << " -> " << cstr.size() + << " (" << std::setprecision(1) << std::fixed + << ( ((float)cstr.size() / (float)allinput.size() - 1.0) * 100.0 ) + << "% increase).\n"; + + std::cout << cstr; + } + else + { + std::string cstr = compress_string( allinput ); + + std::cerr << "Deflated data: " + << allinput.size() << " -> " << cstr.size() + << " (" << std::setprecision(1) << std::fixed + << ( (1.0 - (float)cstr.size() / (float)allinput.size()) * 100.0) + << "% saved).\n"; + + std::cout << cstr; + } +} diff --git a/tests/pr134/hello_module/extensions/testzlib.h b/tests/pr134/hello_module/extensions/testzlib.h new file mode 100644 index 00000000..2bfb62b9 --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.h @@ -0,0 +1,11 @@ +#include +#include + +#ifndef ZLIB_EXAMPLE // include guard +#define ZLIB_EXAMPLE + +std::string compress_string(const std::string& str, + int compressionlevel = Z_BEST_COMPRESSION); +std::string decompress_string(const std::string& str); + +#endif /* ZLIB_EXAMPLE */ diff --git a/tests/pr134/hello_module/extensions/testzlib.sh b/tests/pr134/hello_module/extensions/testzlib.sh new file mode 100644 index 00000000..dae26bde --- /dev/null +++ b/tests/pr134/hello_module/extensions/testzlib.sh @@ -0,0 +1,8 @@ +# compile and run +g++ testzlib.cpp -lz -o testzlib +if [ $? == 0 ]; then + echo Hello Hello Hello Hello Hello Hello! | ./testzlib | ./testzlib -d +fi +# Deflated data: 37 -> 19 (48.6% saved). +# Inflated data: 19 -> 37 (94.7% increase). +# Hello Hello Hello Hello Hello Hello! diff --git a/tests/pr134/hello_module/hello.cpp b/tests/pr134/hello_module/hello.cpp new file mode 100755 index 00000000..c080f7c8 --- /dev/null +++ b/tests/pr134/hello_module/hello.cpp @@ -0,0 +1,96 @@ +#include +#include "extensions/testzlib.h" + +// Module method definitions +static PyObject* hello_world(PyObject *self, PyObject *args) { + printf("Hello, World!"); + Py_RETURN_NONE; +} + +// static PyObject* zlib_example(PyObject *self, PyObject *args) { +// main(); +// Py_RETURN_NONE; +// } + +static PyObject* z_compress(PyObject *self, PyObject *args) { + const char* str_compress; + if (!PyArg_ParseTuple(args, "s", &str_compress)) { + return NULL; + } + + std::string str_compress_s = str_compress; + std::string compressed = compress_string(str_compress_s); + // Copy pointer (compressed string may contain 0 byte) + const char * str_compressed = &*compressed.begin(); + return PyBytes_FromStringAndSize(str_compressed, compressed.length()); +} + +static PyObject* z_uncompress(PyObject *self, PyObject *args) { + const char * str_uncompress; + int str_uncompress_len; + // according to https://docs.python.org/3/c-api/arg.html + if (!PyArg_ParseTuple(args, "y#", &str_uncompress, &str_uncompress_len)) { + return NULL; + } + + std::string uncompressed = decompress_string(std::string (str_uncompress, str_uncompress_len)); + + return PyUnicode_FromString(uncompressed.c_str()); +} + +static PyObject* hello(PyObject *self, PyObject *args) { + const char* name; + if (!PyArg_ParseTuple(args, "s", &name)) { + return NULL; + } + + printf("Hello, %s!\n", name); + Py_RETURN_NONE; +} + +// Method definition object for this extension, these argumens mean: +// ml_name: The name of the method +// ml_meth: Function pointer to the method implementation +// ml_flags: Flags indicating special features of this method, such as +// accepting arguments, accepting keyword arguments, being a +// class method, or being a static method of a class. +// ml_doc: Contents of this method's docstring +static PyMethodDef hello_methods[] = { + { + "hello_world", hello_world, METH_NOARGS, + "Print 'hello world' from a method defined in a C extension." + }, + { + "hello", hello, METH_VARARGS, + "Print 'hello xxx' from a method defined in a C extension." + }, + { + "z_compress", z_compress, METH_VARARGS, + "Compresses a string using C's libz.so" + }, + { + "z_uncompress", z_uncompress, METH_VARARGS, + "Unompresses a string using C's libz.so" + }, + {NULL, NULL, 0, NULL} +}; + +// Module definition +// The arguments of this structure tell Python what to call your extension, +// what it's methods are and where to look for it's method definitions +static struct PyModuleDef hello_definition = { + PyModuleDef_HEAD_INIT, + "_hello", + "A Python module that prints 'hello world' from C code.", + -1, + hello_methods +}; + +// Module initialization +// Python calls this function when importing your extension. It is important +// that this function is named PyInit_[[your_module_name]] exactly, and matches +// the name keyword argument in setup.py's setup() call. +PyMODINIT_FUNC PyInit__hello(void) { + Py_Initialize(); + return PyModule_Create(&hello_definition); +} diff --git a/tests/pr134/hello_module/hello/__init__.py b/tests/pr134/hello_module/hello/__init__.py new file mode 100644 index 00000000..ef20328b --- /dev/null +++ b/tests/pr134/hello_module/hello/__init__.py @@ -0,0 +1 @@ +from ._hello import z_compress, z_uncompress diff --git a/tests/pr134/hello_module/setup.py b/tests/pr134/hello_module/setup.py new file mode 100755 index 00000000..1d3adae1 --- /dev/null +++ b/tests/pr134/hello_module/setup.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import platform +import setuptools.command.build_ext +from setuptools import setup, find_packages, Distribution +from setuptools.extension import Extension, Library +import os + +# despite its name, setuptools.command.build_ext.link_shared_object won't +# link a shared object on Linux, but a static library and patches distutils +# for this ... We're patching this back now. + + +def always_link_shared_object( + self, objects, output_libname, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, export_symbols=None, + debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, + target_lang=None): + self.link( + self.SHARED_LIBRARY, objects, output_libname, + output_dir, libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, extra_preargs, extra_postargs, + build_temp, target_lang + ) + + +setuptools.command.build_ext.libtype = "shared" +setuptools.command.build_ext.link_shared_object = always_link_shared_object + +libtype = setuptools.command.build_ext.libtype +build_ext_cmd = Distribution().get_command_obj('build_ext') +build_ext_cmd.initialize_options() +build_ext_cmd.setup_shlib_compiler() + + +def libname(name): + ''' gets 'name' and returns something like libname.cpython-37m-darwin.so''' + filename = build_ext_cmd.get_ext_filename(name) + fn, ext = os.path.splitext(filename) + return build_ext_cmd.shlib_compiler.library_filename(fn, libtype) + + +pkg_name = 'hello' +zlib_name = '_zlibexample' +zlib_soname = libname(zlib_name) + +build_cmd = Distribution().get_command_obj('build') +build_cmd.finalize_options() +build_platlib = build_cmd.build_platlib + + +def link_args(soname=None): + args = [] + if platform.system() == "Linux": + if soname: + args += ['-Wl,-soname,' + soname] + loader_path = '$ORIGIN' + args += ['-Wl,-rpath,' + loader_path] + elif platform.system() == "Darwin": + if soname: + args += ["-Wl,-dylib", + '-Wl,-install_name,@rpath/%s' % soname] + args += ['-Wl,-rpath,@loader_path/'] + return args + + +hello_module = Extension(pkg_name + '._hello', + language='c++', + sources=['hello.cpp'], + extra_link_args=link_args(), + extra_objects=[build_platlib + '/hello/' + zlib_soname]) +zlib_example = Library(pkg_name + '.' + zlib_name, + language='c++', + extra_compile_args=['-lz'], + extra_link_args=link_args(zlib_soname) + ['-lz'], + sources=['extensions/testzlib.cpp'] + ) + +setup(name='hello', + version='0.1.0', + packages=find_packages(), + description='Hello world module written in C', + ext_modules=[zlib_example, hello_module]) diff --git a/tests/pr134/hello_module/tests/manual_test.py b/tests/pr134/hello_module/tests/manual_test.py new file mode 100644 index 00000000..cb325722 --- /dev/null +++ b/tests/pr134/hello_module/tests/manual_test.py @@ -0,0 +1,3 @@ +if __name__ == '__main__': + from hello import z_compress, z_uncompress + assert z_uncompress(z_compress('test')) == 'test' diff --git a/tests/test_hello.py b/tests/test_hello.py new file mode 100644 index 00000000..d2e0c859 --- /dev/null +++ b/tests/test_hello.py @@ -0,0 +1,97 @@ +import os +import os.path as op +import shutil +from test_manylinux import \ + docker_container, \ + docker_exec, \ + WHEEL_CACHE_FOLDER + +HELLO_WHEEL = 'hello-0.1.0-cp35-cp35m-linux_x86_64.whl' + + +def build_hello_wheel(docker_container): + policy, manylinux_id, python_id, io_folder = docker_container + + docker_exec(manylinux_id, 'yum install -y zlib-devel') + + if op.exists(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)): + # If hello has already been built and put in cache, let's reuse this. + shutil.copy2(op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL), + op.join(io_folder, HELLO_WHEEL)) + else: + docker_exec(manylinux_id, + 'pip wheel -w /io /auditwheel_src/tests/pr134/hello_module/') + shutil.copy2(op.join(io_folder, HELLO_WHEEL), + op.join(WHEEL_CACHE_FOLDER, HELLO_WHEEL)) + filenames = os.listdir(io_folder) + assert filenames == [HELLO_WHEEL] + orig_wheel = filenames[0] + assert 'manylinux' not in orig_wheel + return orig_wheel + + +def repair_hello_wheel(orig_wheel, docker_container): + policy, manylinux_id, python_id, io_folder = docker_container + # Repair the wheel using the manylinux container + repair_command = ( + 'auditwheel repair --plat {policy}_x86_64 -w /io /io/{orig_wheel}' + ).format(policy=policy, orig_wheel=orig_wheel) + docker_exec(manylinux_id, repair_command) + filenames = os.listdir(io_folder) + + # Regardless of build environment, wheel only needs manylinux1 symbols + repaired_wheels = [fn for fn in filenames if policy in fn] + assert repaired_wheels == ['hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl'.format(policy=policy)] + repaired_wheel = repaired_wheels[0] + + return repaired_wheel + + +def test_repair_reccurent_dependency(docker_container): + # tests https://github.com/pypa/auditwheel/issues/136 + policy, manylinux_id, python_id, io_folder = docker_container + orig_wheel = build_hello_wheel(docker_container) + + # attempting repair of the hello wheel + repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) + + output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) + # because this wheel is eligible to the manylinux1 tag, it will + # actually prioritize manylinux1 instead of manylinux2010 + assert ( + 'hello-0.1.0-cp35-cp35m-{policy}_x86_64.whl is consistent with the' + 'following platform tag: "manylinux1_x86_64"' + ).format(policy=policy) in output.replace('\n', '') + + +def test_correct_rpath_hello_wheel(docker_container): + # this tests https://github.com/pypa/auditwheel/issues/137 + policy, manylinux_id, python_id, io_folder = docker_container + orig_wheel = build_hello_wheel(docker_container) + + # attempting repair of the hello wheel + repaired_wheel = repair_hello_wheel(orig_wheel, docker_container) + + # Test whether repaired wheel is functioning. + + # TODO: Remove once pip supports manylinux2010 + docker_exec( + python_id, + "pip install git+https://github.com/wtolson/pip.git@manylinux2010", + ) + + test_commands = [ + 'pip install -U /io/' + repaired_wheel, + 'python /auditwheel_src/tests/pr134/hello_module/tests/manual_test.py', + ] + for cmd in test_commands: + docker_exec(python_id, cmd) + + +# from auditwheel.wheel_abi import analyze_wheel_abi +# def test_analyze_wheel_abi_hello(): +# winfo = analyze_wheel_abi( +# 'tests/python_snappy-0.5.2-pp260-pypy_41-linux_x86_64.whl') +# external_libs = winfo.external_refs['manylinux1_x86_64']['libs'] +# assert len(external_libs) > 0 +# assert set(external_libs) == {'libsnappy.so.1'}