From 680f74f5fe7c06aaabe3e499cd392ebe729b5a40 Mon Sep 17 00:00:00 2001 From: Denis Otkidach Date: Wed, 15 Nov 2023 18:53:11 +0200 Subject: [PATCH 1/4] Switch to pyproject.toml --- pyproject.toml | 57 +++++++++++++++++++ requirements-cython.txt | 2 +- setup.py | 121 +++------------------------------------- 3 files changed, 66 insertions(+), 114 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0b7540d8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[build-system] +requires = ["setuptools >=61", "wheel", "Cython >=3.0.5"] + +[project] +name = "aiokafka" +description = "Kafka integration with asyncio" +readme = "README.rst" +requires-python = ">=3.8" +license = { file = "LICENSE" } +authors = [ + { name = "Andrew Svetlov", email = "andrew.svetlov@gmail.com" }, +] +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: OS Independent", + "Topic :: System :: Networking", + "Topic :: System :: Distributed Computing", + "Framework :: AsyncIO", + "Development Status :: 4 - Beta", +] + +dynamic = ["version"] + +dependencies = [ + "async-timeout", + "packaging", +] + +[optional-dependencies] +snappy = ["cramjam"] +lz4 = ["lz4 >=3.1.3"] +zstd = ["cramjam"] +gssapi = ["gssapi"] +all = ["cramjam", "lz4 >=3.1.3", "gssapi"] + +[tool.setuptools.dynamic] +version = { attr = "aiokafka.__version__" } + +[tool.setuptools] +include-package-data = false + +[tool.setuptools.packages.find] +include = [ + "aiokafka", + "aiokafka.*", +] + +[project.urls] +Documentation = "http://aiokafka.readthedocs.org" +Source = "https://github.com/aio-libs/aiokafka" +Changes = "https://github.com/aio-libs/aiokafka/blob/master/CHANGES.rst" diff --git a/requirements-cython.txt b/requirements-cython.txt index 44e1d680..e4cba70d 100644 --- a/requirements-cython.txt +++ b/requirements-cython.txt @@ -1 +1 @@ -Cython==0.29.32 +Cython==3.0.5 diff --git a/setup.py b/setup.py index 112a1daf..6c70ba1f 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,10 @@ -import os import platform -import re +from Cython.Build import cythonize from setuptools import Extension, setup from setuptools.command.bdist_rpm import bdist_rpm as _bdist_rpm from setuptools.command.build_ext import build_ext - - -try: - from setuptools.errors import CCompilerError, ExecError, PlatformError -except ImportError: - # RTD workaround until it ships setuptools>=v59.0.0 - # See: - # - https://github.com/pypa/setuptools/pull/2858 - # - https://docs.readthedocs.io/en/stable/builds.html#python - from distutils.errors import ( - CCompilerError, - DistutilsExecError as ExecError, - DistutilsPlatformError as PlatformError, - ) +from setuptools.errors import CCompilerError, ExecError, PlatformError # Those are needed to build _hton for windows @@ -33,21 +19,11 @@ CFLAGS.extend(["-Wall", "-Wsign-compare", "-Wconversion"]) LIBRARIES.append("z") -# The extension part is copied from aiohttp's setup.py - -try: - from Cython.Build import cythonize - - USE_CYTHON = True -except ImportError: - USE_CYTHON = False - -ext = ".pyx" if USE_CYTHON else ".c" extensions = [ Extension( "aiokafka.record._crecords.legacy_records", - ["aiokafka/record/_crecords/legacy_records" + ext], + ["aiokafka/record/_crecords/legacy_records.pyx"], libraries=LIBRARIES, extra_compile_args=CFLAGS, extra_link_args=LDFLAGS, @@ -56,7 +32,7 @@ "aiokafka.record._crecords.default_records", [ "aiokafka/record/_crecords/crc32c.c", - "aiokafka/record/_crecords/default_records" + ext, + "aiokafka/record/_crecords/default_records.pyx", ], libraries=LIBRARIES, extra_compile_args=CFLAGS, @@ -64,14 +40,14 @@ ), Extension( "aiokafka.record._crecords.memory_records", - ["aiokafka/record/_crecords/memory_records" + ext], + ["aiokafka/record/_crecords/memory_records.pyx"], libraries=LIBRARIES, extra_compile_args=CFLAGS, extra_link_args=LDFLAGS, ), Extension( "aiokafka.record._crecords.cutil", - ["aiokafka/record/_crecords/crc32c.c", "aiokafka/record/_crecords/cutil" + ext], + ["aiokafka/record/_crecords/crc32c.c", "aiokafka/record/_crecords/cutil.pyx"], libraries=LIBRARIES, extra_compile_args=CFLAGS, extra_link_args=LDFLAGS, @@ -79,10 +55,6 @@ ] -if USE_CYTHON: - extensions = cythonize(extensions) - - class bdist_rpm(_bdist_rpm): def _make_spec_file(self): orig = super()._make_spec_file() @@ -110,84 +82,7 @@ def build_extension(self, ext): raise BuildFailed() -install_requires = [ - "async-timeout", - "cramjam", - "packaging", -] - - -def read(f): - return open(os.path.join(os.path.dirname(__file__), f)).read().strip() - - -extras_require = { - "snappy": ["cramjam"], - "lz4": ["lz4>=3.1.3"], - "zstd": ["cramjam"], - "gssapi": ["gssapi"], -} -extras_require["all"] = sum(extras_require.values(), []) - - -def read_version(): - regexp = re.compile(r"^__version__\W*=\W*'([\d.abrcdev]+)'") - init_py = os.path.join(os.path.dirname(__file__), "aiokafka", "__init__.py") - with open(init_py) as f: - for line in f: - match = regexp.match(line) - if match is not None: - return match.group(1) - else: - raise RuntimeError("Cannot find version in aiokafka/__init__.py") - - -classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Operating System :: OS Independent", - "Topic :: System :: Networking", - "Topic :: System :: Distributed Computing", - "Framework :: AsyncIO", - "Development Status :: 4 - Beta", -] - - -args = dict( - name="aiokafka", - version=read_version(), - description=("Kafka integration with asyncio."), - long_description="\n\n".join((read("README.rst"), read("CHANGES.rst"))), - classifiers=classifiers, - platforms=["POSIX"], - author="Andrew Svetlov", - author_email="andrew.svetlov@gmail.com", - url="http://aiokafka.readthedocs.org", - project_urls={ - "Source": "https://github.com/aio-libs/aiokafka", - }, - download_url="https://pypi.python.org/pypi/aiokafka", - license="Apache 2", - packages=["aiokafka"], - python_requires=">=3.8", - install_requires=install_requires, - extras_require=extras_require, - include_package_data=True, - ext_modules=extensions, +setup( + ext_modules=cythonize(extensions), cmdclass=dict(build_ext=ve_build_ext, bdist_rpm=bdist_rpm), ) - -try: - setup(**args) -except BuildFailed: - print("************************************************************") - print("Cannot compile C accelerator module, use pure python version") - print("************************************************************") - del args["ext_modules"] - del args["cmdclass"] - setup(**args) From 326745b40e50341104d3c344254bd4863bb1fb2b Mon Sep 17 00:00:00 2001 From: Denis Otkidach Date: Wed, 15 Nov 2023 20:37:39 +0200 Subject: [PATCH 2/4] Add 3.12 to all matrices --- .github/workflows/publish.yml | 24 +++++++++++++++++------- .github/workflows/tests.yml | 31 +++++++++++++++++-------------- pyproject.toml | 1 + requirements-ci.txt | 9 ++++----- requirements-dev.txt | 2 +- requirements-win-test.txt | 9 ++++----- 6 files changed, 44 insertions(+), 32 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1fdd5372..bc16e3bf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,12 +18,12 @@ jobs: python-version: "3.8" - name: Prepare C files to include run: | - python -m pip install --upgrade pip setuptools + python -m pip install --upgrade pip build python -m pip install -r requirements-cython.txt # Make sure we install to have all c files to be shiped with bundle python -m pip install -vv -U . # We set -vv to see compiler exceptions/warnings - name: Build source package - run: python setup.py sdist + run: python -m build --sdist - name: Upload source package uses: actions/upload-artifact@v2 with: @@ -51,14 +51,14 @@ jobs: - name: Build wheels env: CIBW_ARCHS_LINUX: ${{matrix.arch}} - CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* + CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* cp312-* CIBW_SKIP: '*-musllinux*' CIBW_BEFORE_BUILD_LINUX: pip install -r requirements-cython.txt && yum install -y zlib-devel # On windows and mac we should have z library preinstalled CIBW_BEFORE_BUILD: pip install -r requirements-cython.txt CIBW_BUILD_VERBOSITY: 2 run: | - python -m pip install --upgrade pip setuptools + python -m pip install --upgrade pip pip install cibuildwheel cibuildwheel --output-dir dist shell: bash @@ -74,7 +74,7 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - python: "3.8" aiokafka_whl: dist/aiokafka-*-cp38-cp38-win_amd64.whl @@ -82,6 +82,10 @@ jobs: aiokafka_whl: dist/aiokafka-*-cp39-cp39-win_amd64.whl - python: "3.10" aiokafka_whl: dist/aiokafka-*-cp310-cp310-win_amd64.whl + - python: "3.11" + aiokafka_whl: dist/aiokafka-*-cp311-cp311-win_amd64.whl + - python: "3.12" + aiokafka_whl: dist/aiokafka-*-cp312-cp312-win_amd64.whl steps: - uses: actions/checkout@v2 @@ -115,7 +119,7 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - python: "3.8" aiokafka_whl: dist/aiokafka-*-cp38-cp38-macosx_10_9_x86_64.whl @@ -125,6 +129,8 @@ jobs: aiokafka_whl: dist/aiokafka-*-cp310-cp310-macosx_10_9_x86_64.whl - python: "3.11" aiokafka_whl: dist/aiokafka-*-cp311-cp311-macosx_10_9_x86_64.whl + - python: "3.12" + aiokafka_whl: dist/aiokafka-*-cp312-cp312-macosx_10_9_x86_64.whl steps: - uses: actions/checkout@v2 @@ -156,7 +162,7 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: - python: "3.8" aiokafka_whl: dist/aiokafka-*-cp38-cp38-manylinux*_x86_64.whl @@ -166,6 +172,8 @@ jobs: aiokafka_whl: dist/aiokafka-*-cp310-cp310-manylinux*_x86_64.whl - python: "3.11" aiokafka_whl: dist/aiokafka-*-cp311-cp311-manylinux*_x86_64.whl + - python: "3.12" + aiokafka_whl: dist/aiokafka-*-cp312-cp312-manylinux*_x86_64.whl steps: - uses: actions/checkout@v2 @@ -213,6 +221,8 @@ jobs: aiokafka_whl: dist/aiokafka-*-cp310-cp310-manylinux*_aarch64.whl - pyver: cp311-cp311 aiokafka_whl: dist/aiokafka-*-cp311-cp311-manylinux*_aarch64.whl + - pyver: cp312-cp312 + aiokafka_whl: dist/aiokafka-*-cp312-cp312-manylinux*_aarch64.whl steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fb026548..b2a77f51 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,7 +73,7 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 @@ -141,7 +141,7 @@ jobs: strategy: matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 @@ -210,7 +210,7 @@ jobs: strategy: matrix: include: - - python: "3.11" + - python: "3.12" kafka: "2.8.1" scala: "2.13" @@ -224,39 +224,42 @@ jobs: - python: "3.10" kafka: "2.8.1" scala: "2.13" + - python: "3.11" + kafka: "2.8.1" + scala: "2.13" # Older brokers against latest python version - - python: "3.11" + - python: "3.12" kafka: "0.9.0.1" scala: "2.11" - - python: "3.11" + - python: "3.12" kafka: "0.10.2.1" scala: "2.11" - - python: "3.11" + - python: "3.12" kafka: "0.11.0.3" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "1.1.1" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.1.1" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.2.2" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.3.1" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.4.1" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.5.1" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.6.3" scala: "2.12" - - python: "3.11" + - python: "3.12" kafka: "2.7.2" scala: "2.13" fail-fast: false diff --git a/pyproject.toml b/pyproject.toml index 0b7540d8..7e3440db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Operating System :: OS Independent", "Topic :: System :: Networking", "Topic :: System :: Distributed Computing", diff --git a/requirements-ci.txt b/requirements-ci.txt index 678f9fd1..d5b93512 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -3,12 +3,11 @@ flake8==4.0.1 black==22.3.0 mypy==0.961 isort[colors]==5.10.0 -pytest==7.1.2 -pytest-cov==3.0.0 -pytest-asyncio==0.18.3 +pytest==7.4.3 +pytest-cov==4.1.0 +pytest-asyncio==0.21.1 pytest-mock==3.12.0 -docker==6.1.2 -chardet==4.0.0 # Until fixed requests is released +docker==6.1.3 lz4==3.1.3 docutils==0.17.1 Pygments==2.15.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 15d668cd..33dd4a2e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,4 @@ -r requirements-docs.txt diff-cover==6.4.2 -setuptools>=34.4.0 \ No newline at end of file +build==1.0.3 diff --git a/requirements-win-test.txt b/requirements-win-test.txt index da081032..71c2bff1 100644 --- a/requirements-win-test.txt +++ b/requirements-win-test.txt @@ -3,11 +3,10 @@ flake8==4.0.1 black==22.3.0 mypy==0.961 isort[colors]==5.10.0 -pytest==7.1.2 -pytest-cov==3.0.0 -pytest-asyncio==0.18.3 +pytest==7.4.3 +pytest-cov==4.1.0 +pytest-asyncio==0.21.1 pytest-mock==3.12.0 -docker==6.0.1 -chardet==4.0.0 # Until fixed requests is released +docker==6.1.3 lz4==3.1.3 cramjam==2.7.0 From 645ad4003dbc4dde43da906c81a33350f405a651 Mon Sep 17 00:00:00 2001 From: Denis Otkidach Date: Sun, 19 Nov 2023 15:42:24 +0200 Subject: [PATCH 3/4] Move pytest config to pyproject.toml --- pyproject.toml | 21 +++++++++++++++++++++ pytest.ini | 15 --------------- 2 files changed, 21 insertions(+), 15 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 7e3440db..458bbd68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,3 +56,24 @@ include = [ Documentation = "http://aiokafka.readthedocs.org" Source = "https://github.com/aio-libs/aiokafka" Changes = "https://github.com/aio-libs/aiokafka/blob/master/CHANGES.rst" + + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +addopts = ["--strict-config", "--strict-markers"] +markers = [ + "ssl: Tests that require SSL certificates to run", +] +filterwarnings = [ + "error", + # FIXME Until we fix socket leaks in tests + "default:unclosed event loop:ResourceWarning", + # https://github.com/docker/docker-py/issues/1293 + "ignore:.*docker.sock.*:ResourceWarning", + "ignore:distutils .* deprecated:DeprecationWarning:docker", + # From gssapi, but with improper stack + "ignore:_SixMetaPathImporter.*not found:ImportWarning", + # Actually comes from docker importing distutils on Windows + "ignore:the imp module is deprecated in favour of importlib:DeprecationWarning:pywintypes", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 6cef3385..00000000 --- a/pytest.ini +++ /dev/null @@ -1,15 +0,0 @@ -[pytest] -filterwarnings = - error - # FIXME Until we fix socket leaks in tests - default:unclosed event loop:ResourceWarning - # https://github.com/docker/docker-py/issues/1293 - ignore:.*docker.sock.*:ResourceWarning - ignore:distutils .* deprecated:DeprecationWarning:docker - # From gssapi, but with improper stack - ignore:_SixMetaPathImporter.*not found:ImportWarning - # Actually comes from docker importing distutils on Windows - ignore:the imp module is deprecated in favour of importlib:DeprecationWarning:pywintypes -markers = - ssl: Tests that require SSL certificates to run -asyncio_mode = auto From 9deee5e93eee0153a3e22c12d8cebe748c094d94 Mon Sep 17 00:00:00 2001 From: Denis Otkidach Date: Sun, 19 Nov 2023 15:55:02 +0200 Subject: [PATCH 4/4] Remove outdated warnings rules --- pyproject.toml | 7 ------- requirements-ci.txt | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 458bbd68..206dc4ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,11 +69,4 @@ filterwarnings = [ "error", # FIXME Until we fix socket leaks in tests "default:unclosed event loop:ResourceWarning", - # https://github.com/docker/docker-py/issues/1293 - "ignore:.*docker.sock.*:ResourceWarning", - "ignore:distutils .* deprecated:DeprecationWarning:docker", - # From gssapi, but with improper stack - "ignore:_SixMetaPathImporter.*not found:ImportWarning", - # Actually comes from docker importing distutils on Windows - "ignore:the imp module is deprecated in favour of importlib:DeprecationWarning:pywintypes", ] diff --git a/requirements-ci.txt b/requirements-ci.txt index d5b93512..78f2d8bc 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -11,6 +11,6 @@ docker==6.1.3 lz4==3.1.3 docutils==0.17.1 Pygments==2.15.0 -gssapi==1.8.2 +gssapi==1.8.3 async-timeout==4.0.1 cramjam==2.7.0