From 490bcb21915b4e5c3d9d1760441d856f310a8ac1 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Fri, 13 Sep 2024 14:16:14 +0300 Subject: [PATCH 1/4] chore: migrate to pyproject.toml --- .dockerignore | 3 +- .github/workflows/container-smoke-test.yml | 17 ++- .github/workflows/lint.yml | 6 +- .github/workflows/schema.yml | 3 +- .github/workflows/tests.yml | 6 +- CONTRIBUTING.md | 5 +- GNUmakefile | 34 +++--- MANIFEST.in | 4 +- README.rst | 11 +- container/Dockerfile | 8 +- pyproject.toml | 111 +++++++++++++++++ requirements/requirements-dev.in | 25 ---- requirements/requirements-dev.txt | 125 +++++++------------ requirements/requirements-typing.in | 8 -- requirements/requirements-typing.txt | 133 +++++++++++++++++---- requirements/requirements.in | 32 ----- requirements/requirements.txt | 52 ++++---- setup.py | 67 +---------- version.py | 62 ---------- website/source/install.rst | 2 +- 20 files changed, 351 insertions(+), 363 deletions(-) delete mode 100644 requirements/requirements-dev.in delete mode 100644 requirements/requirements-typing.in delete mode 100644 requirements/requirements.in delete mode 100644 version.py diff --git a/.dockerignore b/.dockerignore index 2e8e40bc6..2245fd0d9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,9 +6,8 @@ !karapace !go !requirements/*.txt -!setup.py -!version.py !README.rst +!pyproject.toml !container/start.sh !container/healthcheck.py diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml index d0eeb8dfe..e17aa2641 100644 --- a/.github/workflows/container-smoke-test.yml +++ b/.github/workflows/container-smoke-test.yml @@ -13,11 +13,22 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701 + with: + fetch-depth: 0 - - name: Build karapace/version.py - run: python version.py + - name: Install requirements + run: pip install . - - name: Build and start services + - name: Resolve Karapace version + run: | + KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") + echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV + + - name: Build container + run: docker build --build-arg KARAPACE_VERSION=${{ env.KARAPACE_VERSION }} --file=container/Dockerfile . + + - name: Run container run: docker compose --file=container/compose.yml up --build --wait --detach - name: Smoke test registry diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d029ff9f2..ed98fc671 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,8 +26,6 @@ jobs: python-version: '3.12' - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - # required for pylint - - run: make karapace/version.py - run: pip install pre-commit - uses: actions/cache@v3 with: @@ -45,6 +43,6 @@ jobs: python-version: '3.12' - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - - run: pip install -r requirements/requirements.txt -r requirements/requirements-typing.txt - - run: make karapace/version.py + - name: Install requirements and typing requirements + run: pip install .[typing] - run: mypy diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index 6665ab235..851ed47f1 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -21,7 +21,8 @@ jobs: requirements.txt - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - - run: make install + - name: Install requirements + - run: pip install . # Compare with latest release when running on main. - run: make schema against=$(git describe --abbrev=0 --tags) if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1df7f9ca8..5322b4d7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: with: go-version: '1.21.0' - - run: make install version + - run: make install - run: make unit-tests env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" @@ -73,6 +73,8 @@ jobs: steps: - uses: actions/checkout@v4 + - run: make install + - name: Download coverage id: download_coverage uses: actions/download-artifact@v4 @@ -80,8 +82,6 @@ jobs: pattern: coverage-* merge-multiple: true - - run: make karapace/version.py - - name: Post coverage comment id: post_coverage_comment uses: py-cov-action/python-coverage-comment-action@v3 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c2a42844..c2aafcf1b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,8 +19,8 @@ There is very little you need to get started coding for Karapace: ```python python -m venv source /bin/activate -pip install -r ./requirements/requirements-dev.txt -pip install -e . +pip install . +pip install -e .[dev,typing] ``` ## Website @@ -55,7 +55,6 @@ To run the tests use the binary `pytest` available in the virtualenv. It will do used in the tests for you: ```sh -make karapace/version.py pytest tests/unit pytest tests/integration ``` diff --git a/GNUmakefile b/GNUmakefile index 9d5c53a06..522e481e1 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -11,6 +11,14 @@ endif PYTHON_VERSION ?= 3.8 +define PIN_VERSIONS_COMMAND +pip install pip-tools && \ + python -m piptools compile -o /karapace/requirements/requirements.txt /karapace/pyproject.toml && \ + python -m piptools compile --extra dev -o /karapace/requirements/requirements-dev.txt /karapace/pyproject.toml && \ + python -m piptools compile --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml +endef + + export PATH := $(VENV_DIR)/bin:$(PATH) export PS4 := \e[0m\e[32m==> \e[0m export LC_ALL := C @@ -40,31 +48,23 @@ venv/.deps: requirements/requirements-dev.txt requirements/requirements.txt | ve source ./bin/get-protoc source ./bin/get-snappy set -x - $(PIP) install --use-pep517 -r '$(<)' - $(PIP) install --use-pep517 . + $(PIP) install --use-pep517 . .[dev] $(PIP) check touch '$(@)' - -karapace/version.py: - $(PYTHON) version.py - -.PHONY: version -version: venv/.make | karapace/version.py - .PHONY: test tests: unit-tests integration-tests .PHONY: unit-tests unit-tests: export PYTEST_ARGS ?= -unit-tests: karapace/version.py venv/.deps +unit-tests: venv/.deps rm -fr runtime/* $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ rm -fr runtime/* .PHONY: integration-tests unit-tests: export PYTEST_ARGS ?= -integration-tests: karapace/version.py venv/.deps +integration-tests: venv/.deps rm -fr runtime/* $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ rm -fr runtime/* @@ -84,12 +84,14 @@ cleanest: cleaner .PHONY: requirements requirements: export CUSTOM_COMPILE_COMMAND='make requirements' requirements: - pip install --upgrade pip setuptools pip-tools - cd requirements && pip-compile --upgrade --resolver=backtracking requirements.in -o requirements.txt - cd requirements && pip-compile --upgrade --resolver=backtracking requirements-dev.in -o requirements-dev.txt - cd requirements && pip-compile --upgrade --resolver=backtracking requirements-typing.in -o requirements-typing.txt + $(PIP) install --upgrade pip setuptools pip-tools + $(PIP) install .[dev,typing] .PHONY: schema schema: against := origin/main schema: - python3 -m karapace.backup.backends.v3.schema_tool --against=$(against) + $(PYTHON) -m karapace.backup.backends.v3.schema_tool --against=$(against) + +.PHONY: pin-requirements +pin-requirements: + docker run -e CUSTOM_COMPILE_COMMAND='make pin-requirements' -it -v .:/karapace --security-opt label=disable python:$(PYTHON_VERSION)-bullseye /bin/bash -c "$(PIN_VERSIONS_COMMAND)" diff --git a/MANIFEST.in b/MANIFEST.in index 0b55e5636..9a9525ac0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,9 +4,7 @@ include karapace.unit include tests/*.py include scripts/* include README.rst -include version.py -include setup.py -include setup.cfg +include pyproject.toml include LICENSE include MANIFEST.in include *.so diff --git a/README.rst b/README.rst index 650b17165..4d8992d0d 100644 --- a/README.rst +++ b/README.rst @@ -88,7 +88,7 @@ Source install Alternatively you can do a source install using:: - python setup.py install + pip install . Quickstart ========== @@ -651,7 +651,7 @@ If you don't need or want to have the Karapace images around you can now proceed Installed from Sources ---------------------- -If you installed Karapace from the sources via ``python setup.py install``, it can be uninstalled with the following ``pip`` command:: +Karapace is installed ``pip install .``, it can be uninstalled with the following ``pip`` command:: pip uninstall karapace @@ -695,11 +695,8 @@ targets that correctly clean the ``runtime`` directory without deleting it, but keep this in mind whenever you are not using ``make`` (e.g. running tests from your IDE). -Note that the pre-commit checks are currently not working with the default -Python version. This is because isort dropped Python 3.7 support. You have to -use at least Python 3.8 for the pre-commit checks. Use ``pipx`` or ``brew`` or -… to install pre-commit and use the global installation, there is also no -dependency on it. +Use ``pipx`` or ``brew`` to install ``pre-commit`` and use the global installation, +there is also no dependency on it. License ======= diff --git a/container/Dockerfile b/container/Dockerfile index 4fe2a46a5..efa862dff 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -26,8 +26,14 @@ RUN --mount=type=cache,target=/root/.cache/pip \ python3 -m pip install -r /build/requirements.txt COPY . /build/karapace-repo +WORKDIR /build/karapace-repo RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install /build/karapace-repo + if [ -z "${KARAPACE_VERSION}" ]; then \ + PRETEND_VERSION="$(python -c 'from karapace import version; print(version.__version__)')"; \ + else \ + PRETEND_VERSION=$KARAPACE_VERSION; \ + fi; \ + SETUPTOOLS_SCM_PRETEND_VERSION=$PRETEND_VERSION python3 -m pip install --no-deps . # Karapace image, i.e. production. FROM python:3.10.11-slim-bullseye AS karapace diff --git a/pyproject.toml b/pyproject.toml index 7244d92f5..1ba1edcbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,114 @@ +[build-system] +requires = ["setuptools > 64", "setuptools-scm", "setuptools-golang"] +build-backend = "setuptools.build_meta" + +[project] +name = "karapace" +requires-python = ">= 3.8" +dynamic = ["version"] +readme = "README.rst" +license = {file = "LICENSE"} +dependencies = [ + "accept-types < 1", + "aiohttp < 4", + "aiokafka == 0.10.0", + "cachetools == 5.3.3", + "confluent-kafka == 2.4.0", + "isodate < 1", + "jsonschema < 5", + "lz4", + "networkx < 4", + "protobuf < 4", + "pyjwt >= 2.4.0 , < 3", + "python-dateutil < 3", + "python-snappy", + "rich ~= 13.7.1", + "tenacity < 10", + "typing-extensions", + "ujson < 6", + "watchfiles < 1", + "xxhash ~= 3.3", + "zstandard", + "prometheus-client == 0.20.0", + "yarl == 1.12.1", + + # Patched dependencies + # + # Note: It is important to use commits to reference patched dependencies. This + # has two advantages: + # - Reproducible builds + # - The contents of the file change, which invalidates the existing docker + # images and forces a new image generation. + # + "avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py", +] + +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Database :: Database Engines/Servers", + "Topic :: Software Development :: Libraries", +] + +[project.scripts] +karapace = "karapace.karapace_all:main" +karapace_schema_backup = "karapace.backup.cli:main" +karapace_mkpasswd = "karapace.auth:main" + +[project.urls] +Homepage = "https://karapace.io" +Repository = "https://github.com/Aiven-Open/karapace/" +Issues = "https://github.com/Aiven-Open/karapace/issues" + +[project.optional-dependencies] +sentry-sdk = ["sentry-sdk>=1.6.0"] +ujson = ["ujson"] +dev = [ + # Developer QoL + "pdbpp", + + # testing + "filelock", + "hypothesis", + "psutil", + "pytest", + "pytest-cov", + "pytest-random-order", + "pytest-timeout", + "pytest-xdist[psutil]", + "requests", + + # performance test + "locust", + + # Sentry SDK + "sentry-sdk" +] +typing = [ + "mypy", + "sentry-sdk", + "types-cachetools", + "types-jsonschema", + "types-protobuf < 4" +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["karapace"] + +[tool.setuptools_scm] +version_file = "karapace/version.py" + [tool.black] target-version = ["py38"] line-length = 125 diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in deleted file mode 100644 index b11b729cb..000000000 --- a/requirements/requirements-dev.in +++ /dev/null @@ -1,25 +0,0 @@ -# runtime requirements --r requirements.txt - -# Developer QoL -pdbpp - -# testing -filelock -pytest -pytest-cov -pytest-xdist[psutil] -pytest-timeout -pytest-random-order -psutil -requests -hypothesis - -# performance test -locust - -# Sentry SDK -sentry-sdk - -# Golang -setuptools-golang diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 8a7f8ac58..849f7c38f 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -2,47 +2,39 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# 'make requirements' +# make pin-requirements # accept-types==0.4.1 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.0 - # via - # -r requirements.txt - # aiohttp + # via aiohttp aiohttp==3.10.5 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 - # via - # -r requirements.txt - # aiohttp + # via aiohttp anyio==4.4.0 - # via - # -r requirements.txt - # watchfiles + # via watchfiles async-timeout==4.0.3 # via - # -r requirements.txt # aiohttp # aiokafka attrs==24.2.0 # via - # -r requirements.txt # aiohttp # hypothesis # jsonschema # referencing # wmctrl avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) blinker==1.8.2 # via flask brotli==1.1.0 # via geventhttpclient cachetools==5.3.3 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) certifi==2024.7.4 # via # geventhttpclient @@ -55,16 +47,13 @@ click==8.1.7 configargparse==1.7 # via locust confluent-kafka==2.4.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) coverage[toml]==7.6.1 # via pytest-cov cramjam==2.8.3 - # via - # -r requirements.txt - # python-snappy + # via python-snappy exceptiongroup==1.2.2 # via - # -r requirements.txt # anyio # hypothesis # pytest @@ -73,7 +62,7 @@ execnet==2.1.1 fancycompleter==0.9.1 # via pdbpp filelock==3.15.4 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) flask==3.0.3 # via # flask-cors @@ -85,7 +74,6 @@ flask-login==0.6.3 # via locust frozenlist==1.4.1 # via - # -r requirements.txt # aiohttp # aiosignal gevent==24.2.1 @@ -97,10 +85,9 @@ geventhttpclient==2.0.12 greenlet==3.0.3 # via gevent hypothesis==6.111.2 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) idna==3.8 # via - # -r requirements.txt # anyio # requests # yarl @@ -108,136 +95,117 @@ importlib-metadata==8.4.0 # via flask importlib-resources==6.4.4 # via - # -r requirements.txt # jsonschema # jsonschema-specifications iniconfig==2.0.0 # via pytest isodate==0.6.1 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) itsdangerous==2.2.0 # via flask jinja2==3.1.4 # via flask jsonschema==4.23.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2023.12.1 - # via - # -r requirements.txt - # jsonschema + # via jsonschema locust==2.25.0 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) lz4==4.3.3 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 - # via - # -r requirements.txt - # rich + # via rich markupsafe==2.1.5 # via # jinja2 # werkzeug mdurl==0.1.2 - # via - # -r requirements.txt - # markdown-it-py + # via markdown-it-py msgpack==1.0.8 # via locust multidict==6.0.5 # via - # -r requirements.txt # aiohttp # yarl networkx==3.1 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) packaging==24.1 # via - # -r requirements.txt # aiokafka # pytest pdbpp==0.10.3 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) pkgutil-resolve-name==1.3.10 - # via - # -r requirements.txt - # jsonschema + # via jsonschema pluggy==1.5.0 # via pytest prometheus-client==0.20.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) psutil==6.0.0 # via - # -r requirements-dev.in + # karapace (/karapace/pyproject.toml) # locust # pytest-xdist pygments==2.18.0 # via - # -r requirements.txt # pdbpp # rich pyjwt==2.9.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) pyrepl==0.9.0 # via fancycompleter pytest==8.3.2 # via - # -r requirements-dev.in + # karapace (/karapace/pyproject.toml) # pytest-cov # pytest-random-order # pytest-timeout # pytest-xdist pytest-cov==5.0.0 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) pytest-random-order==1.1.1 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) pytest-timeout==2.3.1 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) pytest-xdist[psutil]==3.6.1 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) python-snappy==0.7.2 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) pyzmq==26.2.0 # via locust referencing==0.35.1 # via - # -r requirements.txt # jsonschema # jsonschema-specifications requests==2.32.3 # via - # -r requirements-dev.in + # karapace (/karapace/pyproject.toml) # locust rich==13.7.1 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) roundrobin==0.0.4 # via locust rpds-py==0.20.0 # via - # -r requirements.txt # jsonschema # referencing sentry-sdk==2.13.0 - # via -r requirements-dev.in -setuptools-golang==2.9.0 - # via -r requirements-dev.in + # via karapace (/karapace/pyproject.toml) six==1.16.0 # via - # -r requirements.txt # geventhttpclient # isodate # python-dateutil sniffio==1.3.1 - # via - # -r requirements.txt - # anyio + # via anyio sortedcontainers==2.4.0 # via hypothesis tenacity==9.0.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) tomli==2.0.1 # via # coverage @@ -245,17 +213,17 @@ tomli==2.0.1 # pytest typing-extensions==4.12.2 # via - # -r requirements.txt # anyio + # karapace (/karapace/pyproject.toml) # rich ujson==5.10.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) urllib3==2.2.2 # via # requests # sentry-sdk watchfiles==0.23.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) werkzeug==3.0.4 # via # flask @@ -264,14 +232,13 @@ werkzeug==3.0.4 wmctrl==0.5 # via pdbpp xxhash==3.5.0 - # via -r requirements.txt -yarl==1.9.4 + # via karapace (/karapace/pyproject.toml) +yarl==1.12.1 # via - # -r requirements.txt # aiohttp + # karapace (/karapace/pyproject.toml) zipp==3.20.1 # via - # -r requirements.txt # importlib-metadata # importlib-resources zope-event==5.0 @@ -279,7 +246,7 @@ zope-event==5.0 zope-interface==7.0.2 # via gevent zstandard==0.23.0 - # via -r requirements.txt + # via karapace (/karapace/pyproject.toml) # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements-typing.in b/requirements/requirements-typing.in deleted file mode 100644 index b0179da71..000000000 --- a/requirements/requirements-typing.in +++ /dev/null @@ -1,8 +0,0 @@ --c requirements.txt --c requirements-dev.txt - -mypy -types-jsonschema -sentry-sdk -types-cachetools -types-protobuf<4 diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index 7b064f425..503061bea 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -2,51 +2,140 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# 'make requirements' +# make pin-requirements # +accept-types==0.4.1 + # via karapace (/karapace/pyproject.toml) +aiohappyeyeballs==2.4.0 + # via aiohttp +aiohttp==3.10.6 + # via karapace (/karapace/pyproject.toml) +aiokafka==0.10.0 + # via karapace (/karapace/pyproject.toml) +aiosignal==1.3.1 + # via aiohttp +anyio==4.5.0 + # via watchfiles +async-timeout==4.0.3 + # via + # aiohttp + # aiokafka attrs==24.2.0 # via - # -c requirements-dev.txt - # -c requirements.txt + # aiohttp + # jsonschema # referencing +avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py + # via karapace (/karapace/pyproject.toml) +cachetools==5.3.3 + # via karapace (/karapace/pyproject.toml) certifi==2024.7.4 + # via sentry-sdk +confluent-kafka==2.4.0 + # via karapace (/karapace/pyproject.toml) +cramjam==2.8.4 + # via python-snappy +exceptiongroup==1.2.2 + # via anyio +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +idna==3.10 + # via + # anyio + # yarl +importlib-resources==6.4.5 + # via + # jsonschema + # jsonschema-specifications +isodate==0.6.1 + # via karapace (/karapace/pyproject.toml) +jsonschema==4.23.0 + # via karapace (/karapace/pyproject.toml) +jsonschema-specifications==2023.12.1 + # via jsonschema +lz4==4.3.3 + # via karapace (/karapace/pyproject.toml) +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.1.0 # via - # -c requirements-dev.txt - # sentry-sdk + # aiohttp + # yarl mypy==1.11.2 - # via -r requirements-typing.in + # via karapace (/karapace/pyproject.toml) mypy-extensions==1.0.0 # via mypy +networkx==3.1 + # via karapace (/karapace/pyproject.toml) +packaging==24.1 + # via aiokafka +pkgutil-resolve-name==1.3.10 + # via jsonschema +prometheus-client==0.20.0 + # via karapace (/karapace/pyproject.toml) +protobuf==3.20.3 + # via karapace (/karapace/pyproject.toml) +pygments==2.18.0 + # via rich +pyjwt==2.9.0 + # via karapace (/karapace/pyproject.toml) +python-dateutil==2.9.0.post0 + # via karapace (/karapace/pyproject.toml) +python-snappy==0.7.3 + # via karapace (/karapace/pyproject.toml) referencing==0.35.1 # via - # -c requirements-dev.txt - # -c requirements.txt + # jsonschema + # jsonschema-specifications # types-jsonschema +rich==13.7.1 + # via karapace (/karapace/pyproject.toml) rpds-py==0.20.0 # via - # -c requirements-dev.txt - # -c requirements.txt + # jsonschema # referencing sentry-sdk==2.13.0 + # via karapace (/karapace/pyproject.toml) +six==1.16.0 # via - # -c requirements-dev.txt - # -r requirements-typing.in + # isodate + # python-dateutil +sniffio==1.3.1 + # via anyio +tenacity==9.0.0 + # via karapace (/karapace/pyproject.toml) tomli==2.0.1 - # via - # -c requirements-dev.txt - # mypy + # via mypy types-cachetools==5.5.0.20240820 - # via -r requirements-typing.in + # via karapace (/karapace/pyproject.toml) types-jsonschema==4.23.0.20240813 - # via -r requirements-typing.in + # via karapace (/karapace/pyproject.toml) types-protobuf==3.20.4.6 - # via -r requirements-typing.in + # via karapace (/karapace/pyproject.toml) typing-extensions==4.12.2 # via - # -c requirements-dev.txt - # -c requirements.txt + # anyio + # karapace (/karapace/pyproject.toml) + # multidict # mypy + # rich +ujson==5.10.0 + # via karapace (/karapace/pyproject.toml) urllib3==2.2.2 + # via sentry-sdk +watchfiles==0.24.0 + # via karapace (/karapace/pyproject.toml) +xxhash==3.5.0 + # via karapace (/karapace/pyproject.toml) +yarl==1.12.1 # via - # -c requirements-dev.txt - # sentry-sdk + # aiohttp + # karapace (/karapace/pyproject.toml) +zipp==3.20.2 + # via importlib-resources +zstandard==0.23.0 + # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements.in b/requirements/requirements.in deleted file mode 100644 index fa61d5d54..000000000 --- a/requirements/requirements.in +++ /dev/null @@ -1,32 +0,0 @@ -# PyPI dependencies -accept-types<1 -aiohttp<4 -aiokafka==0.10.0 -cachetools==5.3.3 -confluent-kafka==2.4.0 -isodate<1 -jsonschema<5 -lz4 -networkx==3.1 -protobuf<4 -pyjwt>=2.4.0,<3 -python-dateutil<3 -python-snappy -rich~=13.7.1 -tenacity<10 -typing-extensions -ujson<6 -watchfiles<1 -xxhash~=3.3 -zstandard -prometheus-client==0.20.0 - -# Patched dependencies -# -# Note: It is important to use commits to reference patched dependencies. This -# has two advantages: -# - Reproducible builds -# - The contents of the file change, which invalidates the existing docker -# images and forces a new image generation. -# -https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 368ffb097..d7803e46e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -2,16 +2,16 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# 'make requirements' +# make pin-requirements # accept-types==0.4.1 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.0 # via aiohttp aiohttp==3.10.5 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.4.0 @@ -26,11 +26,11 @@ attrs==24.2.0 # jsonschema # referencing avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c76ca2.tar.gz#subdirectory=lang/py - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) cachetools==5.3.3 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) confluent-kafka==2.4.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) cramjam==2.8.3 # via python-snappy exceptiongroup==1.2.2 @@ -48,13 +48,13 @@ importlib-resources==6.4.4 # jsonschema # jsonschema-specifications isodate==0.6.1 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) jsonschema==4.23.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2023.12.1 # via jsonschema lz4==4.3.3 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -64,29 +64,29 @@ multidict==6.0.5 # aiohttp # yarl networkx==3.1 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) packaging==24.1 # via aiokafka pkgutil-resolve-name==1.3.10 # via jsonschema prometheus-client==0.20.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich pyjwt==2.9.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) python-snappy==0.7.2 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) referencing==0.35.1 # via # jsonschema # jsonschema-specifications rich==13.7.1 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) rpds-py==0.20.0 # via # jsonschema @@ -98,21 +98,23 @@ six==1.16.0 sniffio==1.3.1 # via anyio tenacity==9.0.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) typing-extensions==4.12.2 # via - # -r requirements.in # anyio + # karapace (/karapace/pyproject.toml) # rich ujson==5.10.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) watchfiles==0.23.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) xxhash==3.5.0 - # via -r requirements.in -yarl==1.9.4 - # via aiohttp + # via karapace (/karapace/pyproject.toml) +yarl==1.12.1 + # via + # aiohttp + # karapace (/karapace/pyproject.toml) zipp==3.20.1 # via importlib-resources zstandard==0.23.0 - # via -r requirements.in + # via karapace (/karapace/pyproject.toml) diff --git a/setup.py b/setup.py index cd8bac10e..ad992e355 100644 --- a/setup.py +++ b/setup.py @@ -1,77 +1,12 @@ """ karapace - setup - Copyright (c) 2023 Aiven Ltd See LICENSE for details """ -from setuptools import Extension, find_packages, setup - -import os -import version - -readme_path = os.path.join(os.path.dirname(__file__), "README.rst") -with open(readme_path, encoding="utf8") as fp: - readme_text = fp.read() - -version_for_setup_py = version.get_project_version() -version_for_setup_py = ".dev".join(version_for_setup_py.split("-", 2)[:2]) +from setuptools import Extension, setup setup( - name="karapace", - version=version_for_setup_py, - zip_safe=False, - packages=find_packages(exclude=["test"]), setup_requires=["setuptools-golang"], - install_requires=[ - "accept-types", - "aiohttp", - "aiokafka", - "avro", - "jsonschema", - "networkx", - "protobuf", - "pyjwt", - "python-dateutil", - # compression algorithms supported by confluent-kafka-python - "lz4", - "python-snappy", - "zstandard", - ], - extras_require={ - "sentry-sdk": ["sentry-sdk>=1.6.0"], - "ujson": ["ujson"], - }, - dependency_links=[], - package_data={}, - entry_points={ - "console_scripts": [ - "karapace = karapace.karapace_all:main", - "karapace_schema_backup = karapace.backup.cli:main", - "karapace_mkpasswd = karapace.auth:main", - ], - }, - author="Hannu Valtonen", - author_email="opensource@aiven.io", - license="Apache 2.0", - platforms=["POSIX", "MacOS"], - description="Karapace", - long_description=readme_text, - url="https://github.com/aiven/karapace/", - python_requires=">=3.8", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Database :: Database Engines/Servers", - "Topic :: Software Development :: Libraries", - ], - include_package_data=True, ext_modules=[ Extension( "protopacelib", diff --git a/version.py b/version.py deleted file mode 100644 index 64a193286..000000000 --- a/version.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -karapace - version - -Copyright (c) 2023 Aiven Ltd -See LICENSE for details -""" -from __future__ import annotations - -from typing import Final - -import os -import pathlib -import subprocess - -version_file: Final = pathlib.Path(__file__).parent.resolve() / "karapace/version.py" - - -def save_version(new_ver, old_ver): - if not new_ver: - return False - if not old_ver or new_ver != old_ver: - version_file.write_text(f'"""{__doc__}"""\n__version__ = "{new_ver}"\n') - return True - - -def from_version_file() -> str | None: - try: - import karapace.version - except ImportError: - return None - return karapace.version.__version__ - - -def get_project_version() -> str: - file_ver = from_version_file() - - version = os.getenv("KARAPACE_VERSION") - if version is None: - os.chdir(os.path.dirname(__file__) or ".") - try: - git_out = subprocess.check_output( - ["git", "describe", "--always", "--tags"], stderr=getattr(subprocess, "DEVNULL", None) - ) - except (OSError, subprocess.CalledProcessError): - pass - else: - git_ver = git_out.splitlines()[0].strip().decode("utf-8") - if "." not in git_ver: - git_ver = f"0.0.1-0-unknown-{git_ver}" - version = git_ver - - if save_version(version, file_ver): - return version - - if not file_ver: - raise RuntimeError(f"version not available from git or from file {str(version_file)!r}") - - return file_ver - - -if __name__ == "__main__": - get_project_version() diff --git a/website/source/install.rst b/website/source/install.rst index 39ca4e15d..2779668a2 100644 --- a/website/source/install.rst +++ b/website/source/install.rst @@ -37,4 +37,4 @@ Source install Alternatively you can do a source install using:: - python setup.py install + pip install . From c0b7e6d939fb3eeb092d752576b9e3dc6d19a074 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Wed, 25 Sep 2024 10:31:44 +0300 Subject: [PATCH 2/4] chore: use source layout --- .coveragerc | 6 ++--- .dockerignore | 4 +++- .github/workflows/container-smoke-test.yml | 3 ++- .github/workflows/lint.yml | 9 ++++++-- .github/workflows/schema.yml | 10 +++++--- .github/workflows/tests.yml | 4 ++-- GNUmakefile | 23 +++++++++++++++---- MANIFEST.in | 3 +-- container/Dockerfile | 2 +- pyproject.toml | 7 ++---- {karapace => src/karapace}/__init__.py | 0 .../karapace}/anonymize_schemas/__init__.py | 0 .../anonymize_schemas/anonymize_avro.py | 0 {karapace => src/karapace}/auth.py | 0 .../karapace}/avro_dataclasses/__init__.py | 0 .../karapace}/avro_dataclasses/introspect.py | 0 .../karapace}/avro_dataclasses/models.py | 0 .../karapace}/avro_dataclasses/schema.py | 0 {karapace => src/karapace}/backup/__init__.py | 0 {karapace => src/karapace}/backup/api.py | 0 .../karapace}/backup/backends/__init__.py | 0 .../karapace}/backup/backends/reader.py | 0 .../karapace}/backup/backends/v1.py | 0 .../karapace}/backup/backends/v2.py | 0 .../karapace}/backup/backends/v3/__init__.py | 0 .../backup/backends/v3/avro/DataFile.avsc | 0 .../backup/backends/v3/avro/Header.avsc | 0 .../backup/backends/v3/avro/Metadata.avsc | 0 .../backup/backends/v3/avro/Record.avsc | 0 .../karapace}/backup/backends/v3/backend.py | 0 .../karapace}/backup/backends/v3/checksum.py | 0 .../karapace}/backup/backends/v3/constants.py | 0 .../karapace}/backup/backends/v3/errors.py | 0 .../karapace}/backup/backends/v3/readers.py | 0 .../karapace}/backup/backends/v3/schema.py | 0 .../backup/backends/v3/schema_tool.py | 16 +++++++++++++ .../karapace}/backup/backends/v3/writers.py | 0 .../karapace}/backup/backends/writer.py | 0 {karapace => src/karapace}/backup/cli.py | 0 {karapace => src/karapace}/backup/encoders.py | 0 {karapace => src/karapace}/backup/errors.py | 0 .../karapace}/backup/poll_timeout.py | 0 .../karapace}/backup/safe_writer.py | 0 .../karapace}/backup/topic_configurations.py | 0 {karapace => src/karapace}/client.py | 0 .../karapace}/compatibility/__init__.py | 0 .../compatibility/jsonschema/__init__.py | 0 .../compatibility/jsonschema/checks.py | 0 .../compatibility/jsonschema/types.py | 0 .../compatibility/jsonschema/utils.py | 0 .../compatibility/protobuf/__init__.py | 0 .../compatibility/protobuf/checks.py | 0 {karapace => src/karapace}/config.py | 0 {karapace => src/karapace}/constants.py | 0 .../karapace}/coordinator/__init__.py | 0 .../coordinator/master_coordinator.py | 0 .../coordinator/schema_coordinator.py | 0 {karapace => src/karapace}/dataclasses.py | 0 {karapace => src/karapace}/dependency.py | 0 {karapace => src/karapace}/errors.py | 0 .../karapace}/in_memory_database.py | 0 .../karapace}/instrumentation/__init__.py | 0 .../karapace}/instrumentation/prometheus.py | 0 {karapace => src/karapace}/kafka/__init__.py | 0 {karapace => src/karapace}/kafka/admin.py | 0 {karapace => src/karapace}/kafka/common.py | 0 {karapace => src/karapace}/kafka/consumer.py | 0 {karapace => src/karapace}/kafka/producer.py | 0 {karapace => src/karapace}/kafka/types.py | 0 .../karapace}/kafka_error_handler.py | 0 .../karapace}/kafka_rest_apis/__init__.py | 0 .../kafka_rest_apis/authentication.py | 0 .../kafka_rest_apis/consumer_manager.py | 0 .../karapace}/kafka_rest_apis/error_codes.py | 0 .../karapace}/kafka_rest_apis/schema_cache.py | 0 {karapace => src/karapace}/kafka_utils.py | 0 {karapace => src/karapace}/karapace.py | 0 {karapace => src/karapace}/karapace_all.py | 0 {karapace => src/karapace}/key_format.py | 0 {karapace => src/karapace}/messaging.py | 0 {karapace => src/karapace}/offset_watcher.py | 0 .../karapace}/protobuf/__init__.py | 0 .../karapace}/protobuf/compare_result.py | 0 .../karapace}/protobuf/compare_type_lists.py | 0 .../protobuf/compare_type_storage.py | 0 .../karapace}/protobuf/encoding_variants.py | 0 .../protobuf/enum_constant_element.py | 0 .../karapace}/protobuf/enum_element.py | 0 .../karapace}/protobuf/exception.py | 0 .../karapace}/protobuf/extend_element.py | 0 .../karapace}/protobuf/extensions_element.py | 0 {karapace => src/karapace}/protobuf/field.py | 0 .../karapace}/protobuf/field_element.py | 0 .../karapace}/protobuf/group_element.py | 0 {karapace => src/karapace}/protobuf/io.py | 0 .../karapace}/protobuf/known_dependency.py | 0 .../karapace}/protobuf/kotlin_wrapper.py | 0 .../karapace}/protobuf/location.py | 0 .../karapace}/protobuf/message_element.py | 0 .../karapace}/protobuf/one_of_element.py | 0 .../karapace}/protobuf/option_element.py | 0 .../karapace}/protobuf/option_reader.py | 0 .../karapace}/protobuf/proto_file_element.py | 0 .../protobuf/proto_normalizations.py | 0 .../karapace}/protobuf/proto_parser.py | 0 .../karapace}/protobuf/proto_type.py | 0 .../karapace}/protobuf/protobuf_to_dict.py | 0 .../karapace}/protobuf/protopace/__init__.py | 0 .../karapace}/protobuf/protopace/protopace.py | 0 .../karapace}/protobuf/reserved_element.py | 0 .../karapace}/protobuf/rpc_element.py | 0 {karapace => src/karapace}/protobuf/schema.py | 0 .../karapace}/protobuf/serialization.py | 0 .../karapace}/protobuf/service_element.py | 0 {karapace => src/karapace}/protobuf/syntax.py | 0 .../karapace}/protobuf/syntax_reader.py | 0 .../karapace}/protobuf/type_element.py | 0 .../karapace}/protobuf/type_tree.py | 0 {karapace => src/karapace}/protobuf/utils.py | 0 {karapace => src/karapace}/rapu.py | 0 {karapace => src/karapace}/schema_models.py | 0 {karapace => src/karapace}/schema_reader.py | 0 .../karapace}/schema_references.py | 0 {karapace => src/karapace}/schema_registry.py | 0 .../karapace}/schema_registry_apis.py | 0 {karapace => src/karapace}/schema_type.py | 0 {karapace => src/karapace}/sentry/__init__.py | 0 .../karapace}/sentry/sentry_client.py | 0 .../karapace}/sentry/sentry_client_api.py | 0 {karapace => src/karapace}/serialization.py | 0 {karapace => src/karapace}/statsd.py | 0 {karapace => src/karapace}/typing.py | 0 {karapace => src/karapace}/utils.py | 0 133 files changed, 62 insertions(+), 25 deletions(-) rename {karapace => src/karapace}/__init__.py (100%) rename {karapace => src/karapace}/anonymize_schemas/__init__.py (100%) rename {karapace => src/karapace}/anonymize_schemas/anonymize_avro.py (100%) rename {karapace => src/karapace}/auth.py (100%) rename {karapace => src/karapace}/avro_dataclasses/__init__.py (100%) rename {karapace => src/karapace}/avro_dataclasses/introspect.py (100%) rename {karapace => src/karapace}/avro_dataclasses/models.py (100%) rename {karapace => src/karapace}/avro_dataclasses/schema.py (100%) rename {karapace => src/karapace}/backup/__init__.py (100%) rename {karapace => src/karapace}/backup/api.py (100%) rename {karapace => src/karapace}/backup/backends/__init__.py (100%) rename {karapace => src/karapace}/backup/backends/reader.py (100%) rename {karapace => src/karapace}/backup/backends/v1.py (100%) rename {karapace => src/karapace}/backup/backends/v2.py (100%) rename {karapace => src/karapace}/backup/backends/v3/__init__.py (100%) rename {karapace => src/karapace}/backup/backends/v3/avro/DataFile.avsc (100%) rename {karapace => src/karapace}/backup/backends/v3/avro/Header.avsc (100%) rename {karapace => src/karapace}/backup/backends/v3/avro/Metadata.avsc (100%) rename {karapace => src/karapace}/backup/backends/v3/avro/Record.avsc (100%) rename {karapace => src/karapace}/backup/backends/v3/backend.py (100%) rename {karapace => src/karapace}/backup/backends/v3/checksum.py (100%) rename {karapace => src/karapace}/backup/backends/v3/constants.py (100%) rename {karapace => src/karapace}/backup/backends/v3/errors.py (100%) rename {karapace => src/karapace}/backup/backends/v3/readers.py (100%) rename {karapace => src/karapace}/backup/backends/v3/schema.py (100%) rename {karapace => src/karapace}/backup/backends/v3/schema_tool.py (89%) rename {karapace => src/karapace}/backup/backends/v3/writers.py (100%) rename {karapace => src/karapace}/backup/backends/writer.py (100%) rename {karapace => src/karapace}/backup/cli.py (100%) rename {karapace => src/karapace}/backup/encoders.py (100%) rename {karapace => src/karapace}/backup/errors.py (100%) rename {karapace => src/karapace}/backup/poll_timeout.py (100%) rename {karapace => src/karapace}/backup/safe_writer.py (100%) rename {karapace => src/karapace}/backup/topic_configurations.py (100%) rename {karapace => src/karapace}/client.py (100%) rename {karapace => src/karapace}/compatibility/__init__.py (100%) rename {karapace => src/karapace}/compatibility/jsonschema/__init__.py (100%) rename {karapace => src/karapace}/compatibility/jsonschema/checks.py (100%) rename {karapace => src/karapace}/compatibility/jsonschema/types.py (100%) rename {karapace => src/karapace}/compatibility/jsonschema/utils.py (100%) rename {karapace => src/karapace}/compatibility/protobuf/__init__.py (100%) rename {karapace => src/karapace}/compatibility/protobuf/checks.py (100%) rename {karapace => src/karapace}/config.py (100%) rename {karapace => src/karapace}/constants.py (100%) rename {karapace => src/karapace}/coordinator/__init__.py (100%) rename {karapace => src/karapace}/coordinator/master_coordinator.py (100%) rename {karapace => src/karapace}/coordinator/schema_coordinator.py (100%) rename {karapace => src/karapace}/dataclasses.py (100%) rename {karapace => src/karapace}/dependency.py (100%) rename {karapace => src/karapace}/errors.py (100%) rename {karapace => src/karapace}/in_memory_database.py (100%) rename {karapace => src/karapace}/instrumentation/__init__.py (100%) rename {karapace => src/karapace}/instrumentation/prometheus.py (100%) rename {karapace => src/karapace}/kafka/__init__.py (100%) rename {karapace => src/karapace}/kafka/admin.py (100%) rename {karapace => src/karapace}/kafka/common.py (100%) rename {karapace => src/karapace}/kafka/consumer.py (100%) rename {karapace => src/karapace}/kafka/producer.py (100%) rename {karapace => src/karapace}/kafka/types.py (100%) rename {karapace => src/karapace}/kafka_error_handler.py (100%) rename {karapace => src/karapace}/kafka_rest_apis/__init__.py (100%) rename {karapace => src/karapace}/kafka_rest_apis/authentication.py (100%) rename {karapace => src/karapace}/kafka_rest_apis/consumer_manager.py (100%) rename {karapace => src/karapace}/kafka_rest_apis/error_codes.py (100%) rename {karapace => src/karapace}/kafka_rest_apis/schema_cache.py (100%) rename {karapace => src/karapace}/kafka_utils.py (100%) rename {karapace => src/karapace}/karapace.py (100%) rename {karapace => src/karapace}/karapace_all.py (100%) rename {karapace => src/karapace}/key_format.py (100%) rename {karapace => src/karapace}/messaging.py (100%) rename {karapace => src/karapace}/offset_watcher.py (100%) rename {karapace => src/karapace}/protobuf/__init__.py (100%) rename {karapace => src/karapace}/protobuf/compare_result.py (100%) rename {karapace => src/karapace}/protobuf/compare_type_lists.py (100%) rename {karapace => src/karapace}/protobuf/compare_type_storage.py (100%) rename {karapace => src/karapace}/protobuf/encoding_variants.py (100%) rename {karapace => src/karapace}/protobuf/enum_constant_element.py (100%) rename {karapace => src/karapace}/protobuf/enum_element.py (100%) rename {karapace => src/karapace}/protobuf/exception.py (100%) rename {karapace => src/karapace}/protobuf/extend_element.py (100%) rename {karapace => src/karapace}/protobuf/extensions_element.py (100%) rename {karapace => src/karapace}/protobuf/field.py (100%) rename {karapace => src/karapace}/protobuf/field_element.py (100%) rename {karapace => src/karapace}/protobuf/group_element.py (100%) rename {karapace => src/karapace}/protobuf/io.py (100%) rename {karapace => src/karapace}/protobuf/known_dependency.py (100%) rename {karapace => src/karapace}/protobuf/kotlin_wrapper.py (100%) rename {karapace => src/karapace}/protobuf/location.py (100%) rename {karapace => src/karapace}/protobuf/message_element.py (100%) rename {karapace => src/karapace}/protobuf/one_of_element.py (100%) rename {karapace => src/karapace}/protobuf/option_element.py (100%) rename {karapace => src/karapace}/protobuf/option_reader.py (100%) rename {karapace => src/karapace}/protobuf/proto_file_element.py (100%) rename {karapace => src/karapace}/protobuf/proto_normalizations.py (100%) rename {karapace => src/karapace}/protobuf/proto_parser.py (100%) rename {karapace => src/karapace}/protobuf/proto_type.py (100%) rename {karapace => src/karapace}/protobuf/protobuf_to_dict.py (100%) rename {karapace => src/karapace}/protobuf/protopace/__init__.py (100%) rename {karapace => src/karapace}/protobuf/protopace/protopace.py (100%) rename {karapace => src/karapace}/protobuf/reserved_element.py (100%) rename {karapace => src/karapace}/protobuf/rpc_element.py (100%) rename {karapace => src/karapace}/protobuf/schema.py (100%) rename {karapace => src/karapace}/protobuf/serialization.py (100%) rename {karapace => src/karapace}/protobuf/service_element.py (100%) rename {karapace => src/karapace}/protobuf/syntax.py (100%) rename {karapace => src/karapace}/protobuf/syntax_reader.py (100%) rename {karapace => src/karapace}/protobuf/type_element.py (100%) rename {karapace => src/karapace}/protobuf/type_tree.py (100%) rename {karapace => src/karapace}/protobuf/utils.py (100%) rename {karapace => src/karapace}/rapu.py (100%) rename {karapace => src/karapace}/schema_models.py (100%) rename {karapace => src/karapace}/schema_reader.py (100%) rename {karapace => src/karapace}/schema_references.py (100%) rename {karapace => src/karapace}/schema_registry.py (100%) rename {karapace => src/karapace}/schema_registry_apis.py (100%) rename {karapace => src/karapace}/schema_type.py (100%) rename {karapace => src/karapace}/sentry/__init__.py (100%) rename {karapace => src/karapace}/sentry/sentry_client.py (100%) rename {karapace => src/karapace}/sentry/sentry_client_api.py (100%) rename {karapace => src/karapace}/serialization.py (100%) rename {karapace => src/karapace}/statsd.py (100%) rename {karapace => src/karapace}/typing.py (100%) rename {karapace => src/karapace}/utils.py (100%) diff --git a/.coveragerc b/.coveragerc index 7d422503e..2a6a5d055 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] -branch = True -relative_files = True -source = karapace +branch = true +relative_files = true +source = src/karapace diff --git a/.dockerignore b/.dockerignore index 2245fd0d9..57efb59ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,11 +3,13 @@ * # Include source directories and files required for building. -!karapace !go +!src !requirements/*.txt !README.rst +!LICENSE !pyproject.toml +!setup.py !container/start.sh !container/healthcheck.py diff --git a/.github/workflows/container-smoke-test.yml b/.github/workflows/container-smoke-test.yml index e17aa2641..cced926bd 100644 --- a/.github/workflows/container-smoke-test.yml +++ b/.github/workflows/container-smoke-test.yml @@ -18,10 +18,11 @@ jobs: fetch-depth: 0 - name: Install requirements - run: pip install . + run: make install - name: Resolve Karapace version run: | + source ./venv/bin/activate KARAPACE_VERSION=$(python -c "from karapace import version; print(version.__version__)") echo KARAPACE_VERSION=$KARAPACE_VERSION >> $GITHUB_ENV diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ed98fc671..b8f9b57c3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -44,5 +44,10 @@ jobs: - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - name: Install requirements and typing requirements - run: pip install .[typing] - - run: mypy + run: | + python -m venv venv + source ./venv/bin/activate + pip --require-virtualenv install .[typing] + - run: | + source ./venv/bin/activate + mypy src diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index 851ed47f1..d02a347d8 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -22,12 +22,16 @@ jobs: - name: Install libsnappy-dev run: sudo apt install libsnappy-dev - name: Install requirements - - run: pip install . + run: make install-dev # Compare with latest release when running on main. - - run: make schema against=$(git describe --abbrev=0 --tags) + - run: | + source ./venv/bin/activate + make schema against=$(git describe --abbrev=0 --tags) if: github.ref == 'refs/heads/main' # Compare with main when running on branches/PRs. - - run: make schema + - run: | + source ./venv/bin/activate + make schema if: github.ref != 'refs/heads/main' - run: | diff=$(git --no-pager diff) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5322b4d7e..d3ae1c40f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: with: go-version: '1.21.0' - - run: make install + - run: make install-dev - run: make unit-tests env: COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" @@ -73,7 +73,7 @@ jobs: steps: - uses: actions/checkout@v4 - - run: make install + - run: make install-dev - name: Download coverage id: download_coverage diff --git a/GNUmakefile b/GNUmakefile index 522e481e1..9d318a5cc 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -42,29 +42,42 @@ venv/.make: .PHONY: install install: venv/.deps -venv/.deps: requirements/requirements-dev.txt requirements/requirements.txt | venv/.make +venv/.deps: venv/.make set +x source ./bin/get-java source ./bin/get-protoc source ./bin/get-snappy set -x - $(PIP) install --use-pep517 . .[dev] + $(PIP) install --use-pep517 . $(PIP) check touch '$(@)' +.PHONY: install-dev +install-dev: venv/.deps-dev +venv/.deps-dev: venv/.make + set +x + source ./bin/get-java + source ./bin/get-protoc + source ./bin/get-snappy + set -x + $(PIP) install -e .[dev] + $(PIP) check + touch '$(@)' + + .PHONY: test tests: unit-tests integration-tests .PHONY: unit-tests unit-tests: export PYTEST_ARGS ?= -unit-tests: venv/.deps +unit-tests: venv/.deps-dev rm -fr runtime/* $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/ rm -fr runtime/* .PHONY: integration-tests unit-tests: export PYTEST_ARGS ?= -integration-tests: venv/.deps +integration-tests: venv/.deps-dev rm -fr runtime/* $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/ rm -fr runtime/* @@ -82,7 +95,7 @@ cleanest: cleaner rm -fr '$(VENV_DIR)' .PHONY: requirements -requirements: export CUSTOM_COMPILE_COMMAND='make requirements' +requirements: requirements: $(PIP) install --upgrade pip setuptools pip-tools $(PIP) install .[dev,typing] diff --git a/MANIFEST.in b/MANIFEST.in index 9a9525ac0..4ee222daa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include karapace/*.py +graft src include karapace.json include karapace.unit include tests/*.py @@ -7,6 +7,5 @@ include README.rst include pyproject.toml include LICENSE include MANIFEST.in -include *.so recursive-exclude examples *~ *.pyc \.* diff --git a/container/Dockerfile b/container/Dockerfile index efa862dff..2e1544319 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -29,7 +29,7 @@ COPY . /build/karapace-repo WORKDIR /build/karapace-repo RUN --mount=type=cache,target=/root/.cache/pip \ if [ -z "${KARAPACE_VERSION}" ]; then \ - PRETEND_VERSION="$(python -c 'from karapace import version; print(version.__version__)')"; \ + PRETEND_VERSION="$(python -c 'from src.karapace import version; print(version.__version__)')"; \ else \ PRETEND_VERSION=$KARAPACE_VERSION; \ fi; \ diff --git a/pyproject.toml b/pyproject.toml index 1ba1edcbf..089668037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools > 64", "setuptools-scm", "setuptools-golang"] +requires = ["setuptools > 64", "setuptools-scm > 8.0.0", "setuptools-golang"] build-backend = "setuptools.build_meta" [project] @@ -103,11 +103,8 @@ typing = [ [tool.setuptools] include-package-data = true -[tool.setuptools.packages.find] -include = ["karapace"] - [tool.setuptools_scm] -version_file = "karapace/version.py" +version_file = "src/karapace/version.py" [tool.black] target-version = ["py38"] diff --git a/karapace/__init__.py b/src/karapace/__init__.py similarity index 100% rename from karapace/__init__.py rename to src/karapace/__init__.py diff --git a/karapace/anonymize_schemas/__init__.py b/src/karapace/anonymize_schemas/__init__.py similarity index 100% rename from karapace/anonymize_schemas/__init__.py rename to src/karapace/anonymize_schemas/__init__.py diff --git a/karapace/anonymize_schemas/anonymize_avro.py b/src/karapace/anonymize_schemas/anonymize_avro.py similarity index 100% rename from karapace/anonymize_schemas/anonymize_avro.py rename to src/karapace/anonymize_schemas/anonymize_avro.py diff --git a/karapace/auth.py b/src/karapace/auth.py similarity index 100% rename from karapace/auth.py rename to src/karapace/auth.py diff --git a/karapace/avro_dataclasses/__init__.py b/src/karapace/avro_dataclasses/__init__.py similarity index 100% rename from karapace/avro_dataclasses/__init__.py rename to src/karapace/avro_dataclasses/__init__.py diff --git a/karapace/avro_dataclasses/introspect.py b/src/karapace/avro_dataclasses/introspect.py similarity index 100% rename from karapace/avro_dataclasses/introspect.py rename to src/karapace/avro_dataclasses/introspect.py diff --git a/karapace/avro_dataclasses/models.py b/src/karapace/avro_dataclasses/models.py similarity index 100% rename from karapace/avro_dataclasses/models.py rename to src/karapace/avro_dataclasses/models.py diff --git a/karapace/avro_dataclasses/schema.py b/src/karapace/avro_dataclasses/schema.py similarity index 100% rename from karapace/avro_dataclasses/schema.py rename to src/karapace/avro_dataclasses/schema.py diff --git a/karapace/backup/__init__.py b/src/karapace/backup/__init__.py similarity index 100% rename from karapace/backup/__init__.py rename to src/karapace/backup/__init__.py diff --git a/karapace/backup/api.py b/src/karapace/backup/api.py similarity index 100% rename from karapace/backup/api.py rename to src/karapace/backup/api.py diff --git a/karapace/backup/backends/__init__.py b/src/karapace/backup/backends/__init__.py similarity index 100% rename from karapace/backup/backends/__init__.py rename to src/karapace/backup/backends/__init__.py diff --git a/karapace/backup/backends/reader.py b/src/karapace/backup/backends/reader.py similarity index 100% rename from karapace/backup/backends/reader.py rename to src/karapace/backup/backends/reader.py diff --git a/karapace/backup/backends/v1.py b/src/karapace/backup/backends/v1.py similarity index 100% rename from karapace/backup/backends/v1.py rename to src/karapace/backup/backends/v1.py diff --git a/karapace/backup/backends/v2.py b/src/karapace/backup/backends/v2.py similarity index 100% rename from karapace/backup/backends/v2.py rename to src/karapace/backup/backends/v2.py diff --git a/karapace/backup/backends/v3/__init__.py b/src/karapace/backup/backends/v3/__init__.py similarity index 100% rename from karapace/backup/backends/v3/__init__.py rename to src/karapace/backup/backends/v3/__init__.py diff --git a/karapace/backup/backends/v3/avro/DataFile.avsc b/src/karapace/backup/backends/v3/avro/DataFile.avsc similarity index 100% rename from karapace/backup/backends/v3/avro/DataFile.avsc rename to src/karapace/backup/backends/v3/avro/DataFile.avsc diff --git a/karapace/backup/backends/v3/avro/Header.avsc b/src/karapace/backup/backends/v3/avro/Header.avsc similarity index 100% rename from karapace/backup/backends/v3/avro/Header.avsc rename to src/karapace/backup/backends/v3/avro/Header.avsc diff --git a/karapace/backup/backends/v3/avro/Metadata.avsc b/src/karapace/backup/backends/v3/avro/Metadata.avsc similarity index 100% rename from karapace/backup/backends/v3/avro/Metadata.avsc rename to src/karapace/backup/backends/v3/avro/Metadata.avsc diff --git a/karapace/backup/backends/v3/avro/Record.avsc b/src/karapace/backup/backends/v3/avro/Record.avsc similarity index 100% rename from karapace/backup/backends/v3/avro/Record.avsc rename to src/karapace/backup/backends/v3/avro/Record.avsc diff --git a/karapace/backup/backends/v3/backend.py b/src/karapace/backup/backends/v3/backend.py similarity index 100% rename from karapace/backup/backends/v3/backend.py rename to src/karapace/backup/backends/v3/backend.py diff --git a/karapace/backup/backends/v3/checksum.py b/src/karapace/backup/backends/v3/checksum.py similarity index 100% rename from karapace/backup/backends/v3/checksum.py rename to src/karapace/backup/backends/v3/checksum.py diff --git a/karapace/backup/backends/v3/constants.py b/src/karapace/backup/backends/v3/constants.py similarity index 100% rename from karapace/backup/backends/v3/constants.py rename to src/karapace/backup/backends/v3/constants.py diff --git a/karapace/backup/backends/v3/errors.py b/src/karapace/backup/backends/v3/errors.py similarity index 100% rename from karapace/backup/backends/v3/errors.py rename to src/karapace/backup/backends/v3/errors.py diff --git a/karapace/backup/backends/v3/readers.py b/src/karapace/backup/backends/v3/readers.py similarity index 100% rename from karapace/backup/backends/v3/readers.py rename to src/karapace/backup/backends/v3/readers.py diff --git a/karapace/backup/backends/v3/schema.py b/src/karapace/backup/backends/v3/schema.py similarity index 100% rename from karapace/backup/backends/v3/schema.py rename to src/karapace/backup/backends/v3/schema.py diff --git a/karapace/backup/backends/v3/schema_tool.py b/src/karapace/backup/backends/v3/schema_tool.py similarity index 89% rename from karapace/backup/backends/v3/schema_tool.py rename to src/karapace/backup/backends/v3/schema_tool.py index 5bcf7664a..65a3ea2bf 100644 --- a/karapace/backup/backends/v3/schema_tool.py +++ b/src/karapace/backup/backends/v3/schema_tool.py @@ -58,6 +58,17 @@ def relative_path(path: pathlib.Path) -> pathlib.Path: return pathlib.Path(str_path[len(cwd) + 1 :]) if str_path.startswith(cwd) else path +def target_has_source_layout(git_target: str) -> bool: + with subprocess.Popen( + ["git", "show", f"{git_target}:src"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) as cp: + if cp.returncode == 128: + return False + return True + + def check_compatibility(git_target: str) -> None: errored = False found_any = False @@ -70,8 +81,13 @@ def check_compatibility(git_target: str) -> None: subprocess.run(["git", "fetch", remote, branch], check=True, capture_output=True) + # Does the target version have source layout + source_layout = target_has_source_layout(git_target) + for file in schema_directory.glob(f"*{extension}"): relative = relative_path(file) + if source_layout: + relative = pathlib.Path(*relative.parts[1:]) with subprocess.Popen( ["git", "show", f"{git_target}:{relative}"], stdout=subprocess.PIPE, diff --git a/karapace/backup/backends/v3/writers.py b/src/karapace/backup/backends/v3/writers.py similarity index 100% rename from karapace/backup/backends/v3/writers.py rename to src/karapace/backup/backends/v3/writers.py diff --git a/karapace/backup/backends/writer.py b/src/karapace/backup/backends/writer.py similarity index 100% rename from karapace/backup/backends/writer.py rename to src/karapace/backup/backends/writer.py diff --git a/karapace/backup/cli.py b/src/karapace/backup/cli.py similarity index 100% rename from karapace/backup/cli.py rename to src/karapace/backup/cli.py diff --git a/karapace/backup/encoders.py b/src/karapace/backup/encoders.py similarity index 100% rename from karapace/backup/encoders.py rename to src/karapace/backup/encoders.py diff --git a/karapace/backup/errors.py b/src/karapace/backup/errors.py similarity index 100% rename from karapace/backup/errors.py rename to src/karapace/backup/errors.py diff --git a/karapace/backup/poll_timeout.py b/src/karapace/backup/poll_timeout.py similarity index 100% rename from karapace/backup/poll_timeout.py rename to src/karapace/backup/poll_timeout.py diff --git a/karapace/backup/safe_writer.py b/src/karapace/backup/safe_writer.py similarity index 100% rename from karapace/backup/safe_writer.py rename to src/karapace/backup/safe_writer.py diff --git a/karapace/backup/topic_configurations.py b/src/karapace/backup/topic_configurations.py similarity index 100% rename from karapace/backup/topic_configurations.py rename to src/karapace/backup/topic_configurations.py diff --git a/karapace/client.py b/src/karapace/client.py similarity index 100% rename from karapace/client.py rename to src/karapace/client.py diff --git a/karapace/compatibility/__init__.py b/src/karapace/compatibility/__init__.py similarity index 100% rename from karapace/compatibility/__init__.py rename to src/karapace/compatibility/__init__.py diff --git a/karapace/compatibility/jsonschema/__init__.py b/src/karapace/compatibility/jsonschema/__init__.py similarity index 100% rename from karapace/compatibility/jsonschema/__init__.py rename to src/karapace/compatibility/jsonschema/__init__.py diff --git a/karapace/compatibility/jsonschema/checks.py b/src/karapace/compatibility/jsonschema/checks.py similarity index 100% rename from karapace/compatibility/jsonschema/checks.py rename to src/karapace/compatibility/jsonschema/checks.py diff --git a/karapace/compatibility/jsonschema/types.py b/src/karapace/compatibility/jsonschema/types.py similarity index 100% rename from karapace/compatibility/jsonschema/types.py rename to src/karapace/compatibility/jsonschema/types.py diff --git a/karapace/compatibility/jsonschema/utils.py b/src/karapace/compatibility/jsonschema/utils.py similarity index 100% rename from karapace/compatibility/jsonschema/utils.py rename to src/karapace/compatibility/jsonschema/utils.py diff --git a/karapace/compatibility/protobuf/__init__.py b/src/karapace/compatibility/protobuf/__init__.py similarity index 100% rename from karapace/compatibility/protobuf/__init__.py rename to src/karapace/compatibility/protobuf/__init__.py diff --git a/karapace/compatibility/protobuf/checks.py b/src/karapace/compatibility/protobuf/checks.py similarity index 100% rename from karapace/compatibility/protobuf/checks.py rename to src/karapace/compatibility/protobuf/checks.py diff --git a/karapace/config.py b/src/karapace/config.py similarity index 100% rename from karapace/config.py rename to src/karapace/config.py diff --git a/karapace/constants.py b/src/karapace/constants.py similarity index 100% rename from karapace/constants.py rename to src/karapace/constants.py diff --git a/karapace/coordinator/__init__.py b/src/karapace/coordinator/__init__.py similarity index 100% rename from karapace/coordinator/__init__.py rename to src/karapace/coordinator/__init__.py diff --git a/karapace/coordinator/master_coordinator.py b/src/karapace/coordinator/master_coordinator.py similarity index 100% rename from karapace/coordinator/master_coordinator.py rename to src/karapace/coordinator/master_coordinator.py diff --git a/karapace/coordinator/schema_coordinator.py b/src/karapace/coordinator/schema_coordinator.py similarity index 100% rename from karapace/coordinator/schema_coordinator.py rename to src/karapace/coordinator/schema_coordinator.py diff --git a/karapace/dataclasses.py b/src/karapace/dataclasses.py similarity index 100% rename from karapace/dataclasses.py rename to src/karapace/dataclasses.py diff --git a/karapace/dependency.py b/src/karapace/dependency.py similarity index 100% rename from karapace/dependency.py rename to src/karapace/dependency.py diff --git a/karapace/errors.py b/src/karapace/errors.py similarity index 100% rename from karapace/errors.py rename to src/karapace/errors.py diff --git a/karapace/in_memory_database.py b/src/karapace/in_memory_database.py similarity index 100% rename from karapace/in_memory_database.py rename to src/karapace/in_memory_database.py diff --git a/karapace/instrumentation/__init__.py b/src/karapace/instrumentation/__init__.py similarity index 100% rename from karapace/instrumentation/__init__.py rename to src/karapace/instrumentation/__init__.py diff --git a/karapace/instrumentation/prometheus.py b/src/karapace/instrumentation/prometheus.py similarity index 100% rename from karapace/instrumentation/prometheus.py rename to src/karapace/instrumentation/prometheus.py diff --git a/karapace/kafka/__init__.py b/src/karapace/kafka/__init__.py similarity index 100% rename from karapace/kafka/__init__.py rename to src/karapace/kafka/__init__.py diff --git a/karapace/kafka/admin.py b/src/karapace/kafka/admin.py similarity index 100% rename from karapace/kafka/admin.py rename to src/karapace/kafka/admin.py diff --git a/karapace/kafka/common.py b/src/karapace/kafka/common.py similarity index 100% rename from karapace/kafka/common.py rename to src/karapace/kafka/common.py diff --git a/karapace/kafka/consumer.py b/src/karapace/kafka/consumer.py similarity index 100% rename from karapace/kafka/consumer.py rename to src/karapace/kafka/consumer.py diff --git a/karapace/kafka/producer.py b/src/karapace/kafka/producer.py similarity index 100% rename from karapace/kafka/producer.py rename to src/karapace/kafka/producer.py diff --git a/karapace/kafka/types.py b/src/karapace/kafka/types.py similarity index 100% rename from karapace/kafka/types.py rename to src/karapace/kafka/types.py diff --git a/karapace/kafka_error_handler.py b/src/karapace/kafka_error_handler.py similarity index 100% rename from karapace/kafka_error_handler.py rename to src/karapace/kafka_error_handler.py diff --git a/karapace/kafka_rest_apis/__init__.py b/src/karapace/kafka_rest_apis/__init__.py similarity index 100% rename from karapace/kafka_rest_apis/__init__.py rename to src/karapace/kafka_rest_apis/__init__.py diff --git a/karapace/kafka_rest_apis/authentication.py b/src/karapace/kafka_rest_apis/authentication.py similarity index 100% rename from karapace/kafka_rest_apis/authentication.py rename to src/karapace/kafka_rest_apis/authentication.py diff --git a/karapace/kafka_rest_apis/consumer_manager.py b/src/karapace/kafka_rest_apis/consumer_manager.py similarity index 100% rename from karapace/kafka_rest_apis/consumer_manager.py rename to src/karapace/kafka_rest_apis/consumer_manager.py diff --git a/karapace/kafka_rest_apis/error_codes.py b/src/karapace/kafka_rest_apis/error_codes.py similarity index 100% rename from karapace/kafka_rest_apis/error_codes.py rename to src/karapace/kafka_rest_apis/error_codes.py diff --git a/karapace/kafka_rest_apis/schema_cache.py b/src/karapace/kafka_rest_apis/schema_cache.py similarity index 100% rename from karapace/kafka_rest_apis/schema_cache.py rename to src/karapace/kafka_rest_apis/schema_cache.py diff --git a/karapace/kafka_utils.py b/src/karapace/kafka_utils.py similarity index 100% rename from karapace/kafka_utils.py rename to src/karapace/kafka_utils.py diff --git a/karapace/karapace.py b/src/karapace/karapace.py similarity index 100% rename from karapace/karapace.py rename to src/karapace/karapace.py diff --git a/karapace/karapace_all.py b/src/karapace/karapace_all.py similarity index 100% rename from karapace/karapace_all.py rename to src/karapace/karapace_all.py diff --git a/karapace/key_format.py b/src/karapace/key_format.py similarity index 100% rename from karapace/key_format.py rename to src/karapace/key_format.py diff --git a/karapace/messaging.py b/src/karapace/messaging.py similarity index 100% rename from karapace/messaging.py rename to src/karapace/messaging.py diff --git a/karapace/offset_watcher.py b/src/karapace/offset_watcher.py similarity index 100% rename from karapace/offset_watcher.py rename to src/karapace/offset_watcher.py diff --git a/karapace/protobuf/__init__.py b/src/karapace/protobuf/__init__.py similarity index 100% rename from karapace/protobuf/__init__.py rename to src/karapace/protobuf/__init__.py diff --git a/karapace/protobuf/compare_result.py b/src/karapace/protobuf/compare_result.py similarity index 100% rename from karapace/protobuf/compare_result.py rename to src/karapace/protobuf/compare_result.py diff --git a/karapace/protobuf/compare_type_lists.py b/src/karapace/protobuf/compare_type_lists.py similarity index 100% rename from karapace/protobuf/compare_type_lists.py rename to src/karapace/protobuf/compare_type_lists.py diff --git a/karapace/protobuf/compare_type_storage.py b/src/karapace/protobuf/compare_type_storage.py similarity index 100% rename from karapace/protobuf/compare_type_storage.py rename to src/karapace/protobuf/compare_type_storage.py diff --git a/karapace/protobuf/encoding_variants.py b/src/karapace/protobuf/encoding_variants.py similarity index 100% rename from karapace/protobuf/encoding_variants.py rename to src/karapace/protobuf/encoding_variants.py diff --git a/karapace/protobuf/enum_constant_element.py b/src/karapace/protobuf/enum_constant_element.py similarity index 100% rename from karapace/protobuf/enum_constant_element.py rename to src/karapace/protobuf/enum_constant_element.py diff --git a/karapace/protobuf/enum_element.py b/src/karapace/protobuf/enum_element.py similarity index 100% rename from karapace/protobuf/enum_element.py rename to src/karapace/protobuf/enum_element.py diff --git a/karapace/protobuf/exception.py b/src/karapace/protobuf/exception.py similarity index 100% rename from karapace/protobuf/exception.py rename to src/karapace/protobuf/exception.py diff --git a/karapace/protobuf/extend_element.py b/src/karapace/protobuf/extend_element.py similarity index 100% rename from karapace/protobuf/extend_element.py rename to src/karapace/protobuf/extend_element.py diff --git a/karapace/protobuf/extensions_element.py b/src/karapace/protobuf/extensions_element.py similarity index 100% rename from karapace/protobuf/extensions_element.py rename to src/karapace/protobuf/extensions_element.py diff --git a/karapace/protobuf/field.py b/src/karapace/protobuf/field.py similarity index 100% rename from karapace/protobuf/field.py rename to src/karapace/protobuf/field.py diff --git a/karapace/protobuf/field_element.py b/src/karapace/protobuf/field_element.py similarity index 100% rename from karapace/protobuf/field_element.py rename to src/karapace/protobuf/field_element.py diff --git a/karapace/protobuf/group_element.py b/src/karapace/protobuf/group_element.py similarity index 100% rename from karapace/protobuf/group_element.py rename to src/karapace/protobuf/group_element.py diff --git a/karapace/protobuf/io.py b/src/karapace/protobuf/io.py similarity index 100% rename from karapace/protobuf/io.py rename to src/karapace/protobuf/io.py diff --git a/karapace/protobuf/known_dependency.py b/src/karapace/protobuf/known_dependency.py similarity index 100% rename from karapace/protobuf/known_dependency.py rename to src/karapace/protobuf/known_dependency.py diff --git a/karapace/protobuf/kotlin_wrapper.py b/src/karapace/protobuf/kotlin_wrapper.py similarity index 100% rename from karapace/protobuf/kotlin_wrapper.py rename to src/karapace/protobuf/kotlin_wrapper.py diff --git a/karapace/protobuf/location.py b/src/karapace/protobuf/location.py similarity index 100% rename from karapace/protobuf/location.py rename to src/karapace/protobuf/location.py diff --git a/karapace/protobuf/message_element.py b/src/karapace/protobuf/message_element.py similarity index 100% rename from karapace/protobuf/message_element.py rename to src/karapace/protobuf/message_element.py diff --git a/karapace/protobuf/one_of_element.py b/src/karapace/protobuf/one_of_element.py similarity index 100% rename from karapace/protobuf/one_of_element.py rename to src/karapace/protobuf/one_of_element.py diff --git a/karapace/protobuf/option_element.py b/src/karapace/protobuf/option_element.py similarity index 100% rename from karapace/protobuf/option_element.py rename to src/karapace/protobuf/option_element.py diff --git a/karapace/protobuf/option_reader.py b/src/karapace/protobuf/option_reader.py similarity index 100% rename from karapace/protobuf/option_reader.py rename to src/karapace/protobuf/option_reader.py diff --git a/karapace/protobuf/proto_file_element.py b/src/karapace/protobuf/proto_file_element.py similarity index 100% rename from karapace/protobuf/proto_file_element.py rename to src/karapace/protobuf/proto_file_element.py diff --git a/karapace/protobuf/proto_normalizations.py b/src/karapace/protobuf/proto_normalizations.py similarity index 100% rename from karapace/protobuf/proto_normalizations.py rename to src/karapace/protobuf/proto_normalizations.py diff --git a/karapace/protobuf/proto_parser.py b/src/karapace/protobuf/proto_parser.py similarity index 100% rename from karapace/protobuf/proto_parser.py rename to src/karapace/protobuf/proto_parser.py diff --git a/karapace/protobuf/proto_type.py b/src/karapace/protobuf/proto_type.py similarity index 100% rename from karapace/protobuf/proto_type.py rename to src/karapace/protobuf/proto_type.py diff --git a/karapace/protobuf/protobuf_to_dict.py b/src/karapace/protobuf/protobuf_to_dict.py similarity index 100% rename from karapace/protobuf/protobuf_to_dict.py rename to src/karapace/protobuf/protobuf_to_dict.py diff --git a/karapace/protobuf/protopace/__init__.py b/src/karapace/protobuf/protopace/__init__.py similarity index 100% rename from karapace/protobuf/protopace/__init__.py rename to src/karapace/protobuf/protopace/__init__.py diff --git a/karapace/protobuf/protopace/protopace.py b/src/karapace/protobuf/protopace/protopace.py similarity index 100% rename from karapace/protobuf/protopace/protopace.py rename to src/karapace/protobuf/protopace/protopace.py diff --git a/karapace/protobuf/reserved_element.py b/src/karapace/protobuf/reserved_element.py similarity index 100% rename from karapace/protobuf/reserved_element.py rename to src/karapace/protobuf/reserved_element.py diff --git a/karapace/protobuf/rpc_element.py b/src/karapace/protobuf/rpc_element.py similarity index 100% rename from karapace/protobuf/rpc_element.py rename to src/karapace/protobuf/rpc_element.py diff --git a/karapace/protobuf/schema.py b/src/karapace/protobuf/schema.py similarity index 100% rename from karapace/protobuf/schema.py rename to src/karapace/protobuf/schema.py diff --git a/karapace/protobuf/serialization.py b/src/karapace/protobuf/serialization.py similarity index 100% rename from karapace/protobuf/serialization.py rename to src/karapace/protobuf/serialization.py diff --git a/karapace/protobuf/service_element.py b/src/karapace/protobuf/service_element.py similarity index 100% rename from karapace/protobuf/service_element.py rename to src/karapace/protobuf/service_element.py diff --git a/karapace/protobuf/syntax.py b/src/karapace/protobuf/syntax.py similarity index 100% rename from karapace/protobuf/syntax.py rename to src/karapace/protobuf/syntax.py diff --git a/karapace/protobuf/syntax_reader.py b/src/karapace/protobuf/syntax_reader.py similarity index 100% rename from karapace/protobuf/syntax_reader.py rename to src/karapace/protobuf/syntax_reader.py diff --git a/karapace/protobuf/type_element.py b/src/karapace/protobuf/type_element.py similarity index 100% rename from karapace/protobuf/type_element.py rename to src/karapace/protobuf/type_element.py diff --git a/karapace/protobuf/type_tree.py b/src/karapace/protobuf/type_tree.py similarity index 100% rename from karapace/protobuf/type_tree.py rename to src/karapace/protobuf/type_tree.py diff --git a/karapace/protobuf/utils.py b/src/karapace/protobuf/utils.py similarity index 100% rename from karapace/protobuf/utils.py rename to src/karapace/protobuf/utils.py diff --git a/karapace/rapu.py b/src/karapace/rapu.py similarity index 100% rename from karapace/rapu.py rename to src/karapace/rapu.py diff --git a/karapace/schema_models.py b/src/karapace/schema_models.py similarity index 100% rename from karapace/schema_models.py rename to src/karapace/schema_models.py diff --git a/karapace/schema_reader.py b/src/karapace/schema_reader.py similarity index 100% rename from karapace/schema_reader.py rename to src/karapace/schema_reader.py diff --git a/karapace/schema_references.py b/src/karapace/schema_references.py similarity index 100% rename from karapace/schema_references.py rename to src/karapace/schema_references.py diff --git a/karapace/schema_registry.py b/src/karapace/schema_registry.py similarity index 100% rename from karapace/schema_registry.py rename to src/karapace/schema_registry.py diff --git a/karapace/schema_registry_apis.py b/src/karapace/schema_registry_apis.py similarity index 100% rename from karapace/schema_registry_apis.py rename to src/karapace/schema_registry_apis.py diff --git a/karapace/schema_type.py b/src/karapace/schema_type.py similarity index 100% rename from karapace/schema_type.py rename to src/karapace/schema_type.py diff --git a/karapace/sentry/__init__.py b/src/karapace/sentry/__init__.py similarity index 100% rename from karapace/sentry/__init__.py rename to src/karapace/sentry/__init__.py diff --git a/karapace/sentry/sentry_client.py b/src/karapace/sentry/sentry_client.py similarity index 100% rename from karapace/sentry/sentry_client.py rename to src/karapace/sentry/sentry_client.py diff --git a/karapace/sentry/sentry_client_api.py b/src/karapace/sentry/sentry_client_api.py similarity index 100% rename from karapace/sentry/sentry_client_api.py rename to src/karapace/sentry/sentry_client_api.py diff --git a/karapace/serialization.py b/src/karapace/serialization.py similarity index 100% rename from karapace/serialization.py rename to src/karapace/serialization.py diff --git a/karapace/statsd.py b/src/karapace/statsd.py similarity index 100% rename from karapace/statsd.py rename to src/karapace/statsd.py diff --git a/karapace/typing.py b/src/karapace/typing.py similarity index 100% rename from karapace/typing.py rename to src/karapace/typing.py diff --git a/karapace/utils.py b/src/karapace/utils.py similarity index 100% rename from karapace/utils.py rename to src/karapace/utils.py From 81e656388fd693d0ef7efe053e45c3daa2d4e900 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Fri, 13 Sep 2024 15:11:53 +0300 Subject: [PATCH 3/4] chore: change links to Aiven-Open/karapace --- .github/workflows/container.yml | 2 +- CONTRIBUTING.md | 8 ++++---- README.rst | 8 ++++---- website/source/_templates/hero-texts.html | 2 +- website/source/_templates/page.html | 2 +- website/source/install.rst | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index de1af9b71..0ed6475e2 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -66,7 +66,7 @@ jobs: labels: | org.opencontainers.image.authors=opensource@aiven.io org.opencontainers.image.url=https://karapace.io - org.opencontainers.image.documentation=https://github.com/aiven/karapace/ + org.opencontainers.image.documentation=https://github.com/Aiven-Open/karapace/ org.opencontainers.image.vendor=Aiven org.opencontainers.image.licenses=Apache-2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2aafcf1b..4437a06fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,8 +12,8 @@ Contributions are very welcome on Karapace. When contributing please keep this i There is very little you need to get started coding for Karapace: -- Use [one of the supported python versions](https://github.com/aiven/karapace/blob/master/setup.py) - documented in the `setup.py` classifiers. +- Use [one of the supported python versions](https://github.com/Aiven-Open/karapace/blob/master/pyproject.toml) + documented in the `project:requires-python`. - Create [a virtual environment](https://docs.python.org/3/tutorial/venv.html) and install the dev dependencies in it: ```python @@ -74,7 +74,7 @@ The code is statically checked and formatted using [a few tools][requirements-de To run these automatically on each commit please enable the [pre-commit](https://pre-commit.com) hooks. -[requirements-dev]: https://github.com/aiven/karapace/blob/master/requirements/requirements-dev.txt +[requirements-dev]: https://github.com/Aiven-Open/karapace/blob/master/requirements/requirements-dev.txt ## Manual testing @@ -89,7 +89,7 @@ karapace karapace.config.json ### Configuration To see descriptions of configuration keys see our -[README](https://github.com/aiven/karapace#configuration-keys). +[README](https://github.com/Aiven-Open/karapace#configuration-keys). Each configuration key can be overridden with an environment variable prefixed with `KARAPACE_`, exception being configuration keys that actually start with the `karapace` string. For example, to diff --git a/README.rst b/README.rst index 4d8992d0d..adc5b0e8d 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Karapace ``karapace``. Your Apache Kafka® essentials in one tool. -An `open-source `_ implementation +An `open-source `_ implementation of `Kafka REST `_ and `Schema Registry `_. @@ -81,7 +81,7 @@ override the ``bootstrap_uri`` config value, one would use the environment varia ``KARAPACE_BOOTSTRAP_URI``. Here_ you can find an example configuration file to give you an idea what you need to change. -.. _`Here`: https://github.com/aiven/karapace/blob/master/karapace.config.json +.. _`Here`: https://github.com/Aiven-Open/karapace/blob/master/karapace.config.json Source install -------------- @@ -711,7 +711,7 @@ Contact ======= Bug reports and patches are very welcome, please post them as GitHub issues -and pull requests at https://github.com/aiven/karapace . Any possible +and pull requests at https://github.com/Aiven-Open/karapace . Any possible vulnerabilities or other serious issues should be reported directly to the maintainers . @@ -732,6 +732,6 @@ to them for pioneering the concept. .. _`Aiven`: https://aiven.io/ Recent contributors are listed on the GitHub project page, -https://github.com/aiven/karapace/graphs/contributors +https://github.com/Aiven-Open/karapace/graphs/contributors Copyright ⓒ 2021 Aiven Ltd. diff --git a/website/source/_templates/hero-texts.html b/website/source/_templates/hero-texts.html index b7cf04e34..383385932 100644 --- a/website/source/_templates/hero-texts.html +++ b/website/source/_templates/hero-texts.html @@ -11,7 +11,7 @@

{% include 'github-ribbon.html' %} Learn more about Karapace diff --git a/website/source/_templates/page.html b/website/source/_templates/page.html index add72485f..a9bcb465e 100644 --- a/website/source/_templates/page.html +++ b/website/source/_templates/page.html @@ -1,7 +1,7 @@ {% extends '!page.html' %} {% block body %} {{ super() }} Fork me on GitHub! diff --git a/website/source/install.rst b/website/source/install.rst index 2779668a2..ac6d5459f 100644 --- a/website/source/install.rst +++ b/website/source/install.rst @@ -30,7 +30,7 @@ override the ``bootstrap_uri`` config value, one would use the environment varia ``KARAPACE_BOOTSTRAP_URI``. Here_ you can find an example configuration file to give you an idea what you need to change. -.. _`Here`: https://github.com/aiven/karapace/blob/main/karapace.config.json +.. _`Here`: https://github.com/Aiven-Open/karapace/blob/main/karapace.config.json Source install -------------- From d2cc568ee88c1d635cc1b2b115b4af6be9346d75 Mon Sep 17 00:00:00 2001 From: Jarkko Jaakola Date: Wed, 25 Sep 2024 14:49:54 +0300 Subject: [PATCH 4/4] chore: remove pyenv requirement --- GNUmakefile | 8 +------- README.rst | 6 ------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 9d318a5cc..7f9c90191 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -3,12 +3,6 @@ SHELL := /usr/bin/env bash VENV_DIR ?= $(CURDIR)/venv PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv PYTHON ?= python3 -ifdef CI -PYENV ?= $(PYTHON) -else -PYENV ?= pyenv exec python -endif - PYTHON_VERSION ?= 3.8 define PIN_VERSIONS_COMMAND @@ -36,7 +30,7 @@ all: version venv: venv/.make venv/.make: rm -fr '$(VENV_DIR)' - $(PYENV) -m venv '$(VENV_DIR)' + $(PYTHON) -m venv '$(VENV_DIR)' $(PIP) install --upgrade pip touch '$(@)' diff --git a/README.rst b/README.rst index adc5b0e8d..d1bcbd28f 100644 --- a/README.rst +++ b/README.rst @@ -664,12 +664,6 @@ and install the required software for development. Use ``make unit-tests`` and ``make test`` to execute both. You can set ``PYTEST_ARGS`` to customize the execution (e.g. ``PYTEST_ARGS=--maxfail=1 make test``). -By default ``pyenv`` is expected to be installed and in ``PATH``. This ensures -on all platforms that arbitrary Python versions can be used for development. It -is possible to overwrite this by setting ``PYENV`` to something else (e.g. -``PYENV=python3 make venv`` to simply use the global Python executable). The -default Python version is defined in ``.python-version``. - Karapace currently depends on various system software to be installed. The installation of these is automated for some operation systems, but not all. At the time of writing Java, the Protobuf Compiler, and the Snappy shared library