GitHub actions testing (#29)

1. Moved testing from TravisCI to GitHub Actions
2. Added linter and fixed most style errors
3. Added development section to README
4. Added docker testing environment

.dockerignore

@@ -0,0 +1,15 @@
+# Docs
+docs/
+
+# Python cache files
+**/__pycache__/
+
+# Private and public keys
+*.key
+*.ppk
+*.pub
+
+# Hidden apps directories
+.github/
+.idea/
+.gitignore

.github/workflows/python-tests.yml

@@ -0,0 +1,90 @@
+name: Python unit tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        postgres-version: ["9.6", "10", "11", "12"]
+        django-version: ["2.1", "2.2", "3.0", "3.1", "3.2"]
+        clickhouse-version: ["latest"]
+        redis-version: ["latest"]
+
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres-version }}
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      clickhouse:
+        image: yandex/clickhouse-server:${{ matrix.clickhouse-version }}
+        ports:
+          - 8123:8123
+
+      redis:
+        image: redis:${{ matrix.redis-version }}
+        ports:
+          - 6379:6379
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        # This path is specific to Ubuntu
+        path: ~/.cache/pip
+        # Look to see if there is a cache hit for the corresponding requirements file
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements-test.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+          ${{ runner.os }}-
+      
+    - name: Install pip dependencies
+      run: |
+        python -m pip install --upgrade pip wheel setuptools
+        python -m pip install -r requirements-test.txt
+        python -m pip install -U django==${{ matrix.django-version }}.*
+        python setup.py -q install
+      
+    - name: Lint with flake8
+      run: |       
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude=build/
+        
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Set up test databases
+      run: |
+        psql -tc 'SHOW server_version' -U postgres -h localhost
+        psql -c 'CREATE ROLE test;' -U postgres -h localhost
+        psql -c 'ALTER ROLE test WITH SUPERUSER;' -U postgres -h localhost
+        psql -c 'ALTER ROLE test WITH LOGIN;' -U postgres -h localhost
+        psql -c "ALTER ROLE test PASSWORD 'test';" -U postgres -h localhost
+        psql -c 'CREATE DATABASE test OWNER test;' -U postgres -h localhost
+        psql -c 'CREATE DATABASE test2 OWNER test;' -U postgres -h localhost
+      env:
+        PGPASSWORD: postgres
+
+    - name: Test with unittest
+      run: |
+        python runtests.py

Dockerfile

@@ -0,0 +1,47 @@
+ARG PYTHON_IMAGE_TAG=latest
+
+FROM python:${PYTHON_IMAGE_TAG}  AS image_stage
+
+ARG APP_TAG="1.0.3"
+
+LABEL \
+  org.label-schema.build-date=Now \
+  org.label-schema.maintainer="m1ha@carrotquest.io" \
+  org.label-schema.schema-version="1.0.0-rc1" \
+  org.label-schema.vcs-ref="v${APP_TAG}" \
+  org.label-schema.vcs-url="https://github.com/carrotquest/django-clickhouse" \
+  org.label-schema.vendor="Carrot quest" \
+  org.label-schema.version="${APP_TAG}"
+
+ENV APP_UID ${APP_UID:-1000}
+ENV APP_GID ${APP_GID:-1000}
+ENV APP_NAME ${APP_NAME:-"app"}
+
+# Configure utf-8 locales to make sure Python correctly handles unicode filenames
+# Configure pip local path to copy data from pip_stage
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 DJANGO_SETTINGS_MODULE=tests.settings PYTHONUSERBASE=/pip PATH=/pip/bin:$PATH
+
+RUN set -eu && \
+  groupadd --gid "${APP_GID}" "app" && \
+  useradd --uid ${APP_UID} --gid ${APP_GID} --create-home --shell /bin/bash -d /app app && \
+  mkdir -p /pip && \
+  chmod  755 /app /pip && \
+  chown -R ${APP_UID}:${APP_GID} /app /pip
+
+WORKDIR /app/src
+
+# Install dependencies
+# set -eu "breaks" pipeline on first error
+COPY ./requirements-test.txt /app/requirements-test.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+  set -eu && \
+  python3 -m pip install --upgrade pip setuptools wheel  && \
+  python3 -m pip install --upgrade --requirement /app/requirements-test.txt
+
+COPY . /app/src
+
+RUN python3 setup.py -q install --user
+
+USER ${APP_UID}
+
+CMD ["python3", "runtests.py"]

docker-compose.yml

@@ -0,0 +1,39 @@
+---
+version: "3.9"
+services:
+  redis_db:
+    image: redis
+    command: [sh, -c, "redis-server --save '' --appendonly no"]  # disable persistence
+    mem_limit: 512m
+    cpus: 1
+
+  postgres_db:
+    image: postgres
+    environment:
+      - POSTGRES_PASSWORD=postgres
+    mem_limit: 1g
+    cpus: 1
+
+  clickhouse_db:
+    image: yandex/clickhouse-server
+    mem_limit: 1g
+    cpus: 1
+
+  run_tests:
+    image: django-clickhouse
+    build:
+      context: .
+      args:
+        - PYTHON_VER=latest
+    environment:
+      - REDIS_HOST=redis_db
+      - PGHOST=postgres_db
+      - PGUSER=postgres
+      - PGPASS=postgres
+      - "CLICK_HOUSE_HOST=http://clickhouse_db:8123"
+    depends_on:
+      - redis_db
+      - postgres_db
+      - clickhouse_db
+    mem_limit: 1g
+    cpus: 1

docs/development.md

@@ -0,0 +1,30 @@
+# Development
+## Basic info
+This is an Open source project developed by `Carrot quest` team under MIT license. 
+Feel free to create issues and make pull requests.  
+Query and database system wraps [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) library. 
+If you want to develop QuerySet system, it is better to contribute there.  
+
+
+## General info about testing
+Library test system is based on [django.test](https://docs.djangoproject.com/en/3.2/topics/testing/overview/).
+You can find them in `tests` directory. 
+
+## Tests requirements  
+* [Redis](https://redis.io/)  
+* [Yandex ClickHouse](https://clickhouse.yandex/)  
+* [PostgreSQL](https://www.postgresql.org/)  
+* Pypi libraries listed in `requirements-test.txt` file
+
+## Running tests
+### Running in docker
+1. Install [docker and docker-compose](https://www.docker.com/)  
+2. Run `docker-compose run run_tests` in project directory
+
+### Running in virtual environment
+1. Install all requirements listed above 
+2. [Create virtual environment](https://docs.python.org/3/tutorial/venv.html)
+3. Install requirements  
+  `pip3 install -U -r requirements-test.txt`  
+4. Start tests  
+  `python3 runtests.py`

docs/index.md

@@ -20,3 +20,4 @@
         * [RedisStorage](storages.md#redisstorage)
     * [Monitoring](monitoring.md)
     * [Performance notes](performance.md)
+* [Development](development.md)

requirements-test.txt

@@ -8,4 +8,7 @@ celery
 statsd
 django-pg-returning
 django-pg-bulk-update
-redis
+redis
+
+# Linter
+flake8

runtests.py

@@ -18,6 +18,6 @@
     os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.settings'
     django.setup()
     TestRunner = get_runner(settings)
-    test_runner = TestRunner()
+    test_runner = TestRunner(interactive=False)
     failures = test_runner.run_tests(["tests"])
     sys.exit(bool(failures))

src/django_clickhouse/clickhouse_models.py

@@ -94,7 +94,7 @@ def get_tuple_class(cls, field_names=None, defaults=None):
         return namedtuple("%sTuple" % cls.__name__, field_names, defaults=default_values)
 
     @classmethod
-    def objects_in(cls, database: Database)-> QuerySet:
+    def objects_in(cls, database: Database) -> QuerySet:
         return QuerySet(cls, database)
 
     @classmethod
@@ -121,7 +121,7 @@ def get_database(cls, for_write: bool = False) -> Database:
         return connections[db_alias]
 
     @classmethod
-    def get_django_model_serializer(cls, writable: bool= False, defaults: Optional[dict] = None
+    def get_django_model_serializer(cls, writable: bool = False, defaults: Optional[dict] = None
                                     ) -> Django2ClickHouseModelSerializer:
         serializer_cls = lazy_class_import(cls.django_model_serializer)
         return serializer_cls(cls, writable=writable, defaults=defaults)

src/django_clickhouse/compatibility.py

@@ -28,7 +28,7 @@ def django_pg_returning_available(using: str) -> bool:
     :return: Boolean
     """
     try:
-        import django_pg_returning
+        import django_pg_returning  # noqa: F401
         return connections[using].vendor == 'postgresql'
     except ImportError:
         return False