From 517b99f576c51b3c359b9bdb96591aa5794497d0 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Sun, 13 Feb 2022 21:56:37 +0000 Subject: [PATCH 1/8] add disabled 'add_index' and 'remove_index' method for schema editor --- django_redshift_backend/base.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/django_redshift_backend/base.py b/django_redshift_backend/base.py index d4381e9..3caa993 100644 --- a/django_redshift_backend/base.py +++ b/django_redshift_backend/base.py @@ -135,7 +135,15 @@ def alter_index_together(self, model, old_index_together, new_index_together): # Redshift doesn't support INDEX. return - def _create_index_sql(self, model, fields, suffix="", sql=None): + def add_index(self, model, index, concurrently=False): + # Redshift doesn't support INDEX. + pass + + def remove_index(self, model, index, concurrently=False): + # Redshift doesn't support INDEX. + pass + + def _create_index_sql(self, model, **kwargs): raise NotSupportedError("Redshift doesn't support INDEX") def create_model(self, model): From 81be2b0a78046db121d5d588ea93b9ddfb69c9a1 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Sun, 13 Feb 2022 22:04:14 +0000 Subject: [PATCH 2/8] testing with examples/proj1 should run always --- .github/workflows/test-examples-proj1.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test-examples-proj1.yml b/.github/workflows/test-examples-proj1.yml index cfb20bc..fda9359 100644 --- a/.github/workflows/test-examples-proj1.yml +++ b/.github/workflows/test-examples-proj1.yml @@ -1,12 +1,6 @@ name: Test examples/proj1 -on: - push: - paths: - - 'examples/proj1/*' - pull_request: - paths: - - 'examples/proj1/*' +on: [push, pull_request] jobs: build: From 3454bf9baf151cdaec6b76cb2e9392aa6bdcba12 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Sun, 13 Feb 2022 22:55:16 +0000 Subject: [PATCH 3/8] add debug message --- django_redshift_backend/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/django_redshift_backend/base.py b/django_redshift_backend/base.py index 3caa993..ae43b22 100644 --- a/django_redshift_backend/base.py +++ b/django_redshift_backend/base.py @@ -131,6 +131,10 @@ def _create_like_index_sql(self, model, field): # Redshift doesn't support INDEX. return None + def _create_index_sql(self, model, **kwargs): + # _create_index_sql only called from _create_like_index_sql on django/db/backends/postgresql/schema.py + raise NotSupportedError("Redshift doesn't support INDEX") + def alter_index_together(self, model, old_index_together, new_index_together): # Redshift doesn't support INDEX. return @@ -143,9 +147,6 @@ def remove_index(self, model, index, concurrently=False): # Redshift doesn't support INDEX. pass - def _create_index_sql(self, model, **kwargs): - raise NotSupportedError("Redshift doesn't support INDEX") - def create_model(self, model): """ Takes a model and creates a table for it in the database. From 84069ba8c62dbc67190d94bd3c0aff7f331c9608 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Mon, 14 Feb 2022 01:52:23 +0000 Subject: [PATCH 4/8] fix flake8 --- django_redshift_backend/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_redshift_backend/base.py b/django_redshift_backend/base.py index ae43b22..5044db0 100644 --- a/django_redshift_backend/base.py +++ b/django_redshift_backend/base.py @@ -132,7 +132,8 @@ def _create_like_index_sql(self, model, field): return None def _create_index_sql(self, model, **kwargs): - # _create_index_sql only called from _create_like_index_sql on django/db/backends/postgresql/schema.py + # _create_index_sql only called from _create_like_index_sql + # on django/db/backends/postgresql/schema.py raise NotSupportedError("Redshift doesn't support INDEX") def alter_index_together(self, model, old_index_together, new_index_together): From 43dd872e0bbcc71270a0733e4d0e47e42fe8cd48 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Mon, 14 Feb 2022 05:47:42 +0000 Subject: [PATCH 5/8] add test for sqlmigrate with use postgres --- .github/workflows/test.yml | 11 +++++ doc/dev.rst | 6 +++ setup.cfg | 1 + tests/docker-compose.yml | 12 +++++ tests/test_redshift_backend.py | 17 +++++++ tests/testapp/migrations/0001_initial.py | 63 ++++++++++++++++++++++++ tests/testapp/migrations/__init__.py | 0 tox.ini | 1 + 8 files changed, 111 insertions(+) create mode 100644 tests/docker-compose.yml create mode 100644 tests/testapp/migrations/0001_initial.py create mode 100644 tests/testapp/migrations/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8981066..074c947 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,16 @@ jobs: - django-version: 'main' python-version: '3.10' + services: + postgres: + image: postgres:9.6-alpine + env: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: testing + ports: + - 5439:5432 + steps: - uses: actions/checkout@v2 @@ -50,6 +60,7 @@ jobs: tox -v env: DJANGO: ${{ matrix.django-version }} + TEST_WITH_POSTGRES: 1 - name: Upload coverage uses: codecov/codecov-action@v1 diff --git a/doc/dev.rst b/doc/dev.rst index 7aebdc7..0ff9f64 100644 --- a/doc/dev.rst +++ b/doc/dev.rst @@ -35,6 +35,12 @@ Just run tox:: tox have several sections for testing. +To test the database migration as well, start postgres and test it as follows:: + + $ cd tests + $ docker-compose up -d + $ TEST_WITH_POSTGRES=1 tox + CI (Continuous Integration) ---------------------------- diff --git a/setup.cfg b/setup.cfg index 8832492..0b68b22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,3 +53,4 @@ universal = 0 [flake8] max-line-length=90 ignore = W504 +exclude = tests/testapp/migrations diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..a1396bf --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.1' + +services: + db: + image: postgres:9.6-alpine + restart: always + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: testing + ports: + - 5439:5432 diff --git a/tests/test_redshift_backend.py b/tests/test_redshift_backend.py index 2c14e38..cc42348 100644 --- a/tests/test_redshift_backend.py +++ b/tests/test_redshift_backend.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- +import os import unittest from django.db import connections from django.db.utils import NotSupportedError from django.core.management.color import no_style +from django.db.migrations.loader import MigrationLoader +import pytest def norm_sql(sql): @@ -153,3 +156,17 @@ def test_create_model(self): def test_create_table_meta_keys(self): from testapp.models import TestModelWithMetaKeys self.check_model_creation(TestModelWithMetaKeys, expected_ddl_meta_keys) + + @pytest.mark.skipif(not os.environ.get('TEST_WITH_POSTGRES'), + reason='postgres database is not exist') + def test_sqlmigrate(self): + from django.db import connection + loader = MigrationLoader(connection, replace_migrations=True, load=True) + app_label, migration_name = 'testapp', '0001' + migration = loader.get_migration_by_prefix(app_label, migration_name) + target = (app_label, migration.name) + + plan = [(loader.graph.nodes[target], False)] + sql_statements = loader.collect_sql(plan) + print('\n'.join(sql_statements)) + assert sql_statements # It doesn't matter what SQL is generated. diff --git a/tests/testapp/migrations/0001_initial.py b/tests/testapp/migrations/0001_initial.py new file mode 100644 index 0000000..6e3d97d --- /dev/null +++ b/tests/testapp/migrations/0001_initial.py @@ -0,0 +1,63 @@ +# Generated by Django 3.2.12 on 2022-02-13 21:03 + +from django.db import migrations, models +import django.db.models.deletion +import django_redshift_backend.distkey + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TestModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ctime', models.DateTimeField()), + ('text', models.TextField()), + ('uuid', models.UUIDField()), + ], + ), + migrations.CreateModel( + name='TestParentModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('age', models.IntegerField()), + ], + ), + migrations.CreateModel( + name='TestReferencedModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='TestModelWithMetaKeys', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100)), + ('age', models.IntegerField()), + ('created_at', models.DateTimeField()), + ('fk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='testapp.testreferencedmodel')), + ], + options={ + 'ordering': ['created_at', '-id'], + }, + ), + migrations.CreateModel( + name='TestChildModel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('age', models.IntegerField()), + ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='testapp.testparentmodel')), + ], + ), + migrations.AddIndex( + model_name='testmodelwithmetakeys', + index=django_redshift_backend.distkey.DistKey(fields=['fk'], name='testapp_tes_fk_id_cd99f5_idx'), + ), + ] diff --git a/tests/testapp/migrations/__init__.py b/tests/testapp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tox.ini b/tox.ini index ecc21c2..8400461 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,7 @@ deps = setenv = DJANGO_SETTINGS_MODULE = settings PYTHONPATH = {toxinidir} + TEST_WITH_POSTGRES = {env:TEST_WITH_POSTGRES:} pip_pre = True commands = pytest -v --cov django_redshift_backend --cov-append --cov-report term-missing --cov-report=xml {posargs} From b4c209d0b5c81c21e0100fe770ced735fd6e40f2 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Mon, 14 Feb 2022 05:55:34 +0000 Subject: [PATCH 6/8] fix for django 2.2 --- tests/test_redshift_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_redshift_backend.py b/tests/test_redshift_backend.py index cc42348..5811e79 100644 --- a/tests/test_redshift_backend.py +++ b/tests/test_redshift_backend.py @@ -161,7 +161,7 @@ def test_create_table_meta_keys(self): reason='postgres database is not exist') def test_sqlmigrate(self): from django.db import connection - loader = MigrationLoader(connection, replace_migrations=True, load=True) + loader = MigrationLoader(connection) app_label, migration_name = 'testapp', '0001' migration = loader.get_migration_by_prefix(app_label, migration_name) target = (app_label, migration.name) From e737e8273f3d3cfb857198864e20c1105d3abe66 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Mon, 14 Feb 2022 07:59:04 +0000 Subject: [PATCH 7/8] fix for django 2.2 --- tests/test_redshift_backend.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/test_redshift_backend.py b/tests/test_redshift_backend.py index 5811e79..a12e9da 100644 --- a/tests/test_redshift_backend.py +++ b/tests/test_redshift_backend.py @@ -3,6 +3,7 @@ import os import unittest +import django from django.db import connections from django.db.utils import NotSupportedError from django.core.management.color import no_style @@ -158,15 +159,25 @@ def test_create_table_meta_keys(self): self.check_model_creation(TestModelWithMetaKeys, expected_ddl_meta_keys) @pytest.mark.skipif(not os.environ.get('TEST_WITH_POSTGRES'), - reason='postgres database is not exist') + reason='to run, TEST_WITH_POSTGRES=1 tox') def test_sqlmigrate(self): from django.db import connection - loader = MigrationLoader(connection) + + if django.VERSION < (3, 0): # for dj22 + from django.db.migrations.executor import MigrationExecutor + executor = MigrationExecutor(connection) + loader = executor.loader + collect_sql = executor.collect_sql + else: + from django.db.migrations.loader import MigrationLoader + loader = MigrationLoader(connection) + collect_sql = loader.collect_sql + app_label, migration_name = 'testapp', '0001' migration = loader.get_migration_by_prefix(app_label, migration_name) target = (app_label, migration.name) plan = [(loader.graph.nodes[target], False)] - sql_statements = loader.collect_sql(plan) + sql_statements = collect_sql(plan) print('\n'.join(sql_statements)) assert sql_statements # It doesn't matter what SQL is generated. From d1f9652934113253e9c6b7bf6e9822fb6fe745c5 Mon Sep 17 00:00:00 2001 From: Takayuki SHIMIZUKAWA Date: Mon, 14 Feb 2022 08:12:42 +0000 Subject: [PATCH 8/8] fix lint --- tests/test_redshift_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_redshift_backend.py b/tests/test_redshift_backend.py index a12e9da..9d2e74a 100644 --- a/tests/test_redshift_backend.py +++ b/tests/test_redshift_backend.py @@ -7,7 +7,6 @@ from django.db import connections from django.db.utils import NotSupportedError from django.core.management.color import no_style -from django.db.migrations.loader import MigrationLoader import pytest