feat: add support for json type

django_spanner/__init__.py

@@ -12,12 +12,14 @@
 from uuid import uuid4
 
 import pkg_resources
+from google.cloud.spanner_v1 import JsonObject
 from django.db.models.fields import (
     AutoField,
     SmallAutoField,
     BigAutoField,
     Field,
 )
+from django.db.models import JSONField
 
 # Monkey-patch google.DatetimeWithNanoseconds's __eq__ compare against
 # datetime.datetime.
@@ -59,6 +61,17 @@ def autofield_init(self, *args, **kwargs):
 SmallAutoField.validators = []
 BigAutoField.validators = []
 
+
+def get_prep_value(self, value):
+    # Json encoding and decoding for spanner is done in python-spanner.
+    if not isinstance(value, JsonObject) and isinstance(value, dict):
+        return JsonObject(value)
+
+    return value
+
+
+JSONField.get_prep_value = get_prep_value
+
 old_datetimewithnanoseconds_eq = getattr(
     DatetimeWithNanoseconds, "__eq__", None
 )

django_spanner/base.py

@@ -34,6 +34,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         "DateField": "DATE",
         "DateTimeField": "TIMESTAMP",
         "DecimalField": "NUMERIC",
+        "JSONField": "JSON",
         "DurationField": "INT64",
         "EmailField": "STRING(%(max_length)s)",
         "FileField": "STRING(%(max_length)s)",

django_spanner/features.py

@@ -8,6 +8,7 @@
 
 from django.db.backends.base.features import BaseDatabaseFeatures
 from django.db.utils import InterfaceError
+from django_spanner import USE_EMULATOR
 
 
 class DatabaseFeatures(BaseDatabaseFeatures):
@@ -34,8 +35,11 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     supports_column_check_constraints = True
     supports_table_check_constraints = True
     supports_order_by_nulls_modifier = False
-    # Spanner does not support json
-    supports_json_field = False
+    if USE_EMULATOR:
+        # Emulator does not support json.
+        supports_json_field = False
+    else:
+        supports_json_field = True
     supports_primitives_in_json_field = False
     # Spanner does not support SELECTing an arbitrary expression that also
     # appears in the GROUP BY clause.
@@ -67,7 +71,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "model_fields.test_autofield.SmallAutoFieldTests.test_redundant_backend_range_validators",
         # Spanner does not support deferred unique constraints
         "migrations.test_operations.OperationTests.test_create_model_with_deferred_unique_constraint",
-        # Spanner does not support JSON objects
+        # Spanner does not support JSON object query on fields.
         "db_functions.comparison.test_json_object.JSONObjectTests.test_empty",
         "db_functions.comparison.test_json_object.JSONObjectTests.test_basic",
         "db_functions.comparison.test_json_object.JSONObjectTests.test_expressions",
@@ -268,17 +272,11 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "timezones.tests.NewDatabaseTests.test_query_datetimes",
         # using NULL with + crashes: https://github.com/googleapis/python-spanner-django/issues/201
         "annotations.tests.NonAggregateAnnotationTestCase.test_combined_annotation_commutative",
-        # Spanner loses DecimalField precision due to conversion to float:
-        # https://github.com/googleapis/python-spanner-django/pull/133#pullrequestreview-328482925
-        "aggregation.tests.AggregateTestCase.test_decimal_max_digits_has_no_effect",
-        "aggregation.tests.AggregateTestCase.test_related_aggregate",
+        # Spanner does not support custom precision on DecimalField
         "db_functions.comparison.test_cast.CastTests.test_cast_to_decimal_field",
         "model_fields.test_decimalfield.DecimalFieldTests.test_fetch_from_db_without_float_rounding",
         "model_fields.test_decimalfield.DecimalFieldTests.test_roundtrip_with_trailing_zeros",
-        # Spanner does not support unsigned integer field.
-        "model_fields.test_integerfield.PositiveIntegerFieldTests.test_negative_values",
-        # Spanner doesn't support the variance the standard deviation database
-        # functions:
+        # Spanner doesn't support the variance the standard deviation database functions on full population.
         "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_numerical_aggregates",
         "aggregation_regress.tests.AggregationTests.test_stddev",
         # SELECT list expression references <column> which is neither grouped
@@ -358,12 +356,8 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "transaction_hooks.tests.TestConnectionOnCommit.test_discards_hooks_from_rolled_back_savepoint",
         "transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_rolled_back_with_outer",
         "transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_does_not_affect_outer",
-        # Spanner doesn't support views.
-        "inspectdb.tests.InspectDBTransactionalTests.test_include_views",
-        "introspection.tests.IntrospectionTests.test_table_names_with_views",
-        # Fields: JSON, GenericIPAddressField are mapped to String in Spanner
+        # Field: GenericIPAddressField is mapped to String in Spanner
         "inspectdb.tests.InspectDBTestCase.test_field_types",
-        "inspectdb.tests.InspectDBTestCase.test_json_field",
         # BigIntegerField is mapped to IntegerField in Spanner
         "inspectdb.tests.InspectDBTestCase.test_number_field_types",
         # No sequence for AutoField in Spanner.
@@ -479,6 +473,9 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     if os.environ.get("SPANNER_EMULATOR_HOST", None):
         # Some code isn't yet supported by the Spanner emulator.
         skip_tests += (
+            # Views are not supported by emulator
+            "inspectdb.tests.InspectDBTransactionalTests.test_include_views",  # noqa
+            "introspection.tests.IntrospectionTests.test_table_names_with_views",  # noqa
             # Untyped parameters are not supported:
             # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator#features-and-limitations
             "auth_tests.test_views.PasswordResetTest.test_confirm_custom_reset_url_token_link_redirects_to_set_password_page",  # noqa
@@ -1588,7 +1585,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
             "queries.tests.Queries1Tests.test_ticket2306",  # noqa
             "queries.tests.Queries1Tests.test_ticket2400",  # noqa
             "queries.tests.Queries1Tests.test_ticket2496",  # noqa
-            # "queries.tests.Queries1Tests.test_ticket2902",  # noqa
             "queries.tests.Queries1Tests.test_ticket3037",  # noqa
             "queries.tests.Queries1Tests.test_ticket3141",  # noqa
             "queries.tests.Queries1Tests.test_ticket4358",  # noqa
@@ -1812,7 +1808,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
             "sitemaps_tests.test_http.HTTPSitemapTests.test_paged_sitemap",  # noqa
             "sitemaps_tests.test_http.HTTPSitemapTests.test_requestsite_sitemap",  # noqa
             "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_custom_sitemap",  # noqa
-            # "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_i18nsitemap_index",  # noqa
             "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_index",  # noqa
             "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_limited",  # noqa
             "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_xdefault",  # noqa

django_spanner/introspection.py

@@ -11,6 +11,7 @@
 )
 from django.db.models import Index
 from google.cloud.spanner_v1 import TypeCode
+from django_spanner import USE_EMULATOR
 
 
 class DatabaseIntrospection(BaseDatabaseIntrospection):
@@ -25,7 +26,28 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
         TypeCode.STRING: "CharField",
         TypeCode.TIMESTAMP: "DateTimeField",
         TypeCode.NUMERIC: "DecimalField",
+        TypeCode.JSON: "JSONField",
     }
+    if USE_EMULATOR:
+        # Emulator does not support table_type yet.
+        # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator/issues/43
+        LIST_TABLE_SQL = """
+            SELECT
+                t.table_name, t.table_name
+            FROM
+                information_schema.tables AS t
+            WHERE
+                t.table_catalog = '' and t.table_schema = ''
+        """
+    else:
+        LIST_TABLE_SQL = """
+            SELECT
+                t.table_name, t.table_type
+            FROM
+                information_schema.tables AS t
+            WHERE
+                t.table_catalog = '' and t.table_schema = ''
+        """
 
     def get_field_type(self, data_type, description):
         """A hook for a Spanner database to use the cursor description to
@@ -53,8 +75,15 @@ def get_table_list(self, cursor):
         :rtype: list
         :returns: A list of table and view names in the current database.
         """
+        results = cursor.run_sql_in_snapshot(self.LIST_TABLE_SQL)
+        tables = []
         # The second TableInfo field is 't' for table or 'v' for view.
-        return [TableInfo(row[0], "t") for row in cursor.list_tables()]
+        for row in results:
+            table_type = "t"
+            if row[1] == "VIEW":
+                table_type = "v"
+            tables.append(TableInfo(row[0], table_type))
+        return tables
 
     def get_table_description(self, cursor, table_name):
         """Return a description of the table with the DB-API cursor.description

setup.py

@@ -17,7 +17,7 @@
 # 'Development Status :: 4 - Beta'
 # 'Development Status :: 5 - Production/Stable'
 release_status = "Development Status :: 4 - Beta"
-dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.0.0"]
+dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.11.1"]
 extras = {
     "tracing": [
         "opentelemetry-api >= 1.1.0",

testing/constraints-3.6.txt

@@ -6,7 +6,7 @@
 # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev",
 # Then this file should have foo==1.14.0
 sqlparse==0.3.0
-google-cloud-spanner==3.0.0
+google-cloud-spanner==3.11.1
 opentelemetry-api==1.1.0
 opentelemetry-sdk==1.1.0
 opentelemetry-instrumentation==0.20b0

tests/system/django_spanner/models.py

@@ -34,3 +34,7 @@ class Meta:
                 name="check_start_date",
             ),
         ]
+
+
+class Detail(models.Model):
+    value = models.JSONField()

tests/system/django_spanner/test_json_field.py

@@ -0,0 +1,48 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+from .models import Detail
+import unittest
+from django.test import TransactionTestCase
+from django.db import connection
+from django_spanner import USE_EMULATOR
+from tests.system.django_spanner.utils import (
+    setup_instance,
+    teardown_instance,
+    setup_database,
+    teardown_database,
+)
+
+
+@unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.")
+class TestJsonField(TransactionTestCase):
+    @classmethod
+    def setUpClass(cls):
+        setup_instance()
+        setup_database()
+        with connection.schema_editor() as editor:
+            # Create the tables
+            editor.create_model(Detail)
+
+    @classmethod
+    def tearDownClass(cls):
+        with connection.schema_editor() as editor:
+            # delete the table
+            editor.delete_model(Detail)
+        teardown_database()
+        teardown_instance()
+
+    def test_insert_and_fetch_value(self):
+        """
+        Tests model object creation with Detail model.
+        Inserting json data into the model and retrieving it.
+        """
+        json_data = Detail(value={"name": "Jakob", "age": "26"})
+        json_data.save()
+        qs1 = Detail.objects.all()
+        self.assertEqual(qs1[0].value, {"name": "Jakob", "age": "26"})
+        # Delete data from Detail table.
+        Detail.objects.all().delete()

tests/unit/django_spanner/test_introspection.py

@@ -49,9 +49,9 @@ def test_get_table_list(self):
         cursor = mock.MagicMock()
 
         def list_tables(*args, **kwargs):
-            return [["Table_1"], ["Table_2"]]
+            return [["Table_1", "t"], ["Table_2", "t"]]
 
-        cursor.list_tables = list_tables
+        cursor.run_sql_in_snapshot = list_tables
         table_list = db_introspection.get_table_list(cursor=cursor)
         self.assertEqual(
             table_list,