From 6c1c4a719f5a3d7d67bcafce2b02bc486d7b4130 Mon Sep 17 00:00:00 2001 From: Chantal Loncle <82039410+bog-walk@users.noreply.github.com> Date: Tue, 20 Jun 2023 12:36:27 -0400 Subject: [PATCH] feat: EXPOSED-69 Extend json/json(b) support to H2, Oracle (text), and DAO Add H2 v2 support and adjust unit tests to properly exclude version 1. Add Oracle support for JSON in text format only. Adjust how BooleanColumnType values from the database are handled, as string versions of the boolean are returned by stored JSON booleans. Add support for values returned from DB when DAO is used, as well as unit tests. --- exposed-core/api/exposed-core.api | 1 + .../org/jetbrains/exposed/sql/ColumnType.kt | 17 +++++- .../org/jetbrains/exposed/sql/vendors/H2.kt | 4 +- .../exposed/sql/vendors/OracleDialect.kt | 22 ++++++-- .../shared/types/JsonBColumnTypeTests.kt | 30 ++++++++++- .../tests/shared/types/JsonColumnTypeTests.kt | 54 +++++++++++++++---- .../sql/tests/shared/types/JsonTestsData.kt | 28 +++++++++- 7 files changed, 135 insertions(+), 21 deletions(-) diff --git a/exposed-core/api/exposed-core.api b/exposed-core/api/exposed-core.api index c43c9e48f9..ce1fc06180 100644 --- a/exposed-core/api/exposed-core.api +++ b/exposed-core/api/exposed-core.api @@ -1129,6 +1129,7 @@ public class org/jetbrains/exposed/sql/JsonColumnType : org/jetbrains/exposed/sq public fun (Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function1;)V public final fun getDeserialize ()Lkotlin/jvm/functions/Function1; public final fun getSerialize ()Lkotlin/jvm/functions/Function1; + public fun nonNullValueToString (Ljava/lang/Object;)Ljava/lang/String; public synthetic fun notNullValueToDB (Ljava/lang/Object;)Ljava/lang/Object; public fun notNullValueToDB (Ljava/lang/Object;)Ljava/lang/String; public fun setParameter (Lorg/jetbrains/exposed/sql/statements/api/PreparedStatementApi;ILjava/lang/Object;)V diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/ColumnType.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/ColumnType.kt index 99c251a46e..7720ab4831 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/ColumnType.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/ColumnType.kt @@ -924,20 +924,30 @@ open class JsonColumnType( override fun valueFromDB(value: Any): Any { return when (value) { + is String -> deserialize(value) is PGobject -> deserialize(value.value!!) - else -> deserialize(value as String) + is ByteArray -> deserialize(value.decodeToString()) + else -> value } } @Suppress("UNCHECKED_CAST") override fun notNullValueToDB(value: Any) = serialize(value as T) + override fun nonNullValueToString(value: Any): String { + return when (currentDialect) { + is H2Dialect -> "JSON '${notNullValueToDB(value)}'" + else -> super.nonNullValueToString(value) + } + } + override fun setParameter(stmt: PreparedStatementApi, index: Int, value: Any?) { val parameterValue = when (currentDialect) { is PostgreSQLDialect -> PGobject().apply { type = sqlType() this.value = value as String? } + is H2Dialect -> (value as String).encodeToByteArray() else -> value } super.setParameter(stmt, index, parameterValue) @@ -954,7 +964,10 @@ class JsonBColumnType( serialize: (T) -> String, deserialize: (String) -> T ) : JsonColumnType(serialize, deserialize) { - override fun sqlType(): String = currentDialect.dataTypeProvider.jsonBType() + override fun sqlType(): String = when (currentDialect) { + is H2Dialect -> H2DataTypeProvider.jsonBType() + else -> currentDialect.dataTypeProvider.jsonBType() + } } // Date/Time columns diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/H2.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/H2.kt index d16191165f..f1ae42666f 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/H2.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/H2.kt @@ -1,7 +1,6 @@ package org.jetbrains.exposed.sql.vendors import org.intellij.lang.annotations.Language -import org.jetbrains.exposed.exceptions.UnsupportedByDialectException import org.jetbrains.exposed.exceptions.throwUnsupportedException import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.transactions.TransactionManager @@ -15,8 +14,7 @@ internal object H2DataTypeProvider : DataTypeProvider() { override fun uuidType(): String = "UUID" override fun dateTimeType(): String = "DATETIME(9)" - override fun jsonType(): String = - throw UnsupportedByDialectException("This vendor does not support non-binary text JSON data type", currentDialect) + override fun jsonBType(): String = "JSON" override fun hexToDb(hexString: String): String = "X'$hexString'" } diff --git a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/OracleDialect.kt b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/OracleDialect.kt index 5b53f706dd..b4f77cf650 100644 --- a/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/OracleDialect.kt +++ b/exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/OracleDialect.kt @@ -1,6 +1,5 @@ package org.jetbrains.exposed.sql.vendors -import org.jetbrains.exposed.exceptions.UnsupportedByDialectException import org.jetbrains.exposed.exceptions.throwUnsupportedException import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.transactions.TransactionManager @@ -37,11 +36,14 @@ internal object OracleDataTypeProvider : DataTypeProvider() { override fun booleanFromStringToBoolean(value: String): Boolean = try { value.toLong() != 0L } catch (ex: NumberFormatException) { - error("Unexpected value of type Boolean: $value") + try { + value.lowercase().toBooleanStrict() + } catch (ex: IllegalArgumentException) { + error("Unexpected value of type Boolean: $value") + } } - override fun jsonType(): String = - throw UnsupportedByDialectException("This vendor does not support non-binary text JSON data type", currentDialect) + override fun jsonType(): String = "VARCHAR2(4000)" override fun processForDefaultValue(e: Expression<*>): String = when { e is LiteralOp<*> && (e.columnType as? IDateColumnType)?.hasTimePart == false -> "DATE ${super.processForDefaultValue(e)}" @@ -146,6 +148,18 @@ internal object OracleFunctionProvider : FunctionProvider() { append(")") } + override fun jsonExtract( + expression: Expression, + vararg path: String, + toScalar: Boolean, + queryBuilder: QueryBuilder + ) = queryBuilder { + append(if (toScalar) "JSON_VALUE" else "JSON_QUERY") + append("(", expression, ", ") + path.appendTo { +"'$.$it'" } + append(")") + } + override fun update( target: Table, columnsAndValues: List, Any?>>, diff --git a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonBColumnTypeTests.kt b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonBColumnTypeTests.kt index 6d495168ea..50f32e4a35 100644 --- a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonBColumnTypeTests.kt +++ b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonBColumnTypeTests.kt @@ -7,7 +7,7 @@ import org.jetbrains.exposed.sql.tests.shared.assertEquals import org.junit.Test class JsonBColumnTypeTests : DatabaseTestsBase() { - private val binaryJsonNotSupportedDB = listOf(TestDB.SQLITE, TestDB.SQLSERVER) + TestDB.allH2TestDB + TestDB.ORACLE + private val binaryJsonNotSupportedDB = listOf(TestDB.SQLITE, TestDB.SQLSERVER) + TestDB.ORACLE @Test fun testInsertAndSelect() { @@ -35,4 +35,32 @@ class JsonBColumnTypeTests : DatabaseTestsBase() { assertEquals(updatedData, tester.selectAll().single()[tester.jsonBColumn]) } } + + @Test + fun testDAOFunctionsWithJsonBColumn() { + val dataTable = JsonTestsData.JsonBTable + val dataEntity = JsonTestsData.JsonBEntity + + withDb(excludeSettings = binaryJsonNotSupportedDB) { testDb -> + excludingH2Version1(testDb) { + SchemaUtils.create(dataTable) + + val dataA = DataHolder(User("Admin", "Alpha"), 10, true, null) + val newUser = dataEntity.new { + jsonBColumn = dataA + } + + assertEquals(dataA, dataEntity.findById(newUser.id)?.jsonBColumn) + + val updatedUser = dataA.copy(logins = 99) + dataTable.update { + it[jsonBColumn] = updatedUser + } + + assertEquals(updatedUser, dataEntity.all().single().jsonBColumn) + + SchemaUtils.drop(dataTable) + } + } + } } diff --git a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonColumnTypeTests.kt b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonColumnTypeTests.kt index d71e9c4575..92e0810166 100644 --- a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonColumnTypeTests.kt +++ b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonColumnTypeTests.kt @@ -9,16 +9,15 @@ import org.jetbrains.exposed.sql.tests.TestDB import org.jetbrains.exposed.sql.tests.currentDialectTest import org.jetbrains.exposed.sql.tests.shared.assertEquals import org.jetbrains.exposed.sql.tests.shared.expectException +import org.jetbrains.exposed.sql.vendors.OracleDialect import org.jetbrains.exposed.sql.vendors.PostgreSQLDialect import org.jetbrains.exposed.sql.vendors.SQLServerDialect import org.junit.Test class JsonColumnTypeTests : DatabaseTestsBase() { - private val notYetSupportedDB = TestDB.allH2TestDB + TestDB.ORACLE - @Test fun testInsertAndSelect() { - withJsonTable(exclude = notYetSupportedDB) { tester, _, _ -> + withJsonTable { tester, _, _ -> val newData = DataHolder(User("Pro", "Alpha"), 999, true, "A") val newId = tester.insertAndGetId { it[jsonColumn] = newData @@ -31,7 +30,7 @@ class JsonColumnTypeTests : DatabaseTestsBase() { @Test fun testUpdate() { - withJsonTable(exclude = notYetSupportedDB) { tester, _, data1 -> + withJsonTable { tester, _, data1 -> assertEquals(data1, tester.selectAll().single()[tester.jsonColumn]) val updatedData = data1.copy(active = false) @@ -45,9 +44,9 @@ class JsonColumnTypeTests : DatabaseTestsBase() { @Test fun testSelectWithSliceExtract() { - withJsonTable(exclude = notYetSupportedDB) { tester, user1, data1 -> - // SQLServer returns null if extracted JSON is not scalar - val requiresScalar = currentDialectTest is SQLServerDialect + withJsonTable(exclude = TestDB.allH2TestDB) { tester, user1, data1 -> + // SQLServer & Oracle return null if extracted JSON is not scalar + val requiresScalar = currentDialectTest is SQLServerDialect || currentDialectTest is OracleDialect val isActive = tester.jsonColumn.jsonExtract("active", toScalar = requiresScalar) val result1 = tester.slice(isActive).selectAll().singleOrNull() assertEquals(data1.active, result1?.get(isActive)) @@ -65,7 +64,7 @@ class JsonColumnTypeTests : DatabaseTestsBase() { @Test fun testSelectWhereWithExtract() { - withJsonTable(exclude = notYetSupportedDB) { tester, _, data1 -> + withJsonTable(exclude = TestDB.allH2TestDB) { tester, _, data1 -> val newId = tester.insertAndGetId { it[jsonColumn] = data1.copy(logins = 1000) } @@ -87,7 +86,7 @@ class JsonColumnTypeTests : DatabaseTestsBase() { fun testWithNonSerializableClass() { data class Fake(val number: Int) - withDb(excludeSettings = notYetSupportedDB) { testDb -> + withDb { testDb -> excludingH2Version1(testDb) { expectException { // Throws with message: Serializer for class 'Fake' is not found. @@ -99,4 +98,41 @@ class JsonColumnTypeTests : DatabaseTestsBase() { } } } + + @Test + fun testDAOFunctionsWithJsonColumn() { + val dataTable = JsonTestsData.JsonTable + val dataEntity = JsonTestsData.JsonEntity + + withDb { testDb -> + excludingH2Version1(testDb) { + SchemaUtils.create(dataTable) + + val dataA = DataHolder(User("Admin", "Alpha"), 10, true, null) + val newUser = dataEntity.new { + jsonColumn = dataA + } + + assertEquals(dataA, dataEntity.findById(newUser.id)?.jsonColumn) + + val updatedUser = dataA.copy(user = User("Lead", "Beta")) + dataTable.update { + it[jsonColumn] = updatedUser + } + + assertEquals(updatedUser, dataEntity.all().single().jsonColumn) + + if (testDb !in TestDB.allH2TestDB) { + dataEntity.new { jsonColumn = dataA } + val path = if (currentDialectTest is PostgreSQLDialect) arrayOf("user", "team") else arrayOf("user.team") + val userTeam = dataTable.jsonColumn.jsonExtract(*path) + val userInTeamB = dataEntity.find { userTeam like "B%" }.single() + + assertEquals(updatedUser, userInTeamB.jsonColumn) + } + + SchemaUtils.drop(dataTable) + } + } + } } diff --git a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonTestsData.kt b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonTestsData.kt index 67ed04dd09..581f8fed83 100644 --- a/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonTestsData.kt +++ b/exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/types/JsonTestsData.kt @@ -2,7 +2,11 @@ package org.jetbrains.exposed.sql.tests.shared.types import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json +import org.jetbrains.exposed.dao.IntEntity +import org.jetbrains.exposed.dao.IntEntityClass +import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.sql.SchemaUtils import org.jetbrains.exposed.sql.Transaction import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.tests.DatabaseTestsBase @@ -16,6 +20,18 @@ object JsonTestsData { object JsonBTable : IntIdTable("j_b_table") { val jsonBColumn = jsonb("j_b_column", Json.Default) } + + class JsonEntity(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(JsonTable) + + var jsonColumn by JsonTable.jsonColumn + } + + class JsonBEntity(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(JsonBTable) + + var jsonBColumn by JsonBTable.jsonBColumn + } } fun DatabaseTestsBase.withJsonTable( @@ -24,14 +40,18 @@ fun DatabaseTestsBase.withJsonTable( ) { val tester = JsonTestsData.JsonTable - withTables(exclude, tester) { testDb -> + withDb(excludeSettings = exclude) { testDb -> excludingH2Version1(testDb) { + SchemaUtils.create(tester) + val user1 = User("Admin", null) val data1 = DataHolder(user1, 10, true, null) tester.insert { it[jsonColumn] = data1 } statement(tester, user1, data1) + + SchemaUtils.drop(tester) } } } @@ -42,14 +62,18 @@ fun DatabaseTestsBase.withJsonBTable( ) { val tester = JsonTestsData.JsonBTable - withTables(exclude, tester) { testDb -> + withDb(excludeSettings = exclude) { testDb -> excludingH2Version1(testDb) { + SchemaUtils.create(tester) + val user1 = User("Admin", null) val data1 = DataHolder(user1, 10, true, null) tester.insert { it[jsonBColumn] = data1 } statement(tester, user1, data1) + + SchemaUtils.drop(tester) } } }