From 1c7116283a6592bcd471825849a3d95fb8d16d9b Mon Sep 17 00:00:00 2001 From: Jan Galinski Date: Wed, 24 Jul 2024 02:42:39 +0200 Subject: [PATCH] refactor --- .gitignore | 2 + .../example/java/BankAccountCreatedTest.java | 4 +- .../test/kotlin/BankAccountCreatedDataTest.kt | 5 +- .../src/test/kotlin/JavaKotlinInterOpTest.kt | 3 +- .../main/kotlin/AvroKotlinSerialization.kt | 97 +++++++++++++------ .../main/kotlin/_kserializer-reflection.kt | 29 ------ .../src/main/kotlin/_reflection.kt | 43 ++++++++ .../GenericRecordSerializationStrategy.kt | 12 +++ .../strategy/KotlinxDataClassStrategy.kt | 22 +++++ .../strategy/KotlinxEnumClassStrategy.kt | 22 +++++ .../strategy/SpecificRecordBaseStrategy.kt | 20 ++++ .../test/kotlin/Avro4kSerializationTest.kt | 2 +- .../kotlin/AvroKotlinSerializationTest.kt | 34 ++++++- .../src/test/kotlin/_test/BarString.kt | 20 ++++ .../BooleanLogicalTypeSerializerTest.kt | 3 +- .../BytesLogicalTypeSerializerTest.kt | 2 +- .../DoubleLogicalTypeSerializerTest.kt | 2 +- .../FloatLogicalTypeSerializerTest.kt | 2 +- .../IntLogicalTypeSerializerTest.kt | 2 +- .../LongLogicalTypeSerializerTest.kt | 2 +- .../StringLogicalTypeSerializerTest.kt | 2 +- avro-kotlin/src/main/kotlin/AvroKotlin.kt | 40 ++++++-- .../src/main/kotlin/codec/AvroCodec.kt | 5 +- .../main/kotlin/model/wrapper/AvroSchema.kt | 15 +-- .../kotlin/repository/AvroSchemaResolver.kt | 11 +++ .../repository/AvroSchemaResolverMap.kt | 37 +++++++ ...ver.kt => AvroSchemaResolverMutableMap.kt} | 14 +-- .../src/main/kotlin/repository/_resolver.kt | 42 -------- .../src/main/kotlin/value/_compatibility.kt | 49 ++++++++++ .../src/test/kotlin/AvroSchemaResolverTest.kt | 2 +- .../src/test/kotlin/_test/FooStringTest.kt | 2 +- .../kotlin/codec/GenericRecordCodecTest.kt | 2 +- .../kotlin/codec/SpecificRecordCodecTest.kt | 2 +- ...kt => AvroSchemaResolverMutableMapTest.kt} | 8 +- lib/avro4k-core/pom.xml | 13 ++- pom.xml | 2 +- 36 files changed, 421 insertions(+), 153 deletions(-) delete mode 100644 avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt create mode 100644 avro-kotlin-serialization/src/main/kotlin/_reflection.kt create mode 100644 avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt create mode 100644 avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt create mode 100644 avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt create mode 100644 avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt create mode 100644 avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt create mode 100644 avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt rename avro-kotlin/src/main/kotlin/repository/{MutableAvroSchemaResolver.kt => AvroSchemaResolverMutableMap.kt} (67%) delete mode 100644 avro-kotlin/src/main/kotlin/repository/_resolver.kt create mode 100644 avro-kotlin/src/main/kotlin/value/_compatibility.kt rename avro-kotlin/src/test/kotlin/repository/{MutableAvroSchemaResolverTest.kt => AvroSchemaResolverMutableMapTest.kt} (73%) diff --git a/.gitignore b/.gitignore index 3123bc40..3a33ab27 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ build/ ### DEV _tmp/ .repository/ + +.console.sh diff --git a/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java b/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java index aede51bc..7285259f 100644 --- a/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java +++ b/_examples/java-example/src/test/java/io/toolisticon/kotlin/avro/example/java/BankAccountCreatedTest.java @@ -1,12 +1,12 @@ package io.toolisticon.kotlin.avro.example.java; import io.toolisticon.example.bank.BankAccountCreated; +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMap; import org.javamoney.moneta.Money; import org.junit.jupiter.api.Test; import java.util.UUID; -import static io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver; import static io.toolisticon.kotlin.avro.codec.SpecificRecordCodec.specificRecordSingleObjectDecoder; import static io.toolisticon.kotlin.avro.codec.SpecificRecordCodec.specificRecordSingleObjectEncoder; import static org.assertj.core.api.Assertions.assertThat; @@ -21,7 +21,7 @@ void encodeAndDecodeEventWithMoneyLogicalType() { .setCustomerId("1") .setInitialBalance(Money.of(100.123456, "EUR")) .build(); - final var resolver = avroSchemaResolver(BankAccountCreated.getClassSchema()); + final var resolver = new AvroSchemaResolverMap(BankAccountCreated.getClassSchema()); final var encoded = specificRecordSingleObjectEncoder().encode(bankAccountCreated); diff --git a/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt b/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt index 1cc73b91..e0da5cc9 100644 --- a/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt +++ b/_examples/kotlin-example/src/test/kotlin/BankAccountCreatedDataTest.kt @@ -1,9 +1,9 @@ package io.toolisticon.kotlin.avro.example +import io.toolisticon.kotlin.avro.AvroKotlin import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.example.customerid.CustomerId import io.toolisticon.kotlin.avro.example.money.MoneyLogicalType -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization import io.toolisticon.kotlin.avro.value.CanonicalName.Companion.toCanonicalName import io.toolisticon.kotlin.avro.value.Name.Companion.toName @@ -17,7 +17,6 @@ internal class BankAccountCreatedDataTest { @Test fun `show schema`() { val schema = KotlinExample.avro.schema(BankAccountCreatedData::class) - println(schema) assertThat(schema.canonicalName).isEqualTo("io.toolisticon.bank.BankAccountCreated".toCanonicalName()) assertThat(schema.fields).hasSize(3) @@ -49,7 +48,7 @@ internal class BankAccountCreatedDataTest { val customerId = CustomerId.random() val event = BankAccountCreatedData(accountId, customerId, amount) - val resolver = avroSchemaResolver(KotlinExample.avro.schema(BankAccountCreatedData::class)) + val resolver = AvroKotlin.avroSchemaResolver(KotlinExample.avro.schema(BankAccountCreatedData::class)) val record = KotlinExample.avro.toRecord(event) diff --git a/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt b/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt index d4cc47fc..81008b19 100644 --- a/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt +++ b/_examples/kotlin-example/src/test/kotlin/JavaKotlinInterOpTest.kt @@ -1,10 +1,10 @@ package io.toolisticon.kotlin.avro.example import io.toolisticon.example.bank.BankAccountCreated +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.example.customerid.CustomerId import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.assertj.core.api.Assertions.assertThat @@ -54,6 +54,5 @@ internal class JavaKotlinInterOpTest { assertThat(decoded.accountId).isEqualTo(orig.accountId) assertThat(decoded.customerId).isEqualTo(orig.customerId) assertThat(decoded.initialBalance).isEqualTo(orig.initialBalance) - } } diff --git a/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt b/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt index 7caa77ea..2c966780 100644 --- a/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt +++ b/avro-kotlin-serialization/src/main/kotlin/AvroKotlinSerialization.kt @@ -5,32 +5,31 @@ import io.toolisticon.kotlin.avro.AvroKotlin import io.toolisticon.kotlin.avro.codec.AvroCodec import io.toolisticon.kotlin.avro.codec.GenericRecordCodec import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom import io.toolisticon.kotlin.avro.repository.AvroSchemaResolver -import io.toolisticon.kotlin.avro.repository.MutableAvroSchemaResolver +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMutableMap import io.toolisticon.kotlin.avro.serialization.avro4k.avro4k import io.toolisticon.kotlin.avro.serialization.spi.AvroSerializationModuleFactoryServiceLoader import io.toolisticon.kotlin.avro.serialization.spi.SerializerModuleKtx.reduce +import io.toolisticon.kotlin.avro.value.AvroSchemaCompatibilityMap import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.modules.SerializersModule import kotlinx.serialization.serializer import mu.KLogging -import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord +import java.lang.Runtime.Version import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KClass -import kotlin.reflect.KType import kotlin.reflect.full.createType @OptIn(ExperimentalSerializationApi::class) class AvroKotlinSerialization( - private val avro4k: Avro, - private val schemaResolver: MutableAvroSchemaResolver = MutableAvroSchemaResolver.EMPTY, + val avro4k: Avro, + private val schemaResolver: AvroSchemaResolverMutableMap = AvroSchemaResolverMutableMap.EMPTY, private val genericData: GenericData = AvroKotlin.genericData -) { +) : AvroSchemaResolver by schemaResolver { companion object : KLogging() { fun configure(vararg serializersModules: SerializersModule): AvroKotlinSerialization { @@ -42,8 +41,18 @@ class AvroKotlinSerialization( } } - @PublishedApi - internal val avro4kSingleObject = AvroSingleObject(schemaRegistry = schemaResolver.avro4k(), avro = avro4k) + init { + /** + * We _need_ `kotlinx.serialization >= 1.7`. spring boot provides an outdated version (1.6.3). + * This is a pita to resolve. This check makes sure, any misconfigurations are found on app-start. + */ + check(Version.parse(KSerializer::class.java.`package`.implementationVersion) >= Version.parse("1.7")) { "avro4k uses features that required kotlinx.serialization version >= 1.7.0. Make sure to include the correct versions, especially when you use spring-boot." } + } + + val avro4kSingleObject = AvroSingleObject(schemaRegistry = schemaResolver.avro4k(), avro = avro4k) + + val compatibilityCache = AvroSchemaCompatibilityMap() + private val kserializerCache = ConcurrentHashMap, KSerializer<*>>() private val schemaCache = ConcurrentHashMap, AvroSchema>() @@ -51,32 +60,66 @@ class AvroKotlinSerialization( Avro { serializersModule = AvroSerializationModuleFactoryServiceLoader() } ) - @Suppress("UNCHECKED_CAST") - fun toGenericRecord(data: T): GenericRecord { - val kserializer = serializer(data::class) as KSerializer - val schema = avro4k.schema(kserializer) + fun singleObjectEncoder(): AvroCodec.SingleObjectEncoder = AvroCodec.SingleObjectEncoder { data -> + @Suppress("UNCHECKED_CAST") + val serializer = serializer(data::class) as KSerializer + val writerSchema = schema(data::class) - return avro4k.encodeToGenericData(schema, kserializer, data) as GenericRecord + val bytes = avro4kSingleObject.encodeToByteArray(writerSchema.get(), serializer, data) + + SingleObjectEncodedBytes.of(bytes) } - fun fromGenericRecord(record: GenericRecord): T { - TODO("Not yet implemented") + fun singleObjectDecoder(): AvroCodec.SingleObjectDecoder = AvroCodec.SingleObjectDecoder { bytes -> + val writerSchema = schemaResolver[bytes.fingerprint] + val klass = AvroKotlin.loadClassForSchema(writerSchema) + + @Suppress("UNCHECKED_CAST") + avro4kSingleObject.decodeFromByteArray(serializer(klass), bytes.value) as T } - @Suppress("UNCHECKED_CAST") - fun toSingleObjectEncoded(data: T): SingleObjectEncodedBytes { + fun genericRecordEncoder(): AvroCodec.GenericRecordEncoder = AvroCodec.GenericRecordEncoder { data -> + @Suppress("UNCHECKED_CAST") val serializer = serializer(data::class) as KSerializer val writerSchema = schema(data::class) - val bytes = avro4kSingleObject.encodeToByteArray(writerSchema.get(), serializer, data) + avro4k.encodeToGenericData(writerSchema.get(), serializer, data) as GenericRecord + } + + /** + * @param klass - optional. If we do know the klass already, we can pass it to avoid a second lookup. + */ + fun genericRecordDecoder(klass: KClass? = null) = AvroCodec.GenericRecordDecoder { record -> + val writerSchema = AvroSchema(record.schema) + val readerKlass: KClass = klass ?: AvroKotlin.loadClassForSchema(writerSchema) + + @Suppress("UNCHECKED_CAST") + val kserializer = serializer(readerKlass) as KSerializer + val readerSchema = schema(readerKlass) + val compatibility = compatibilityCache.compatibleToReadFrom(writerSchema, readerSchema) + + require(compatibility.isCompatible) { "Reader/writer schema are incompatible." } + + avro4k.decodeFromGenericData(writerSchema = writerSchema.get(), deserializer = kserializer, record) + } + + @Suppress("UNCHECKED_CAST") + fun toGenericRecord(data: T): GenericRecord { + val kserializer = serializer(data::class) as KSerializer + val schema = avro4k.schema(kserializer) - return SingleObjectEncodedBytes.of(bytes) + return avro4k.encodeToGenericData(schema, kserializer, data) as GenericRecord } + fun toSingleObjectEncoded(data: T): SingleObjectEncodedBytes = singleObjectEncoder().encode(data) + /** * @return kotlinx-serializer for given class. */ fun serializer(klass: KClass<*>) = kserializerCache.computeIfAbsent(klass) { key -> + + require(klass.isSerializable()) + // TODO: if we use SpecificRecords, we could derive the schema from the class directly logger.trace { "add kserializer for $key." } @@ -97,17 +140,8 @@ class AvroKotlinSerialization( inline fun fromRecord(record: GenericRecord): T = fromRecord(record, T::class) - @Suppress("UNCHECKED_CAST") fun fromRecord(record: GenericRecord, type: KClass): T { - val writerSchema = AvroSchema(record.schema) - - val kserializer = serializer(type) as KSerializer - val readerSchema = schema(type) - - // TODO nicer? - require(readerSchema.compatibleToReadFrom(writerSchema).result.incompatibilities.isEmpty()) { "Reader/writer schema are incompatible" } - - return avro4k.decodeFromGenericData(writerSchema = writerSchema.get(), deserializer = kserializer, record) as T + return genericRecordDecoder(type).decode(record) } fun encodeSingleObject( @@ -147,4 +181,7 @@ class AvroKotlinSerialization( ) fun registerSchema(schema: AvroSchema): AvroKotlinSerialization = apply { schemaResolver + schema } + + fun cachedSerializerClasses() = kserializerCache.keys().toList().toSet() + fun cachedSchemaClasses() = schemaCache.keys().toList().toSet() } diff --git a/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt b/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt deleted file mode 100644 index d1a17a77..00000000 --- a/avro-kotlin-serialization/src/main/kotlin/_kserializer-reflection.kt +++ /dev/null @@ -1,29 +0,0 @@ -package io.toolisticon.kotlin.avro.serialization - -import kotlinx.serialization.KSerializer -import kotlinx.serialization.Serializable -import kotlin.reflect.KClass -import kotlin.reflect.full.companionObject -import kotlin.reflect.full.companionObjectInstance -import kotlin.reflect.full.functions - -/** - * Reflective access to [KSerializer<*>] for a given type. - * - * The type has to be a data class or enum and annotated with [Serializable]. - * - * @param type the class to access the serializer on - * @return kserializer of given type - * @throws IllegalArgumentException when type is not a valid [Serializable] type - */ -@Throws(IllegalArgumentException::class) -fun KClass<*>.kserializer(): KSerializer<*> { - require(this.isData) { "Type ${this.qualifiedName} is not a data class." } - require(this.annotations.any { it is Serializable }) { "Type ${this.qualifiedName} is not serializable." } - - val serializerFn = requireNotNull( - this.companionObject?.functions?.find { it.name == "serializer" } - ) { "Type ${this.qualifiedName} must have a Companion.serializer, as created by the serialization compiler plugin." } - - return serializerFn.call(this.companionObjectInstance) as KSerializer<*> -} diff --git a/avro-kotlin-serialization/src/main/kotlin/_reflection.kt b/avro-kotlin-serialization/src/main/kotlin/_reflection.kt new file mode 100644 index 00000000..73ee3def --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/_reflection.kt @@ -0,0 +1,43 @@ +package io.toolisticon.kotlin.avro.serialization + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import org.apache.avro.specific.SpecificRecordBase +import kotlin.reflect.KClass +import kotlin.reflect.full.companionObject +import kotlin.reflect.full.companionObjectInstance +import kotlin.reflect.full.functions + +/** + * Reflective access to [KSerializer<*>] for a given type. + * + * The type has to be a data class or enum and annotated with [Serializable]. + * + * @return kserializer of given type + * @throws IllegalArgumentException when type is not a valid [Serializable] type + */ +@Throws(IllegalArgumentException::class) +@Deprecated("provided directly by kotlinx.serialization") +fun KClass<*>.kserializer(): KSerializer<*> { + require(this.isData) { "Type ${this.qualifiedName} is not a data class." } + require(this.isSerializable()) { "Type ${this.qualifiedName} is not serializable." } + + val serializerFn = + requireNotNull(this.companionObject?.functions?.find { it.name == "serializer" }) { "Type ${this.qualifiedName} must have a Companion.serializer, as created by the serialization compiler plugin." } + + return serializerFn.call(this.companionObjectInstance) as KSerializer<*> +} + +fun KClass<*>.isSerializable(): Boolean = this.annotations.any { it is Serializable } + +fun KClass<*>.isKotlinxDataClass(): Boolean { + // TODO: can this check be replaced by some convenience magic from kotlinx.serialization + return this.isData && this.isSerializable() +} + +fun KClass<*>.isKotlinxEnumClass(): Boolean { + // TODO: can this check be replaced by some convenience magic from kotlinx.serialization + return this.java.isEnum && this.isSerializable() +} + +fun KClass<*>.isGeneratedSpecificRecordBase(): Boolean = SpecificRecordBase::class.java.isAssignableFrom(this.java) diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt new file mode 100644 index 00000000..24dee04c --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/GenericRecordSerializationStrategy.kt @@ -0,0 +1,12 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import org.apache.avro.generic.GenericRecord +import java.util.function.Predicate +import kotlin.reflect.KClass + +interface GenericRecordSerializationStrategy : Predicate> { + + fun deserialize(serializedType: KClass<*>, data: GenericRecord): T + + fun serialize(data: T): GenericRecord +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt new file mode 100644 index 00000000..a2f9a9e4 --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxDataClassStrategy.kt @@ -0,0 +1,22 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization +import io.toolisticon.kotlin.avro.serialization.isKotlinxDataClass +import org.apache.avro.generic.GenericRecord +import kotlin.reflect.KClass + +class KotlinxDataClassStrategy( + private val avroKotlinSerialization: AvroKotlinSerialization +) : GenericRecordSerializationStrategy { + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isKotlinxDataClass() + + @Suppress("UNCHECKED_CAST") + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + return avroKotlinSerialization.genericRecordDecoder(serializedType).decode(data) as T + } + + override fun serialize(data: T): GenericRecord { + return avroKotlinSerialization.genericRecordEncoder().encode(data) + } +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt new file mode 100644 index 00000000..018f44dd --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/KotlinxEnumClassStrategy.kt @@ -0,0 +1,22 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.serialization.AvroKotlinSerialization +import io.toolisticon.kotlin.avro.serialization.isKotlinxEnumClass +import org.apache.avro.generic.GenericRecord +import kotlin.reflect.KClass + +class KotlinxEnumClassStrategy( + private val avroKotlinSerialization: AvroKotlinSerialization +) : GenericRecordSerializationStrategy { + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isKotlinxEnumClass() + + @Suppress("UNCHECKED_CAST") + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + return avroKotlinSerialization.genericRecordDecoder(serializedType).decode(data) as T + } + + override fun serialize(data: T): GenericRecord { + return avroKotlinSerialization.genericRecordEncoder().encode(data) + } +} diff --git a/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt b/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt new file mode 100644 index 00000000..eff6951b --- /dev/null +++ b/avro-kotlin-serialization/src/main/kotlin/strategy/SpecificRecordBaseStrategy.kt @@ -0,0 +1,20 @@ +package io.toolisticon.kotlin.avro.serialization.strategy + +import io.toolisticon.kotlin.avro.codec.SpecificRecordCodec +import io.toolisticon.kotlin.avro.serialization.isGeneratedSpecificRecordBase +import org.apache.avro.generic.GenericRecord +import org.apache.avro.specific.SpecificRecordBase +import kotlin.reflect.KClass + +class SpecificRecordBaseStrategy : GenericRecordSerializationStrategy { + private val converter = SpecificRecordCodec.specificRecordToGenericRecordConverter() + + override fun test(serializedType: KClass<*>): Boolean = serializedType.isGeneratedSpecificRecordBase() + + override fun deserialize(serializedType: KClass<*>, data: GenericRecord): T { + @Suppress("UNCHECKED_CAST") + return SpecificRecordCodec.genericRecordToSpecificRecordConverter(serializedType.java).convert(data) as T + } + + override fun serialize(data: T): GenericRecord = converter.convert(data as SpecificRecordBase) +} diff --git a/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt b/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt index 63246244..aad3eff0 100644 --- a/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/Avro4kSerializationTest.kt @@ -1,8 +1,8 @@ package io.toolisticon.kotlin.avro.serialization import com.github.avrokotlin.avro4k.* +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization._test.DummyEnum import io.toolisticon.kotlin.avro.serialization.avro4k.Avro4kSchemaRegistry import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes diff --git a/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt b/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt index 5e731d77..5c944aa8 100644 --- a/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/AvroKotlinSerializationTest.kt @@ -1,9 +1,11 @@ package io.toolisticon.kotlin.avro.serialization +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.model.SchemaType -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver +import io.toolisticon.kotlin.avro.serialization._test.BarString import io.toolisticon.kotlin.avro.serialization._test.DummyEnum import io.toolisticon.kotlin.avro.serialization._test.Foo +import io.toolisticon.kotlin.avro.serialization._test.barStringSchema import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test @@ -28,8 +30,6 @@ internal class AvroKotlinSerializationTest { val encoded = avro.encodeSingleObject(foo) - println(encoded.hex.formatted) - val decoded: Foo = avro.decodeFromSingleObject( schemaResolver = avroSchemaResolver(avro.schema(Foo::class)), singleObjectEncodedBytes = encoded @@ -42,8 +42,34 @@ internal class AvroKotlinSerializationTest { fun `enum from-to single object encoded`() { val data = DummyEnum.BAR - val soe = avro.toSingleObjectEncoded(data) + val soe = avro.singleObjectEncoder().encode(data) + val decoded = avro.singleObjectDecoder().decode(soe) + assertThat(decoded).isEqualTo(data) } + + @Test + fun `get schema from BarString`() { + assertThat(avro.cachedSchemaClasses()).isEmpty() + assertThat(avro.cachedSerializerClasses()).isEmpty() + + val schema = avro.schema(BarString::class) + + assertThat(schema.fingerprint).isEqualTo(barStringSchema.fingerprint) + + assertThat(avro.cachedSchemaClasses()).containsExactly(BarString::class) + assertThat(avro.cachedSerializerClasses()).containsExactly(BarString::class) + + assertThat(avro[barStringSchema.fingerprint]).isEqualTo(schema) + + val data = BarString("foo") + val encoded = avro.singleObjectEncoder().encode(data) + + val decoded = avro.singleObjectDecoder().decode(encoded) + + assertThat(decoded).isEqualTo(data) + } + } + diff --git a/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt b/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt new file mode 100644 index 00000000..281d1076 --- /dev/null +++ b/avro-kotlin-serialization/src/test/kotlin/_test/BarString.kt @@ -0,0 +1,20 @@ +package io.toolisticon.kotlin.avro.serialization._test + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import kotlinx.serialization.Serializable +import org.apache.avro.SchemaBuilder + +/** + * It cannot get much simpler than this ... + */ +@Serializable +data class BarString(val name: String) + +val barStringSchema = AvroSchema( + SchemaBuilder.record("BarString") + .namespace("io.toolisticon.kotlin.avro.serialization._test") + .fields() + .requiredString("name") + .endRecord() +) + diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt index c02dff0c..e9ba576f 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/BooleanLogicalTypeSerializerTest.kt @@ -1,9 +1,8 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestBooleanLogicalType -import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestDoubleLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt index cb280a42..64dab568 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/BytesLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestBytesLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt index 9b276d45..4227c831 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/DoubleLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestDoubleLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt index 065f28a3..449ae849 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/FloatLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestFloatLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt index c288c5f0..49189298 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/IntLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestIntLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt index 0ac4dc09..fe2a1bf5 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/LongLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro.serialization.serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestLongLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt b/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt index 422c5b60..afe0d068 100644 --- a/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt +++ b/avro-kotlin-serialization/src/test/kotlin/serializer/StringLogicalTypeSerializerTest.kt @@ -1,7 +1,7 @@ package serializer +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.codec.GenericRecordCodec -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.TestStringLogicalType import io.toolisticon.kotlin.avro.serialization.serializer._fixtures.avroSerialization import kotlinx.serialization.Serializable diff --git a/avro-kotlin/src/main/kotlin/AvroKotlin.kt b/avro-kotlin/src/main/kotlin/AvroKotlin.kt index 5375466f..491bc943 100644 --- a/avro-kotlin/src/main/kotlin/AvroKotlin.kt +++ b/avro-kotlin/src/main/kotlin/AvroKotlin.kt @@ -1,6 +1,7 @@ package io.toolisticon.kotlin.avro import _ktx.ResourceKtx.resourceUrl +import io.toolisticon.kotlin.avro._ktx.KotlinKtx.head import io.toolisticon.kotlin.avro.declaration.ProtocolDeclaration import io.toolisticon.kotlin.avro.declaration.SchemaDeclaration import io.toolisticon.kotlin.avro.logical.AvroLogicalType @@ -8,8 +9,10 @@ import io.toolisticon.kotlin.avro.model.AvroType import io.toolisticon.kotlin.avro.model.wrapper.AvroProtocol import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.repository.AvroSchemaResolver +import io.toolisticon.kotlin.avro.repository.AvroSchemaResolverMap import io.toolisticon.kotlin.avro.value.* import io.toolisticon.kotlin.avro.value.CanonicalName.Companion.toCanonicalName +import org.apache.avro.AvroRuntimeException import org.apache.avro.LogicalTypes import org.apache.avro.Protocol import org.apache.avro.Schema @@ -76,12 +79,42 @@ object AvroKotlin { fun avroType(schema: AvroSchema): AvroType = AvroType.avroType(schema) + fun loadClassForSchema(schema: AvroSchema): KClass { + val nullableJavaClassClass: Class<*>? = specificData.getClass(schema.get()) + + @Suppress("UNCHECKED_CAST") + return if (nullableJavaClassClass != null) + nullableJavaClassClass.kotlin as KClass + else throw AvroRuntimeException("Klass could not be found for ${schema.canonicalName.fqn}") + } + @JvmStatic val genericData: GenericData get() = GenericData() @JvmStatic val specificData: SpecificData get() = SpecificData() + @JvmStatic + fun avroSchemaResolver(schemas: List): AvroSchemaResolverMap { + val (first, other) = schemas.head() + return avroSchemaResolver(first, *(other.toTypedArray())) + } + + @JvmStatic + fun avroSchemaResolver(firstSchema: AvroSchema, vararg otherSchemas: AvroSchema): AvroSchemaResolverMap { + val store = buildMap { + put(firstSchema.fingerprint, firstSchema) + + if (otherSchemas.isNotEmpty()) { + putAll(otherSchemas.associateBy { it.fingerprint }) + } else { + // in case we have a single schema, also provide this for key=NULL, so invoke works + put(AvroFingerprint.NULL, firstSchema) + } + } + + return AvroSchemaResolverMap(store) + } fun canonicalName(namespace: String, name: String): CanonicalName = (namespace to name).toCanonicalName() @@ -165,11 +198,6 @@ object AvroKotlin { ) } - @JvmStatic - fun avroSchemaResolver(schema: Schema) = io.toolisticon.kotlin.avro.repository.avroSchemaResolver( - firstSchema = AvroSchema(schema) - ) - val avroLogicalTypes by lazy { LogicalTypes.getCustomRegisteredTypes().values.filterIsInstance>() .toList() @@ -233,6 +261,4 @@ object AvroKotlin { } fun Result?>.orEmpty(): List = getOrNull() ?: emptyList() - - } diff --git a/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt b/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt index 136adfab..764a32b0 100644 --- a/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt +++ b/avro-kotlin/src/main/kotlin/codec/AvroCodec.kt @@ -5,6 +5,7 @@ import io.toolisticon.kotlin.avro.value.BinaryEncodedBytes import io.toolisticon.kotlin.avro.value.JsonString import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecord import org.apache.avro.io.DecoderFactory import org.apache.avro.io.EncoderFactory import org.apache.avro.specific.SpecificData @@ -46,7 +47,9 @@ object AvroCodec { fun interface SingleObjectEncoder : Encoder fun interface SingleObjectDecoder : Decoder + fun interface GenericRecordEncoder : Encoder + fun interface GenericRecordDecoder : Decoder + interface BinaryEncoder : Encoder interface BinaryDecoder : Decoder - } diff --git a/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt b/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt index 8ac3a33f..7325a7d6 100644 --- a/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt +++ b/avro-kotlin/src/main/kotlin/model/wrapper/AvroSchema.kt @@ -239,23 +239,24 @@ object AvroSchemaChecks { val AvroSchema.isUnion: Boolean get() = get().isUnion val AvroSchema.isUnionType: Boolean get() = SchemaType.UNION == type && isUnion && unionTypes.isNotEmpty() - /** * Check if we can decode using this schema if the encoder used * [writer] schema. * * @param writer - the schema used to encode data - * @return [SchemaCompatibility.SchemaPairCompatibility] with reader=this + * @return [AvroSchemaCompatibility] with reader=this */ - fun AvroSchema.compatibleToReadFrom(writer: AvroSchema): SchemaCompatibility.SchemaPairCompatibility = - SchemaCompatibility.checkReaderWriterCompatibility(get(), writer.get()) + fun AvroSchema.compatibleToReadFrom(writer: AvroSchema): AvroSchemaCompatibility = AvroSchemaCompatibility( + value = SchemaCompatibility.checkReaderWriterCompatibility(get(), writer.get()) + ) /** * Check data encoded using this schema could be decoded from [reader] schema. * * @param reader - the schema to decode the data - * @return [SchemaCompatibility.SchemaPairCompatibility] with writer=this + * @return [AvroSchemaCompatibility] with writer=this */ - fun AvroSchema.compatibleToBeReadFrom(reader: AvroSchema): SchemaCompatibility.SchemaPairCompatibility = - SchemaCompatibility.checkReaderWriterCompatibility(reader.get(), get()) + fun AvroSchema.compatibleToBeReadFrom(reader: AvroSchema) = AvroSchemaCompatibility( + value = SchemaCompatibility.checkReaderWriterCompatibility(reader.get(), get()) + ) } diff --git a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt index aab94a1d..31e9eb16 100644 --- a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolver.kt @@ -20,3 +20,14 @@ fun interface AvroSchemaResolver : SchemaStore { @Throws(MissingSchemaException::class) override fun findByFingerprint(fingerprint: Long): Schema = this[AvroFingerprint(fingerprint)].get() } + + +interface SchemaResolverMap : AvroSchemaResolver, Map { + + @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") + @kotlin.jvm.Throws(MissingSchemaException::class) + override fun get(fingerprint: AvroFingerprint): AvroSchema +} + +internal fun missingSchemaException(fingerprint: AvroFingerprint) = + MissingSchemaException("Cannot resolve schema for fingerprint: $fingerprint[${fingerprint.value}]") diff --git a/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt new file mode 100644 index 00000000..7fffbfdf --- /dev/null +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMap.kt @@ -0,0 +1,37 @@ +package io.toolisticon.kotlin.avro.repository + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import io.toolisticon.kotlin.avro.value.AvroFingerprint +import org.apache.avro.Schema + + +data class AvroSchemaResolverMap( + private val store: Map = emptyMap() +) : SchemaResolverMap, Map by store { + companion object { + val EMPTY = AvroSchemaResolverMap() + + } + + constructor(schema: Schema) : this(AvroSchema(schema)) + constructor(schema: AvroSchema) : this((EMPTY + schema).store) + + @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") + override fun get(fingerprint: AvroFingerprint): AvroSchema = store[fingerprint] ?: throw missingSchemaException( + fingerprint + ) + + operator fun plus(schema: AvroSchema): AvroSchemaResolverMap = copy( + store = buildMap { + putAll(store) + put(schema.fingerprint, schema) + } + ) + + operator fun plus(other: SchemaResolverMap): AvroSchemaResolverMap = copy(store = buildMap { + putAll(store) + putAll(other) + }) + + fun toMutableMap(): AvroSchemaResolverMutableMap = AvroSchemaResolverMutableMap.EMPTY + this +} diff --git a/avro-kotlin/src/main/kotlin/repository/MutableAvroSchemaResolver.kt b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt similarity index 67% rename from avro-kotlin/src/main/kotlin/repository/MutableAvroSchemaResolver.kt rename to avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt index d4ac1c4a..6a082e7e 100644 --- a/avro-kotlin/src/main/kotlin/repository/MutableAvroSchemaResolver.kt +++ b/avro-kotlin/src/main/kotlin/repository/AvroSchemaResolverMutableMap.kt @@ -11,14 +11,14 @@ import java.util.concurrent.ConcurrentHashMap * and keeps all known instances in an in-memory map. */ @JvmInline -value class MutableAvroSchemaResolver private constructor( +value class AvroSchemaResolverMutableMap private constructor( private val store: MutableMap = ConcurrentHashMap() -) : AvroSchemaResolver, Map by store { +) : SchemaResolverMap, Map by store { companion object { - val EMPTY = MutableAvroSchemaResolver() + val EMPTY = AvroSchemaResolverMutableMap() } - constructor(schema: Schema) : this((EMPTY + AvroSchema(schema)).store) + constructor(schema: Schema) : this(AvroSchema(schema)) constructor(schema: AvroSchema) : this((EMPTY + schema).store) @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") @@ -27,11 +27,13 @@ value class MutableAvroSchemaResolver private constructor( fingerprint ) - operator fun plus(schema: AvroSchema): MutableAvroSchemaResolver = apply { + operator fun plus(schema: AvroSchema): AvroSchemaResolverMutableMap = apply { store[schema.fingerprint] = schema } - operator fun plus(other: MutableAvroSchemaResolver): MutableAvroSchemaResolver = apply { + operator fun plus(other: SchemaResolverMap): AvroSchemaResolverMutableMap = apply { store.putAll(other) } + + fun toMap() = AvroSchemaResolverMap.EMPTY + this } diff --git a/avro-kotlin/src/main/kotlin/repository/_resolver.kt b/avro-kotlin/src/main/kotlin/repository/_resolver.kt deleted file mode 100644 index b5584033..00000000 --- a/avro-kotlin/src/main/kotlin/repository/_resolver.kt +++ /dev/null @@ -1,42 +0,0 @@ -@file:JvmName("RepositoryKt") -package io.toolisticon.kotlin.avro.repository - -import io.toolisticon.kotlin.avro._ktx.KotlinKtx.head -import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema -import io.toolisticon.kotlin.avro.value.AvroFingerprint -import org.apache.avro.message.MissingSchemaException - -internal fun missingSchemaException(fingerprint: AvroFingerprint) = - MissingSchemaException("Cannot resolve schema for fingerprint: $fingerprint[${fingerprint.value}]") - -fun avroSchemaResolver(schemas: List): AvroSchemaResolver { - val (first, other) = schemas.head() - return avroSchemaResolver(first, *(other.toTypedArray())) -} - - -fun avroSchemaResolver(firstSchema: AvroSchema, vararg otherSchemas: AvroSchema): AvroSchemaResolver = object : AvroSchemaResolver { - private val store = buildMap { - put(firstSchema.fingerprint, firstSchema) - - if (otherSchemas.isNotEmpty()) { - putAll(otherSchemas.associateBy { it.fingerprint }) - } else { - // in case we have a single schema, also provide this for key=NULL, so invoke works - put(AvroFingerprint.NULL, firstSchema) - } - } - - override fun get(fingerprint: AvroFingerprint): AvroSchema = store[fingerprint] ?: throw missingSchemaException(fingerprint) -} - -operator fun AvroSchemaResolver.plus(other: AvroSchemaResolver): AvroSchemaResolver = AvroSchemaResolver { fingerprint -> - try { - // first try us - this[fingerprint] - } catch (e: MissingSchemaException) { - // then try other - raise exception if still no hit - other[fingerprint] - } -} - diff --git a/avro-kotlin/src/main/kotlin/value/_compatibility.kt b/avro-kotlin/src/main/kotlin/value/_compatibility.kt new file mode 100644 index 00000000..c3bf8701 --- /dev/null +++ b/avro-kotlin/src/main/kotlin/value/_compatibility.kt @@ -0,0 +1,49 @@ +package io.toolisticon.kotlin.avro.value + +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema +import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom +import org.apache.avro.SchemaCompatibility.SchemaCompatibilityResult +import org.apache.avro.SchemaCompatibility.SchemaPairCompatibility +import java.util.concurrent.ConcurrentHashMap + +/** + * Used as a key in [AvroSchemaCompatibilityMap] to cache [AvroSchemaCompatibility] results. + */ +@JvmInline +value class AvroFingerprintPair private constructor(override val value: Pair) : PairType { + companion object { + fun of(writerSchema: AvroSchema, readerSchema: AvroSchema) = AvroFingerprintPair(writerSchema.fingerprint, readerSchema.fingerprint) + } + + constructor(writer: AvroFingerprint, reader: AvroFingerprint) : this(writer to reader) + + val writerSchema: AvroFingerprint get() = value.first + val readerSchema: AvroFingerprint get() = value.second +} + +/** + * Wraps apache-avro [SchemaPairCompatibility] and allows simple(typesafe access to + * helper functions and derived attributes. + */ +@JvmInline +value class AvroSchemaCompatibility(override val value: SchemaPairCompatibility) : ValueType { + val result: SchemaCompatibilityResult get() = value.result + + val isCompatible: Boolean get() = result.incompatibilities.isEmpty() +} + +/** + * Mutable cache for [AvroSchemaCompatibility], so based on a writer- and reader-schema + * fingerprint, we only calculate once. + */ +@JvmInline +value class AvroSchemaCompatibilityMap( + private val value: MutableMap = ConcurrentHashMap() +) { + + fun compatibleToReadFrom(writerSchema: AvroSchema, readerSchema: AvroSchema): AvroSchemaCompatibility { + val key = AvroFingerprintPair.of(writerSchema, readerSchema) + + return value.computeIfAbsent(key) { _ -> readerSchema.compatibleToReadFrom(writerSchema) } + } +} diff --git a/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt b/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt index a21f23fd..9ced74cd 100644 --- a/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt +++ b/avro-kotlin/src/test/kotlin/AvroSchemaResolverTest.kt @@ -1,7 +1,7 @@ package io.toolisticon.kotlin.avro +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.TestFixtures.BankAccountCreatedFixtures.SCHEMA_BANK_ACCOUNT_CREATED -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.value.AvroFingerprint import org.apache.avro.message.MissingSchemaException import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt b/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt index 9498e58b..577f7744 100644 --- a/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt +++ b/avro-kotlin/src/test/kotlin/_test/FooStringTest.kt @@ -1,10 +1,10 @@ package io.toolisticon.kotlin.avro._test import io.toolisticon.kotlin.avro.AvroKotlin +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.model.wrapper.AvroSchema import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToBeReadFrom import io.toolisticon.kotlin.avro.model.wrapper.AvroSchemaChecks.compatibleToReadFrom -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import org.apache.avro.SchemaCompatibility import org.apache.avro.generic.GenericData import org.assertj.core.api.Assertions.assertThat diff --git a/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt b/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt index b7d3a64a..404c7351 100644 --- a/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt +++ b/avro-kotlin/src/test/kotlin/codec/GenericRecordCodecTest.kt @@ -1,9 +1,9 @@ package io.toolisticon.kotlin.avro.codec +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro._test.FooString import io.toolisticon.kotlin.avro._test.FooString2 import io.toolisticon.kotlin.avro.codec.GenericRecordCodec.convert -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import io.toolisticon.kotlin.avro.value.SingleObjectEncodedBytes import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test diff --git a/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt b/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt index e56db9dc..06ae0f9f 100644 --- a/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt +++ b/avro-kotlin/src/test/kotlin/codec/SpecificRecordCodecTest.kt @@ -1,8 +1,8 @@ package io.toolisticon.kotlin.avro.codec +import io.toolisticon.kotlin.avro.AvroKotlin.avroSchemaResolver import io.toolisticon.kotlin.avro.TestFixtures.BankAccountCreatedFixtures import io.toolisticon.kotlin.avro._test.BankAccountCreatedData -import io.toolisticon.kotlin.avro.repository.avroSchemaResolver import lib.test.event.BankAccountCreated import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test diff --git a/avro-kotlin/src/test/kotlin/repository/MutableAvroSchemaResolverTest.kt b/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt similarity index 73% rename from avro-kotlin/src/test/kotlin/repository/MutableAvroSchemaResolverTest.kt rename to avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt index 30ecfdb4..0c1995bd 100644 --- a/avro-kotlin/src/test/kotlin/repository/MutableAvroSchemaResolverTest.kt +++ b/avro-kotlin/src/test/kotlin/repository/AvroSchemaResolverMutableMapTest.kt @@ -5,23 +5,23 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test -internal class MutableAvroSchemaResolverTest { +internal class AvroSchemaResolverMutableMapTest { @Test fun `empty instance`() { - assertThat(MutableAvroSchemaResolver.EMPTY).isEmpty() + assertThat(AvroSchemaResolverMutableMap.EMPTY).isEmpty() } @Test fun `init with single schema`() { val schema = loadAvroSchema("avro/lib/test/event/BankAccountCreated.avsc") - assertThat(MutableAvroSchemaResolver(schema)).hasSize(1) + assertThat(AvroSchemaResolverMutableMap(schema)).hasSize(1) } @Test fun `add schema`() { val schema = loadAvroSchema("avro/lib/test/event/BankAccountCreated.avsc") - val resolver = MutableAvroSchemaResolver(schema) + val resolver = AvroSchemaResolverMutableMap(schema) assertThat(resolver).hasSize(1) resolver.plus(loadAvroSchema("avro/lib/test/dummy/NestedDummy.avsc")) assertThat(resolver).hasSize(2) diff --git a/lib/avro4k-core/pom.xml b/lib/avro4k-core/pom.xml index f33531c9..5c81f5eb 100644 --- a/lib/avro4k-core/pom.xml +++ b/lib/avro4k-core/pom.xml @@ -1,5 +1,5 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -29,6 +29,10 @@ org.apache.avro avro + + org.jetbrains.kotlinx + kotlinx-serialization-json-jvm + org.jetbrains.kotlinx kotlinx-serialization-core-jvm @@ -46,9 +50,14 @@ org.jetbrains.kotlinx kotlinx-serialization-core-jvm - ${kotlinx-serialization.version} compile + + + org.jetbrains.kotlinx + kotlinx-serialization-json-jvm + runtime + diff --git a/pom.xml b/pom.xml index 83ef19ce..6560043a 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ io.toolisticon.maven.parent maven-parent-kotlin-base - 2024.7.0 + 2024.7.1