add bigint test

airbytehq · Nov 15, 2024 · fba64f7 · fba64f7
1 parent 1abbaa9
commit fba64f7
Show file tree

Hide file tree

Showing 4 changed files with 28 additions and 11 deletions.
diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValue.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValue.kt
@@ -6,6 +6,7 @@ package io.airbyte.cdk.load.data
 
 import com.fasterxml.jackson.databind.JsonNode
 import java.math.BigDecimal
+import java.math.BigInteger
 import java.time.LocalDate
 import java.time.LocalDateTime
 import java.time.LocalTime
@@ -23,6 +24,7 @@ sealed interface AirbyteValue {
                 is Boolean -> BooleanValue(value)
                 is Int -> IntegerValue(value.toLong())
                 is Long -> IntegerValue(value)
+                is BigInteger -> IntegerValue(value)
                 is Double -> NumberValue(BigDecimal.valueOf(value))
                 is BigDecimal -> NumberValue(value)
                 is LocalDate -> DateValue(value.toString())
@@ -60,7 +62,8 @@ value class BooleanValue(val value: Boolean) : AirbyteValue, Comparable<BooleanV
 }
 
 @JvmInline
-value class IntegerValue(val value: Long) : AirbyteValue, Comparable<IntegerValue> {
+value class IntegerValue(val value: BigInteger) : AirbyteValue, Comparable<IntegerValue> {
+    constructor(value: Long): this(BigInteger.valueOf(value))
     override fun compareTo(other: IntegerValue): Int = value.compareTo(other.value)
 }
 

diff --git a/...te-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/json/JsonToAirbyteValue.kt b/...te-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/json/JsonToAirbyteValue.kt
@@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode
 import io.airbyte.cdk.load.data.*
 import io.airbyte.cdk.util.Jsons
 import java.math.BigDecimal
+import java.math.BigInteger
 
 /**
  * Converts from json to airbyte value, performing the minimum validation necessary to marshal to a
@@ -90,10 +91,10 @@ class JsonToAirbyteValue {
     private fun toInteger(json: JsonNode): IntegerValue {
         val longVal =
             when {
-                json.isBoolean -> if (json.asBoolean()) 1L else 0L
-                json.isIntegralNumber -> json.asLong()
-                json.isFloatingPointNumber -> json.asDouble().toLong()
-                json.isTextual -> json.asText().toLong()
+                json.isBoolean -> if (json.asBoolean()) BigInteger.ONE else BigInteger.ZERO
+                json.isIntegralNumber -> json.bigIntegerValue()
+                json.isFloatingPointNumber -> json.bigIntegerValue()
+                json.isTextual -> json.asText().toBigInteger()
                 else -> throw IllegalArgumentException("Could not convert $json to Integer")
             }
         return IntegerValue(longVal)
@@ -103,8 +104,8 @@ class JsonToAirbyteValue {
         val numVal =
             when {
                 json.isBoolean -> BigDecimal(if (json.asBoolean()) 1.0 else 0.0)
-                json.isIntegralNumber -> json.asLong().toBigDecimal()
-                json.isFloatingPointNumber -> json.asDouble().toBigDecimal()
+                json.isIntegralNumber -> json.decimalValue()
+                json.isFloatingPointNumber -> json.decimalValue()
                 json.isTextual -> json.asText().toBigDecimal()
                 else -> throw IllegalArgumentException("Could not convert $json to Number")
             }

diff --git a/...c/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt b/...c/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt
@@ -28,16 +28,16 @@ class AirbyteValueWithMetaToOutputRecord {
                 Instant.ofEpochMilli(
                     (value.values[DestinationRecord.Meta.COLUMN_NAME_AB_EXTRACTED_AT]
                             as IntegerValue)
-                        .value
+                        .value.toLong()
                 ),
             loadedAt = null,
             data = value.values[DestinationRecord.Meta.COLUMN_NAME_DATA] as ObjectValue,
             generationId =
                 (value.values[DestinationRecord.Meta.COLUMN_NAME_AB_GENERATION_ID] as IntegerValue)
-                    .value,
+                    .value.toLong(),
             airbyteMeta =
                 OutputRecord.Meta(
-                    syncId = (meta.values["sync_id"] as IntegerValue).value,
+                    syncId = (meta.values["sync_id"] as IntegerValue).value.toLong(),
                     changes =
                         (meta.values["changes"] as ArrayValue)
                             .values

diff --git a/...ad/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt b/...ad/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt
@@ -52,6 +52,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage
 import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange
 import io.airbyte.protocol.models.v0.AirbyteStateMessage
 import java.math.BigDecimal
+import java.math.BigInteger
 import java.time.LocalDate
 import java.time.LocalDateTime
 import java.time.LocalTime
@@ -83,6 +84,8 @@ data class StronglyTyped(
     val topLevelFloatLosesPrecision: Boolean = true,
     /** Whether floats nested inside objects/arrays are represented as float64. */
     val nestedFloatLosesPrecision: Boolean = true,
+    /** Whether the destination supports integers larger than int64 */
+    val integerCanBeLarge: Boolean = true,
 ) : AllTypesBehavior
 
 data object Untyped : AllTypesBehavior
@@ -1561,7 +1564,8 @@ abstract class BasicFunctionalityIntegrationTest(
                 ),
                 // A record with all fields unset
                 makeRecord("""{"id": 3}"""),
-                // A record that verifies floating-point behavior.
+                // A record that verifies numeric behavior.
+                // 99999999999999999999999999999999 is out of range for int64.
                 // 67.174118 cannot be represented as a standard float64
                 // (it turns into 67.17411800000001).
                 makeRecord(
@@ -1570,6 +1574,7 @@ abstract class BasicFunctionalityIntegrationTest(
                           "id": 4,
                           "struct": {"foo": 67.174118},
                           "number": 67.174118
+                          "integer": 99999999999999999999999999999999
                         }
                     """.trimIndent(),
                 ),
@@ -1595,6 +1600,7 @@ abstract class BasicFunctionalityIntegrationTest(
 
         val nestedFloat: BigDecimal
         val topLevelFloat: BigDecimal
+        val bigInt: BigInteger?
         val badValuesData: Map<String, Any?>
         val badValuesChanges: MutableList<Change>
         when (allTypesBehavior) {
@@ -1611,6 +1617,11 @@ abstract class BasicFunctionalityIntegrationTest(
                     } else {
                         BigDecimal("67.174118")
                     }
+                bigInt = if (allTypesBehavior.integerCanBeLarge) {
+                    BigInteger("99999999999999999999999999999999")
+                } else {
+                    null
+                }
                 badValuesData =
                     mapOf(
                         "id" to 5,
@@ -1649,6 +1660,7 @@ abstract class BasicFunctionalityIntegrationTest(
             Untyped -> {
                 nestedFloat = BigDecimal("67.174118")
                 topLevelFloat = BigDecimal("67.174118")
+                bigInt = BigInteger("99999999999999999999999999999999")
                 badValuesData =
                     mapOf(
                         "id" to 5,
@@ -1724,6 +1736,7 @@ abstract class BasicFunctionalityIntegrationTest(
                             "id" to 4,
                             "struct" to mapOf("foo" to nestedFloat),
                             "number" to topLevelFloat,
+                            "integer" to bigInt,
                         ),
                     airbyteMeta = OutputRecord.Meta(syncId = 42),
                 ),