From 49292eccabc4110baf1eb18366e5abc372146ae8 Mon Sep 17 00:00:00 2001 From: Russell Cohen Date: Mon, 9 Sep 2024 15:07:41 -0400 Subject: [PATCH 1/2] Update serde implementation to support out of range floats --- .../codegen/serde/SerializeImplGenerator.kt | 33 ++++++++++++++++++- .../rust/codegen/serde/SupportStructures.kt | 15 ++++++--- .../rust/codegen/serde/SerdeDecoratorTest.kt | 27 +++++++++++++-- 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt index ce54932598..5bdcb9c7bc 100644 --- a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt +++ b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt @@ -10,6 +10,8 @@ import software.amazon.smithy.model.shapes.BlobShape import software.amazon.smithy.model.shapes.BooleanShape import software.amazon.smithy.model.shapes.CollectionShape import software.amazon.smithy.model.shapes.DocumentShape +import software.amazon.smithy.model.shapes.DoubleShape +import software.amazon.smithy.model.shapes.FloatShape import software.amazon.smithy.model.shapes.MapShape import software.amazon.smithy.model.shapes.MemberShape import software.amazon.smithy.model.shapes.NumberShape @@ -208,13 +210,42 @@ class SerializeImplGenerator(private val codegenContext: CodegenContext) { * For enums, it adds `as_str()` to convert it into a string directly. */ private fun serializeNumber(shape: NumberShape): RuntimeType { + val numericType = SimpleShapes.getValue(shape::class) + return when (shape) { + is FloatShape, is DoubleShape -> serializeFloat(shape) + else -> + RuntimeType.forInlineFun( + numericType.toString(), + PrimitiveShapesModule, + ) { + implSerializeConfigured(symbolBuilder(shape, numericType).build()) { + rustTemplate("self.value.serialize(serializer)") + } + } + } + } + + private fun serializeFloat(shape: NumberShape): RuntimeType { val numericType = SimpleShapes.getValue(shape::class) return RuntimeType.forInlineFun( numericType.toString(), PrimitiveShapesModule, ) { implSerializeConfigured(symbolBuilder(shape, numericType).build()) { - rustTemplate("self.value.serialize(serializer)") + rustTemplate( + """ + if self.value.is_nan() { + serializer.serialize_str("NaN") + } else if *self.value == #{ty}::INFINITY { + serializer.serialize_str("Infinity") + } else if *self.value == #{ty}::NEG_INFINITY { + serializer.serialize_str("-Infinity") + } else { + self.value.serialize(serializer) + } + """, + "ty" to numericType, + ) } } } diff --git a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SupportStructures.kt b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SupportStructures.kt index f1031e93a2..e034cf4168 100644 --- a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SupportStructures.kt +++ b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SupportStructures.kt @@ -47,7 +47,7 @@ object SupportStructures { { use #{serde}::Serialize; value - .serialize_ref(&#{SerializationSettings} { redact_sensitive_fields: true }) + .serialize_ref(&#{SerializationSettings}::redact_sensitive_fields()) .serialize(serializer) } """, @@ -70,7 +70,7 @@ object SupportStructures { { use #{serde}::Serialize; value - .serialize_ref(&#{SerializationSettings} { redact_sensitive_fields: false }) + .serialize_ref(&#{SerializationSettings}::leak_sensitive_fields()) .serialize(serializer) } """, @@ -211,7 +211,6 @@ object SupportStructures { private fun serializationSettings() = RuntimeType.forInlineFun("SerializationSettings", supportModule) { - // TODO(serde): Consider removing `derive(Default)` rustTemplate( """ /// Settings for use when serializing structures @@ -220,6 +219,12 @@ object SupportStructures { pub struct SerializationSettings { /// Replace all sensitive fields with `` during serialization pub redact_sensitive_fields: bool, + + /// Serialize Nan, infinity and negative infinity as strings. + /// + /// For protocols like JSON, this avoids the loss-of-information that occurs when these out-of-range values + /// are serialized as null. + pub out_of_range_floats_as_strings: bool, } impl SerializationSettings { @@ -227,10 +232,10 @@ object SupportStructures { /// /// Note: This may alter the type of the serialized output and make it impossible to deserialize as /// numerical fields will be replaced with strings. - pub const fn redact_sensitive_fields() -> Self { Self { redact_sensitive_fields: true } } + pub const fn redact_sensitive_fields() -> Self { Self { redact_sensitive_fields: true, out_of_range_floats_as_strings: false } } /// Preserve the contents of sensitive fields during serializing - pub const fn leak_sensitive_fields() -> Self { Self { redact_sensitive_fields: false } } + pub const fn leak_sensitive_fields() -> Self { Self { redact_sensitive_fields: false, out_of_range_floats_as_strings: false } } } """, ) diff --git a/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt b/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt index 902e4f9b86..5ba596055e 100644 --- a/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt +++ b/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt @@ -70,7 +70,9 @@ class SerdeDecoratorTest { blob: SensitiveBlob, constrained: Constrained, recursive: Recursive, - map: EnumKeyedMap + map: EnumKeyedMap, + float: Float, + double: Double } structure Constrained { @@ -134,6 +136,8 @@ class SerdeDecoratorTest { structure Nested { @required int: Integer, + float: Float, + double: Double, sensitive: Timestamps, notSensitive: AlsoTimestamps, manyEnums: TestEnumList, @@ -202,8 +206,12 @@ class SerdeDecoratorTest { .e(Some(TestEnum::A)) .document(Some(Document::String("hello!".into()))) .blob(Some(Blob::new("hello"))) + .float(Some(f32::INFINITY)) + .double(Some(f64::NAN)) .nested(Some(Nested::builder() .int(5) + .float(Some(f32::NEG_INFINITY)) + .double(Some(f64::NEG_INFINITY)) .sensitive(Some(sensitive_map.clone())) .not_sensitive(Some(sensitive_map)) .many_enums(Some(vec![TestEnum::A])) @@ -274,6 +282,8 @@ class SerdeDecoratorTest { "e": "A", "nested": { "int": 5, + "float": "-Infinity", + "double": "-Infinity", "sensitive": { "a": "1970-01-01T00:00:00Z" }, @@ -289,7 +299,9 @@ class SerdeDecoratorTest { "enum": "B" }, "document": "hello!", - "blob": "aGVsbG8=" + "blob": "aGVsbG8=", + "float": "Infinity", + "double": "NaN" }""".replace("\\s".toRegex(), "") private val expectedRedacted = @@ -298,6 +310,8 @@ class SerdeDecoratorTest { "e": "", "nested": { "int": 5, + "float": "-Infinity", + "double": "-Infinity", "sensitive": { "a": "" }, @@ -311,7 +325,9 @@ class SerdeDecoratorTest { }, "union": "", "document": "hello!", - "blob": "" + "blob": "", + "float": "Infinity", + "double": "NaN" } """.replace("\\s".toRegex(), "") @@ -343,8 +359,12 @@ class SerdeDecoratorTest { .e("A".into()) .document(Document::String("hello!".into())) .blob(Blob::new("hello")) + .float(f32::INFINITY) + .double(f64::NAN) .nested(Nested::builder() .int(5) + .float(f32::NEG_INFINITY) + .double(f64::NEG_INFINITY) .sensitive("a", DateTime::from(UNIX_EPOCH)) .not_sensitive("a", DateTime::from(UNIX_EPOCH)) .many_enums("A".into()) @@ -355,6 +375,7 @@ class SerdeDecoratorTest { .build() .unwrap(); let mut settings = #{crate}::serde::SerializationSettings::default(); + settings.out_of_range_floats_as_strings = true; let serialized = #{serde_json}::to_string(&input.serialize_ref(&settings)).expect("failed to serialize"); assert_eq!(serialized, ${expectedNoRedactions.dq()}); settings.redact_sensitive_fields = true; From c20fec517b85a5f7c48d837ee423d526cb6b42ae Mon Sep 17 00:00:00 2001 From: Russell Cohen Date: Mon, 30 Sep 2024 10:09:22 -0400 Subject: [PATCH 2/2] Obey settings --- .../amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt | 3 +++ .../amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt index 5bdcb9c7bc..b7c6c862d0 100644 --- a/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt +++ b/codegen-serde/src/main/kotlin/software/amazon/smithy/rust/codegen/serde/SerializeImplGenerator.kt @@ -234,6 +234,9 @@ class SerializeImplGenerator(private val codegenContext: CodegenContext) { implSerializeConfigured(symbolBuilder(shape, numericType).build()) { rustTemplate( """ + if !self.settings.out_of_range_floats_as_strings { + return self.value.serialize(serializer) + } if self.value.is_nan() { serializer.serialize_str("NaN") } else if *self.value == #{ty}::INFINITY { diff --git a/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt b/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt index 5ba596055e..f9657f0003 100644 --- a/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt +++ b/codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt @@ -381,6 +381,9 @@ class SerdeDecoratorTest { settings.redact_sensitive_fields = true; let serialized = #{serde_json}::to_string(&input.serialize_ref(&settings)).expect("failed to serialize"); assert_eq!(serialized, ${expectedRedacted.dq()}); + settings.out_of_range_floats_as_strings = false; + let serialized = #{serde_json}::to_string(&input.serialize_ref(&settings)).expect("failed to serialize"); + assert_ne!(serialized, ${expectedRedacted.dq()}); """, *codegenScope, )