cpp arrow type schema for union types

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-b35cba3304a890ee701e159b7bcc8df64877412531afae9ccf875838e1d262a9
+4945c6a5462b379e74846b23eb272fadc9b00bf029ae55e902345ee24bc887e5

crates/re_types_builder/src/codegen/cpp/mod.rs

@@ -217,7 +217,7 @@ impl QuotedObject {
     pub fn new(arrow_registry: &ArrowRegistry, objects: &Objects, obj: &crate::Object) -> Self {
         match obj.specifics {
             crate::ObjectSpecifics::Struct => Self::from_struct(arrow_registry, objects, obj),
-            crate::ObjectSpecifics::Union { .. } => Self::from_union(objects, obj),
+            crate::ObjectSpecifics::Union { .. } => Self::from_union(arrow_registry, objects, obj),
         }
     }
 
@@ -326,7 +326,11 @@ impl QuotedObject {
         Self { hpp, cpp }
     }
 
-    fn from_union(objects: &Objects, obj: &crate::Object) -> QuotedObject {
+    fn from_union(
+        arrow_registry: &ArrowRegistry,
+        objects: &Objects,
+        obj: &crate::Object,
+    ) -> QuotedObject {
         // We implement sum-types as tagged unions;
         // Putting non-POD types in a union requires C++11.
         //
@@ -376,6 +380,9 @@ impl QuotedObject {
         let mut hpp_includes = Includes::default();
         hpp_includes.system.insert("cstdint".to_owned()); // we use `uint32_t` etc everywhere.
         hpp_includes.system.insert("utility".to_owned()); // std::move
+        hpp_includes.system.insert("cstring".to_owned()); // std::memcpy
+        let mut cpp_includes = Includes::default();
+        let mut hpp_declarations = ForwardDecls::default();
 
         let enum_data_declarations = obj
             .fields
@@ -395,7 +402,7 @@ impl QuotedObject {
 
         let mut methods = Vec::new();
 
-        // Static constructors:
+        // Add one static constructor for every field.
         for obj_field in &obj.fields {
             methods.push(static_constructor_for_enum_type(
                 objects,
@@ -425,6 +432,13 @@ impl QuotedObject {
             // `enum Angle { Radians(f32), Degrees(f32) };`
         };
 
+        methods.push(arrow_data_type_method(
+            &arrow_registry.get(&obj.fqname),
+            &mut hpp_includes,
+            &mut cpp_includes,
+            &mut hpp_declarations,
+        ));
+
         let destructor = if obj.has_default_destructor(objects) {
             // No destructor needed
             quote! {}
@@ -488,11 +502,9 @@ impl QuotedObject {
             }
         };
 
-        hpp_includes.system.insert("cstring".to_owned()); // std::memcpy
-        let hpp_methods = methods.iter().map(|m| m.to_hpp_tokens());
-
         let swap_comment = comment("This bitwise swap would fail for self-referential types, but we don't have any of those.");
 
+        let hpp_methods = methods.iter().map(|m| m.to_hpp_tokens());
         let hpp = quote! {
             #hpp_includes
 
@@ -561,7 +573,16 @@ impl QuotedObject {
             }
         };
 
-        let cpp = quote! {}; // TODO(emilk): add Arrow serialization code here!
+        let cpp_methods = methods.iter().map(|m| m.to_cpp_tokens(&pascal_case_ident));
+        let cpp = quote! {
+            #cpp_includes
+
+            namespace rr {
+                namespace #namespace_ident {
+                    #(#cpp_methods)*
+                }
+            }
+        };
 
         Self { hpp, cpp }
     }
@@ -892,8 +913,7 @@ fn quote_arrow_data_type(
 
         DataType::Extension(fqname, datatype, _metadata) => {
             // If we're not at the top level, we should have already a `to_arrow_datatype` method that we can relay to.
-            // TODO(andreas): Unions don't have `to_arrow_datatype` yet.
-            if is_top_level_type || matches!(datatype.as_ref(), DataType::Union(..)) {
+            if is_top_level_type {
                 // TODO(andreas): We're no`t emitting the actual extension types here yet which is why we're skipping the extension type at top level.
                 // Currently, we wrap only Components in extension types but this is done in `rerun_c`.
                 // In the future we'll add the extension type here to the schema.

rerun_cpp/src/components/affix_fuzzer14.cpp

@@ -3,28 +3,13 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/affix_fuzzer1.hpp"
+#include "../datatypes/affix_fuzzer3.hpp"
 #include "affix_fuzzer14.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> AffixFuzzer14::to_arrow_datatype() {
-            return arrow::dense_union({
-                arrow::field("_null_markers", arrow::null(), true, nullptr),
-                arrow::field("degrees", arrow::float32(), false, nullptr),
-                arrow::field("radians", arrow::float32(), false, nullptr),
-                arrow::field(
-                    "craziness",
-                    arrow::list(arrow::field(
-                        "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), false, nullptr)),
-                    false,
-                    nullptr),
-                arrow::field("fixed_size_shenanigans",
-                             arrow::fixed_size_list(
-                                 arrow::field("item", arrow::float32(), false, nullptr), 3),
-                             false,
-                             nullptr),
-            });
+            return rr::datatypes::AffixFuzzer3::to_arrow_datatype();
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/components/affix_fuzzer15.cpp

@@ -3,28 +3,13 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/affix_fuzzer1.hpp"
+#include "../datatypes/affix_fuzzer3.hpp"
 #include "affix_fuzzer15.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> AffixFuzzer15::to_arrow_datatype() {
-            return arrow::dense_union({
-                arrow::field("_null_markers", arrow::null(), true, nullptr),
-                arrow::field("degrees", arrow::float32(), false, nullptr),
-                arrow::field("radians", arrow::float32(), false, nullptr),
-                arrow::field(
-                    "craziness",
-                    arrow::list(arrow::field(
-                        "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), false, nullptr)),
-                    false,
-                    nullptr),
-                arrow::field("fixed_size_shenanigans",
-                             arrow::fixed_size_list(
-                                 arrow::field("item", arrow::float32(), false, nullptr), 3),
-                             false,
-                             nullptr),
-            });
+            return rr::datatypes::AffixFuzzer3::to_arrow_datatype();
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/components/affix_fuzzer16.cpp

@@ -3,34 +3,14 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/affix_fuzzer1.hpp"
+#include "../datatypes/affix_fuzzer3.hpp"
 #include "affix_fuzzer16.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> AffixFuzzer16::to_arrow_datatype() {
             return arrow::list(arrow::field(
-                "item",
-                arrow::dense_union({
-                    arrow::field("_null_markers", arrow::null(), true, nullptr),
-                    arrow::field("degrees", arrow::float32(), false, nullptr),
-                    arrow::field("radians", arrow::float32(), false, nullptr),
-                    arrow::field(
-                        "craziness",
-                        arrow::list(arrow::field("item",
-                                                 rr::datatypes::AffixFuzzer1::to_arrow_datatype(),
-                                                 false,
-                                                 nullptr)),
-                        false,
-                        nullptr),
-                    arrow::field("fixed_size_shenanigans",
-                                 arrow::fixed_size_list(
-                                     arrow::field("item", arrow::float32(), false, nullptr), 3),
-                                 false,
-                                 nullptr),
-                }),
-                false,
-                nullptr));
+                "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), false, nullptr));
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/components/affix_fuzzer17.cpp

@@ -3,34 +3,14 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/affix_fuzzer1.hpp"
+#include "../datatypes/affix_fuzzer3.hpp"
 #include "affix_fuzzer17.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> AffixFuzzer17::to_arrow_datatype() {
             return arrow::list(arrow::field(
-                "item",
-                arrow::dense_union({
-                    arrow::field("_null_markers", arrow::null(), true, nullptr),
-                    arrow::field("degrees", arrow::float32(), false, nullptr),
-                    arrow::field("radians", arrow::float32(), false, nullptr),
-                    arrow::field(
-                        "craziness",
-                        arrow::list(arrow::field("item",
-                                                 rr::datatypes::AffixFuzzer1::to_arrow_datatype(),
-                                                 false,
-                                                 nullptr)),
-                        false,
-                        nullptr),
-                    arrow::field("fixed_size_shenanigans",
-                                 arrow::fixed_size_list(
-                                     arrow::field("item", arrow::float32(), false, nullptr), 3),
-                                 false,
-                                 nullptr),
-                }),
-                true,
-                nullptr));
+                "item", rr::datatypes::AffixFuzzer3::to_arrow_datatype(), true, nullptr));
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/components/affix_fuzzer18.cpp

@@ -3,96 +3,14 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/affix_fuzzer1.hpp"
+#include "../datatypes/affix_fuzzer4.hpp"
 #include "affix_fuzzer18.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> AffixFuzzer18::to_arrow_datatype() {
             return arrow::list(arrow::field(
-                "item",
-                arrow::dense_union({
-                    arrow::field("_null_markers", arrow::null(), true, nullptr),
-                    arrow::field(
-                        "single_required",
-                        arrow::dense_union({
-                            arrow::field("_null_markers", arrow::null(), true, nullptr),
-                            arrow::field("degrees", arrow::float32(), false, nullptr),
-                            arrow::field("radians", arrow::float32(), false, nullptr),
-                            arrow::field("craziness",
-                                         arrow::list(arrow::field(
-                                             "item",
-                                             rr::datatypes::AffixFuzzer1::to_arrow_datatype(),
-                                             false,
-                                             nullptr)),
-                                         false,
-                                         nullptr),
-                            arrow::field(
-                                "fixed_size_shenanigans",
-                                arrow::fixed_size_list(
-                                    arrow::field("item", arrow::float32(), false, nullptr), 3),
-                                false,
-                                nullptr),
-                        }),
-                        false,
-                        nullptr),
-                    arrow::field(
-                        "many_required",
-                        arrow::list(arrow::field(
-                            "item",
-                            arrow::dense_union({
-                                arrow::field("_null_markers", arrow::null(), true, nullptr),
-                                arrow::field("degrees", arrow::float32(), false, nullptr),
-                                arrow::field("radians", arrow::float32(), false, nullptr),
-                                arrow::field("craziness",
-                                             arrow::list(arrow::field(
-                                                 "item",
-                                                 rr::datatypes::AffixFuzzer1::to_arrow_datatype(),
-                                                 false,
-                                                 nullptr)),
-                                             false,
-                                             nullptr),
-                                arrow::field(
-                                    "fixed_size_shenanigans",
-                                    arrow::fixed_size_list(
-                                        arrow::field("item", arrow::float32(), false, nullptr), 3),
-                                    false,
-                                    nullptr),
-                            }),
-                            false,
-                            nullptr)),
-                        false,
-                        nullptr),
-                    arrow::field(
-                        "many_optional",
-                        arrow::list(arrow::field(
-                            "item",
-                            arrow::dense_union({
-                                arrow::field("_null_markers", arrow::null(), true, nullptr),
-                                arrow::field("degrees", arrow::float32(), false, nullptr),
-                                arrow::field("radians", arrow::float32(), false, nullptr),
-                                arrow::field("craziness",
-                                             arrow::list(arrow::field(
-                                                 "item",
-                                                 rr::datatypes::AffixFuzzer1::to_arrow_datatype(),
-                                                 false,
-                                                 nullptr)),
-                                             false,
-                                             nullptr),
-                                arrow::field(
-                                    "fixed_size_shenanigans",
-                                    arrow::fixed_size_list(
-                                        arrow::field("item", arrow::float32(), false, nullptr), 3),
-                                    false,
-                                    nullptr),
-                            }),
-                            true,
-                            nullptr)),
-                        false,
-                        nullptr),
-                }),
-                true,
-                nullptr));
+                "item", rr::datatypes::AffixFuzzer4::to_arrow_datatype(), true, nullptr));
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/components/transform3d.cpp

@@ -3,24 +3,13 @@
 
 #include <arrow/api.h>
 
-#include "../datatypes/translation_and_mat3x3.hpp"
-#include "../datatypes/translation_rotation_scale3d.hpp"
+#include "../datatypes/transform3d.hpp"
 #include "transform3d.hpp"
 
 namespace rr {
     namespace components {
         std::shared_ptr<arrow::DataType> Transform3D::to_arrow_datatype() {
-            return arrow::dense_union({
-                arrow::field("_null_markers", arrow::null(), true, nullptr),
-                arrow::field("TranslationAndMat3x3",
-                             rr::datatypes::TranslationAndMat3x3::to_arrow_datatype(),
-                             false,
-                             nullptr),
-                arrow::field("TranslationRotationScale",
-                             rr::datatypes::TranslationRotationScale3D::to_arrow_datatype(),
-                             false,
-                             nullptr),
-            });
+            return rr::datatypes::Transform3D::to_arrow_datatype();
         }
     } // namespace components
 } // namespace rr

rerun_cpp/src/datatypes/affix_fuzzer3.cpp

@@ -1,4 +1,30 @@
 // NOTE: This file was autogenerated by re_types_builder; DO NOT EDIT.
 // Based on "crates/re_types/definitions/rerun/testing/datatypes/fuzzy.fbs"
 
+#include <arrow/api.h>
+
+#include "../datatypes/affix_fuzzer1.hpp"
 #include "affix_fuzzer3.hpp"
+
+namespace rr {
+    namespace datatypes {
+        std::shared_ptr<arrow::DataType> AffixFuzzer3::to_arrow_datatype() {
+            return arrow::dense_union({
+                arrow::field("_null_markers", arrow::null(), true, nullptr),
+                arrow::field("degrees", arrow::float32(), false, nullptr),
+                arrow::field("radians", arrow::float32(), false, nullptr),
+                arrow::field(
+                    "craziness",
+                    arrow::list(arrow::field(
+                        "item", rr::datatypes::AffixFuzzer1::to_arrow_datatype(), false, nullptr)),
+                    false,
+                    nullptr),
+                arrow::field("fixed_size_shenanigans",
+                             arrow::fixed_size_list(
+                                 arrow::field("item", arrow::float32(), false, nullptr), 3),
+                             false,
+                             nullptr),
+            });
+        }
+    } // namespace datatypes
+} // namespace rr