All components are now consistently implemented by a datatype (#6823)

### What

Previously, we were a bit inconsistent with this: Most components were
backed up by a single datatype but some weren't. This is now enforced
everywhere.

The expectation was that this saves a lot of code since we can forward
to fewer serialized types. In reality this isn't _as_ good as expected
since a bunch of new datatypes had to be introduced:

* `Float64`
* `Blob` used by `Blob` component - there should probably be no blob
component but instead components with a deeper semantic
* `Utf8List` for the sole purpose of implementing `VisualizerOverrides`
* `ViewCoordinates` for the sole purpose of implementing
`ViewCoordinates`

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6823?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6823?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/6823)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

crates/build/re_dev_tools/src/build_examples/snippets.rs

@@ -91,13 +91,12 @@ fn collect_snippets_recursively(
         let name = path.file_stem().unwrap().to_str().unwrap().to_owned();
 
         let config_key = path.strip_prefix(snippet_root_path)?.with_extension("");
-
-        let config_key = config_key.to_str().unwrap();
+        let config_key = config_key.to_str().unwrap().replace('\\', "/");
 
         let is_opted_out = config
             .opt_out
             .run
-            .get(config_key)
+            .get(&config_key)
             .is_some_and(|languages| languages.iter().any(|v| v == "py"));
         if is_opted_out {
             println!(
@@ -130,7 +129,7 @@ fn collect_snippets_recursively(
         println!("Adding {}", path.display());
         let extra_args: Vec<String> = config
             .extra_args
-            .get(config_key)
+            .get(&config_key)
             .cloned()
             .unwrap_or_default()
             .into_iter()

crates/build/re_types_builder/src/codegen/python/views.rs

@@ -40,7 +40,7 @@ fn init_method(reporter: &Reporter, objects: &Objects, obj: &Object) -> String {
     origin: EntityPathLike = "/",
     contents: SpaceViewContentsLike = "$origin/**",
     name: Utf8Like | None = None,
-    visible: blueprint_components.VisibleLike | None = None,
+    visible: datatypes.BoolLike | None = None,
     defaults: list[Union[AsComponents, ComponentBatchLike]] = [],
     overrides: dict[EntityPathLike, list[ComponentBatchLike]] = {},
     "#

crates/build/re_types_builder/src/objects.rs

@@ -98,10 +98,11 @@ impl Objects {
                             }
                         }
                     }
-                } else {
-                    // Note that we *do* allow primitive fields on components for the moment. Not doing so creates a lot of bloat.
-                    if obj.kind == ObjectKind::Archetype || obj.kind == ObjectKind::View {
-                        reporter.error(virtpath, &obj.fqname, format!("Field {:?} s a primitive field of type {:?}. Primitive types are only allowed on DataTypes & Components.", field.fqname, field.typ));
+                } else if obj.kind != ObjectKind::Datatype {
+                    let is_enum_component = obj.kind == ObjectKind::Component && obj.is_enum(); // Enum components are allowed to have no datatype.
+                    let is_test_component = obj.kind == ObjectKind::Component && obj.is_testing(); // Test components are allowed to have datatypes for the moment. TODO(andreas): Should clean this up as well!
+                    if !is_enum_component && !is_test_component {
+                        reporter.error(virtpath, &obj.fqname, format!("Field {:?} s a primitive field of type {:?}. Primitive types are only allowed on DataTypes.", field.fqname, field.typ));
                     }
                 }
 

crates/store/re_chunk_store/tests/memory_test.rs

@@ -100,7 +100,7 @@ fn scalar_memory_overhead() {
             let entity_path = re_log_types::entity_path!("scalar");
             let timepoint =
                 TimePoint::default().with(Timeline::new("log_time", TimeType::Time), i as i64);
-            let scalars = Scalar::to_arrow([Scalar(i as f64)]).unwrap();
+            let scalars = Scalar::to_arrow([Scalar::from(i as f64)]).unwrap();
 
             let row = PendingRow::new(
                 timepoint,

crates/store/re_entity_db/tests/time_histograms.rs

@@ -465,7 +465,7 @@ fn time_histograms() -> anyhow::Result<()> {
                     TimePoint::from_iter([
                         (timeline_frame, 1000), //
                     ]),
-                    [&[ClearIsRecursive(true)] as _],
+                    [&[ClearIsRecursive(true.into())] as _],
                 )
                 .build()?
         };

crates/store/re_query/src/latest_at/query.rs

@@ -98,7 +98,7 @@ impl Caches {
                     #[allow(clippy::collapsible_if)] // readability
                     if clear_entity_path == *entity_path
                         || cached.mono::<ClearIsRecursive>(&crate::PromiseResolver {})
-                            == Some(ClearIsRecursive(true))
+                            == Some(ClearIsRecursive(true.into()))
                     {
                         if compare_indices(*cached.index(), max_clear_index)
                             == std::cmp::Ordering::Greater

crates/store/re_types/definitions/rerun/blueprint.fbs

@@ -1,4 +1,5 @@
 include "./blueprint/datatypes/tensor_dimension_index_slider.fbs";
+include "./blueprint/datatypes/utf8_list.fbs";
 
 include "./blueprint/components/active_tab.fbs";
 include "./blueprint/components/auto_layout.fbs";

crates/store/re_types/definitions/rerun/blueprint/components/auto_layout.fbs

@@ -19,5 +19,5 @@ struct AutoLayout (
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct"
 ) {
-  auto_layout: bool (order: 100);
+  auto_layout: rerun.datatypes.Bool (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/auto_space_views.fbs

@@ -19,5 +19,5 @@ struct AutoSpaceViews (
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct"
 ) {
-  auto_space_views: bool (order: 100);
+  auto_space_views: rerun.datatypes.Bool (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/column_share.fbs

@@ -18,5 +18,5 @@ table ColumnShare (
     "attr.rust.derive": "Default"
 ) {
     /// The layout shares of a column in the container.
-    share: float (order: 100);
+    share: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/grid_columns.fbs

@@ -16,5 +16,5 @@ table GridColumns (
     "attr.rust.override_crate": "re_types_blueprint"
 ) {
     /// The number of columns.
-    columns: uint (order: 100);
+    columns: rerun.datatypes.UInt32 (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/row_share.fbs

@@ -17,5 +17,5 @@ table RowShare (
     "attr.rust.derive": "Default"
 ) {
     /// The layout share of a row in the container.
-    share: float (order: 100);
+    share: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/visible.fbs

@@ -18,5 +18,5 @@ struct Visible (
     "attr.rust.repr": "transparent",
     "attr.rust.tuple_struct"
 ) {
-    visible: bool (order: 100);
+    visible: rerun.datatypes.Bool (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/components/visualizer_overrides.fbs

@@ -51,5 +51,5 @@ table VisualizerOverrides (
     /// - SegmentationImage
     /// - SeriesLine
     /// - SeriesPoint
-    visualizers: [string] (order: 100);
+    visualizers: rerun.blueprint.datatypes.Utf8List (order: 100);
 }

crates/store/re_types/definitions/rerun/blueprint/datatypes/utf8_list.fbs

@@ -0,0 +1,17 @@
+namespace rerun.blueprint.datatypes;
+
+/// A list of strings of text, encoded as UTF-8.
+//
+// NOTE: Apache Arrow uses UTF-8 encoding of its String type, as does Rust.
+table Utf8List (
+  "attr.docs.unreleased",
+  "attr.arrow.transparent",
+  "attr.python.aliases": "Sequence[str]",
+  "attr.rerun.scope": "blueprint",
+  "attr.rust.derive": "PartialEq, Eq, PartialOrd, Ord, Default",
+  "attr.rust.repr": "transparent",
+  "attr.rust.tuple_struct",
+  "attr.rust.override_crate": "re_types_blueprint"
+) {
+  value: [string] (order: 100);
+}

crates/store/re_types/definitions/rerun/components/blob.fbs

@@ -14,8 +14,8 @@ table Blob (
   "attr.arrow.transparent",
   "attr.python.aliases": "bytes, npt.NDArray[np.uint8]",
   "attr.python.array_aliases": "bytes, npt.NDArray[np.uint8]",
-  "attr.rust.derive": "PartialEq, Eq",
+  "attr.rust.derive": "Default, PartialEq, Eq",
   "attr.rust.repr": "transparent"
 ) {
-  data: [ubyte] (order: 100);
+  data: rerun.datatypes.Blob (order: 100);
 }

crates/store/re_types/definitions/rerun/components/clear_is_recursive.fbs

@@ -19,5 +19,5 @@ struct ClearIsRecursive (
   "attr.rust.tuple_struct"
 ) {
   /// If true, also clears all recursive children entities.
-  recursive: bool (order: 100);
+  recursive: rerun.datatypes.Bool (order: 100);
 }

crates/store/re_types/definitions/rerun/components/depth_meter.fbs

@@ -23,5 +23,5 @@ struct DepthMeter (
   "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent"
 ) {
-  value: float (order: 100);
+  value: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/disconnected_space.fbs

@@ -24,5 +24,5 @@ struct DisconnectedSpace (
   ///
   /// Set to true to disconnect the entity from its parent.
   /// Set to false to disable the effects of this component, (re-)connecting the entity to its parent again.
-  is_disconnected: bool (order: 100);
+  is_disconnected: rerun.datatypes.Bool (order: 100);
 }

crates/store/re_types/definitions/rerun/components/draw_order.fbs

@@ -21,5 +21,5 @@ struct DrawOrder (
   "attr.rust.derive": "Copy",
   "attr.rust.repr": "transparent"
 ) {
-   value: float (order: 100);
+   value: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/marker_size.fbs

@@ -16,5 +16,5 @@ struct MarkerSize (
   "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent"
 ) {
-  value: float (order: 100);
+  value: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/radius.fbs

@@ -23,5 +23,5 @@ struct Radius (
   "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent"
 ) {
-  value: float (order: 100);
+  value: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/scalar.fbs

@@ -18,5 +18,5 @@ struct Scalar (
   "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent"
 ) {
-  value: double (order: 100);
+  value: rerun.datatypes.Float64 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/stroke_width.fbs

@@ -16,5 +16,5 @@ struct StrokeWidth (
   "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent"
 ) {
-  width: float (order: 100);
+  width: rerun.datatypes.Float32 (order: 100);
 }

crates/store/re_types/definitions/rerun/components/view_coordinates.fbs

@@ -39,8 +39,9 @@ struct ViewCoordinates (
   "attr.python.aliases": "npt.ArrayLike",
   "attr.python.array_aliases": "npt.ArrayLike",
   "attr.rust.derive": "Copy, PartialEq, Eq, bytemuck::Pod, bytemuck::Zeroable",
-  "attr.rust.repr": "transparent"
+  "attr.rust.repr": "transparent",
+  "attr.cpp.no_field_ctors"
 ) {
   /// The directions of the [x, y, z] axes.
-  coordinates: [ubyte: 3] (order: 100);
+  coordinates: rerun.datatypes.ViewCoordinates (order: 100);
 }

crates/store/re_types/definitions/rerun/datatypes.fbs

@@ -1,11 +1,13 @@
 include "./datatypes/angle.fbs";
 include "./datatypes/annotation_info.fbs";
+include "./datatypes/blob.fbs";
 include "./datatypes/bool.fbs";
 include "./datatypes/class_description_map_elem.fbs";
 include "./datatypes/class_description.fbs";
 include "./datatypes/class_id.fbs";
 include "./datatypes/entity_path.fbs";
 include "./datatypes/float32.fbs";
+include "./datatypes/float64.fbs";
 include "./datatypes/keypoint_id.fbs";
 include "./datatypes/keypoint_pair.fbs";
 include "./datatypes/mat3x3.fbs";
@@ -36,6 +38,7 @@ include "./datatypes/uvec4d.fbs";
 include "./datatypes/vec2d.fbs";
 include "./datatypes/vec3d.fbs";
 include "./datatypes/vec4d.fbs";
+include "./datatypes/view_coordinates.fbs";
 include "./datatypes/visible_time_range.fbs";
 
 namespace rerun.datatypes;

crates/store/re_types/definitions/rerun/datatypes/blob.fbs

@@ -0,0 +1,24 @@
+include "arrow/attributes.fbs";
+include "python/attributes.fbs";
+include "rust/attributes.fbs";
+include "docs/attributes.fbs";
+include "rerun/attributes.fbs";
+
+include "rerun/datatypes.fbs";
+
+namespace rerun.datatypes;
+
+// ---
+
+/// A binary blob of data.
+table Blob (
+  "attr.docs.unreleased",
+  "attr.arrow.transparent",
+  "attr.python.aliases": "bytes, npt.NDArray[np.uint8]",
+  "attr.python.array_aliases": "bytes, npt.NDArray[np.uint8]",
+  "attr.rust.derive": "PartialEq, Eq",
+  "attr.rust.repr": "transparent",
+  "attr.rust.tuple_struct"
+) {
+  data: [ubyte] (order: 100);
+}

crates/store/re_types/definitions/rerun/datatypes/bool.fbs

@@ -15,6 +15,7 @@ struct Bool (
     "attr.arrow.transparent",
     "attr.python.aliases": "bool",
     "attr.rust.derive": "Copy, Default, PartialEq, Eq, PartialOrd, Ord",
+    "attr.rust.override_crate": "re_types_core",
     "attr.rust.repr": "transparent",
     "attr.rust.tuple_struct"
 ) {

crates/store/re_types/definitions/rerun/datatypes/float32.fbs

@@ -9,7 +9,8 @@ namespace rerun.datatypes;
 struct Float32 (
   "attr.arrow.transparent",
   "attr.python.aliases": "float",
-  "attr.rust.derive": "Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
+  "attr.python.array_aliases": "npt.NDArray[Any], npt.ArrayLike, Sequence[Sequence[float]], Sequence[float]",
+  "attr.rust.derive": "Default, Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.override_crate": "re_types_core",
   "attr.rust.repr": "transparent",
   "attr.rust.tuple_struct"

crates/store/re_types/definitions/rerun/datatypes/float64.fbs

@@ -0,0 +1,20 @@
+include "arrow/attributes.fbs";
+include "python/attributes.fbs";
+include "fbs/attributes.fbs";
+include "rust/attributes.fbs";
+
+namespace rerun.datatypes;
+
+/// A double-precision 64-bit IEEE 754 floating point number.
+struct Float64 (
+  "attr.docs.unreleased",
+  "attr.arrow.transparent",
+  "attr.python.aliases": "float",
+  "attr.python.array_aliases": "npt.NDArray[Any], npt.ArrayLike, Sequence[Sequence[float]], Sequence[float]",
+  "attr.rust.derive": "Default, Copy, PartialEq, PartialOrd, bytemuck::Pod, bytemuck::Zeroable",
+  "attr.rust.override_crate": "re_types_core",
+  "attr.rust.repr": "transparent",
+  "attr.rust.tuple_struct"
+) {
+  value: double (order: 100);
+}

crates/store/re_types/definitions/rerun/datatypes/uint32.fbs

@@ -11,7 +11,8 @@ struct UInt32 (
   "attr.arrow.transparent",
   "attr.python.aliases": "int",
   "attr.python.array_aliases": "int, npt.NDArray[np.uint32]",
-  "attr.rust.derive": "Copy, PartialEq, Eq, PartialOrd, Ord",
+  "attr.rust.derive": "Default, Copy, PartialEq, Eq, PartialOrd, Ord",
+  "attr.rust.override_crate": "re_types_core",
   "attr.rust.override_crate": "re_types_core",
   "attr.rust.tuple_struct"
 ) {

crates/store/re_types/definitions/rerun/datatypes/uint64.fbs

@@ -11,7 +11,7 @@ struct UInt64 (
   "attr.arrow.transparent",
   "attr.python.aliases": "int",
   "attr.python.array_aliases": "int, npt.NDArray[np.uint64]",
-  "attr.rust.derive": "Copy, PartialEq, Eq, PartialOrd, Ord",
+  "attr.rust.derive": "Default, Copy, PartialEq, Eq, PartialOrd, Ord",
   "attr.rust.override_crate": "re_types_core",
   "attr.rust.tuple_struct"
 ) {

crates/store/re_types/definitions/rerun/datatypes/view_coordinates.fbs

@@ -0,0 +1,49 @@
+include "arrow/attributes.fbs";
+include "python/attributes.fbs";
+include "rust/attributes.fbs";
+
+include "rerun/attributes.fbs";
+
+namespace rerun.datatypes;
+
+
+// TODO(#6320)
+/*
+enum ViewDir: byte {
+    Up = 1,
+    Down = 2,
+    Right = 3,
+    Left = 4,
+    Forward = 5,
+    Back = 6,
+}
+*/
+
+/// How we interpret the coordinate system of an entity/space.
+///
+/// For instance: What is "up"? What does the Z axis mean? Is this right-handed or left-handed?
+///
+/// The three coordinates are always ordered as [x, y, z].
+///
+/// For example [Right, Down, Forward] means that the X axis points to the right, the Y axis points
+/// down, and the Z axis points forward.
+///
+/// The following constants are used to represent the different directions:
+///  * Up = 1
+///  * Down = 2
+///  * Right = 3
+///  * Left = 4
+///  * Forward = 5
+///  * Back = 6
+struct ViewCoordinates (
+  "attr.docs.unreleased",
+  "attr.arrow.transparent",
+  "attr.python.aliases": "npt.ArrayLike",
+  "attr.python.array_aliases": "npt.ArrayLike",
+  "attr.rust.derive": "Copy, PartialEq, Eq, bytemuck::Pod, bytemuck::Zeroable",
+  "attr.rust.repr": "transparent",
+  "attr.rust.tuple_struct"
+) {
+  /// The directions of the [x, y, z] axes.
+  coordinates: [ubyte: 3] (order: 100);
+}