Python codegen: big cleaning and paving the way towards transforms

.github/workflows/reusable_build_and_test_wheels.yml

@@ -243,7 +243,7 @@ jobs:
         # TODO(jleibs): understand why deps can't be installed in the same step as the wheel
         shell: bash
         run: |
-          pip install deprecated numpy>=1.23 pillow pyarrow==10.0.1 pytest==7.1.2
+          pip install attrs>=23.1.0 deprecated numpy>=1.23 pillow pyarrow==10.0.1 pytest==7.1.2
 
       - name: Install built wheel
         if: needs.set-config.outputs.RUN_TESTS == 'true'

.github/workflows/reusable_checks.yml

@@ -62,7 +62,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "3.11"
           cache: "pip"
           cache-dependency-path: "rerun_py/requirements-lint.txt"
 

crates/re_types/definitions/python/attributes.fbs

@@ -1,10 +1,5 @@
 namespace python.attributes;
 
-/// Marks a field as transparent, meaning its type will be replaced by the underlying type.
-///
-/// Only applies to fields whose type is an object with a single-field.
-attribute "attr.python.transparent";
-
 /// Defines the type aliases for a component, e.g. the types that make up `ComponentLike`.
 ///
 /// Only applies to structs/unions that are components.

crates/re_types/definitions/rerun/components/color.fbs

@@ -16,8 +16,8 @@ namespace rerun.components;
 /// \py If there is an alpha, we assume it is in linear space, and separate (NOT pre-multiplied).
 struct Color (
   "attr.arrow.transparent",
-  "attr.python.aliases": "int, npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]",
-  "attr.python.array_aliases": "Sequence[int], npt.NDArray[np.uint8], npt.NDArray[np.uint32], npt.NDArray[np.float32], npt.NDArray[np.float64]",
+  "attr.python.aliases": "int, Sequence[int], npt.NDArray[Union[np.uint8, np.float32, np.float64]]",
+  "attr.python.array_aliases": "Sequence[Sequence[int]], npt.NDArray[Union[np.uint8, np.uint32, np.float32, np.float64]]",
   "attr.rerun.legacy_fqname": "rerun.colorrgba",
   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, bytemuck::Pod, bytemuck::Zeroable",
   "attr.rust.repr": "transparent",

crates/re_types/definitions/rerun/components/label.fbs

@@ -13,6 +13,7 @@ namespace rerun.components;
 table Label (
   "attr.arrow.transparent",
   "attr.python.aliases": "str",
+  "attr.python.array_aliases": "Sequence[str]",
   "attr.rerun.legacy_fqname": "rerun.label",
   "attr.rust.derive": "Debug, Clone, PartialEq, Eq, PartialOrd, Ord",
   "attr.rust.repr": "transparent",

crates/re_types/definitions/rerun/components/point2d.fbs

@@ -10,32 +10,14 @@ namespace rerun.components;
 // ---
 
 /// A point in 2D space.
+// TODO(cmc): bring back attr.rust.tuple_struct
 struct Point2D (
+  "attr.arrow.transparent",
   "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]",
   "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]",
   "attr.rerun.legacy_fqname": "rerun.point2d",
   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
   order: 100
 ) {
-  x: float (order: 100);
-  y: float (order: 200);
+  xy: rerun.datatypes.Point2D (order: 100);
 }
-
-// ---
-
-// TODO(cmc): use the following definition instead once we've finalized the switch to HOPE.
-
-// /// A point in 2D space.
-// struct Point2D (
-//   "attr.arrow.transparent",
-//   "attr.python.aliases": "npt.NDArray[np.float32], Sequence[float], Tuple[float, float]",
-//   "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[float]",
-//   "attr.rust.tuple_struct",
-//   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
-//   order: 100
-// ) {
-//   position: rerun.datatypes.Vec2D (
-//     "attr.python.transparent",
-//     order: 100
-//   );
-// }

crates/re_types/definitions/rerun/datatypes.fbs

@@ -1,3 +1,5 @@
+include "./datatypes/point2d.fbs";
 include "./datatypes/vec2d.fbs";
 
+
 namespace rerun.datatypes;

crates/re_types/definitions/rerun/datatypes/point2d.fbs

@@ -0,0 +1,19 @@
+include "arrow/attributes.fbs";
+include "python/attributes.fbs";
+include "fbs/attributes.fbs";
+include "rust/attributes.fbs";
+
+namespace rerun.datatypes;
+
+// ---
+
+/// A point in 2D space.
+struct Point2D (
+  "attr.python.aliases": "Sequence[float]",
+  "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[npt.NDArray[np.float32]], Sequence[Tuple[float, float]], Sequence[float]",
+  "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
+  order: 100
+) {
+  x: float (order: 100);
+  y: float (order: 200);
+}

crates/re_types/definitions/rerun/datatypes/vec2d.fbs

@@ -1,4 +1,5 @@
 include "arrow/attributes.fbs";
+include "python/attributes.fbs";
 include "fbs/attributes.fbs";
 include "rust/attributes.fbs";
 
@@ -9,6 +10,8 @@ namespace rerun.datatypes;
 /// A vector in 2D space.
 struct Vec2D (
   "attr.arrow.transparent",
+  "attr.python.aliases": "Tuple[float, float]",
+  "attr.python.array_aliases": "npt.NDArray[np.float32], Sequence[Tuple[float, float]], Sequence[float]",
   "attr.rust.derive": "Debug, Default, Clone, Copy, PartialEq, PartialOrd",
   "attr.rust.tuple_struct",
   order: 100

crates/re_types/definitions/rerun/testing/archetypes/fuzzy.fbs

@@ -21,30 +21,30 @@ table AffixFuzzer1 (
   fuzz1004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_required", required, order: 1004);
   fuzz1005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_required", required, order: 1005);
   fuzz1006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_required", required, order: 1006);
-  fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007);
+  // fuzz1007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_required", required, order: 1007);
 
   fuzz1101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_required", required, order: 1101);
   fuzz1102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_required", required, order: 1102);
   fuzz1103: [rerun.testing.components.AffixFuzzer3] ("attr.rerun.component_required", required, order: 1103);
   fuzz1104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_required", required, order: 1104);
   fuzz1105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_required", required, order: 1105);
   fuzz1106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_required", required, order: 1106);
-  fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107);
+  // fuzz1107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_required", required, order: 1107);
 
   fuzz2001: rerun.testing.components.AffixFuzzer1 ("attr.rerun.component_optional", order: 2001);
   fuzz2002: rerun.testing.components.AffixFuzzer2 ("attr.rerun.component_optional", order: 2002);
   fuzz2003: rerun.testing.components.AffixFuzzer3 ("attr.rerun.component_optional", order: 2003);
   fuzz2004: rerun.testing.components.AffixFuzzer4 ("attr.rerun.component_optional", order: 2004);
   fuzz2005: rerun.testing.components.AffixFuzzer5 ("attr.rerun.component_optional", order: 2005);
   fuzz2006: rerun.testing.components.AffixFuzzer6 ("attr.rerun.component_optional", order: 2006);
-  fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007);
+  // fuzz2007: rerun.testing.components.AffixFuzzer7 ("attr.rerun.component_optional", order: 2007);
 
   fuzz2101: [rerun.testing.components.AffixFuzzer1] ("attr.rerun.component_optional", order: 2101);
   fuzz2102: [rerun.testing.components.AffixFuzzer2] ("attr.rerun.component_optional", order: 2102);
   fuzz2103: [rerun.testing.components.AffixFuzzer3] ("attr.rerun.component_optional", order: 2103);
   fuzz2104: [rerun.testing.components.AffixFuzzer4] ("attr.rerun.component_optional", order: 2104);
   fuzz2105: [rerun.testing.components.AffixFuzzer5] ("attr.rerun.component_optional", order: 2105);
   fuzz2106: [rerun.testing.components.AffixFuzzer6] ("attr.rerun.component_optional", order: 2106);
-  fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107);
+  // fuzz2107: [rerun.testing.components.AffixFuzzer7] ("attr.rerun.component_optional", order: 2107);
 }
 

crates/re_types/definitions/rerun/testing/components/fuzzy.fbs

@@ -56,17 +56,17 @@ table AffixFuzzer6 (
   single_optional: rerun.testing.datatypes.AffixFuzzer1;
 }
 
-table AffixFuzzer7 (
-  "attr.rust.derive": "Debug, Clone, PartialEq",
-  order: 700
-) {
-  many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100);
-  single_float_optional: float (order: 101);
-  single_string_required: string (order: 102, required);
-  single_string_optional: string (order: 103);
-  many_floats_optional: [float] (order: 104);
-  many_strings_required: [string] (order: 105, required);
-  many_strings_optional: [string] (order: 106);
-  // TODO(cmc): the ugly bug we need to take care of at some point
-  // many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107);
-}
+// table AffixFuzzer7 (
+//   "attr.rust.derive": "Debug, Clone, PartialEq",
+//   order: 700
+// ) {
+//   many_optional: [rerun.testing.datatypes.AffixFuzzer1] (order: 100);
+//   single_float_optional: float (order: 101);
+//   single_string_required: string (order: 102, required);
+//   single_string_optional: string (order: 103);
+//   many_floats_optional: [float] (order: 104);
+//   many_strings_required: [string] (order: 105, required);
+//   many_strings_optional: [string] (order: 106);
+//   // TODO(cmc): the ugly bug we need to take care of at some point
+//   // many_transparent_optionals: rerun.testing.datatypes.AffixFuzzer2 (order: 107);
+// }

crates/re_types/source_hash.txt

@@ -1,4 +1,4 @@
 # This is a sha256 hash for all direct and indirect dependencies of this crate's build script.
 # It can be safely removed at anytime to force the build script to run again.
 # Check out build.rs to see how it's computed.
-d828fdb05c35a01b92dd30b7ce40987b87c5dfe8d1bb8f729004ae88b62c830c
+51003743447ae6a7b2985c1b6761319e42eb446fcb86248431871bddf5817985