diff --git a/Cargo.toml b/Cargo.toml index 715bcfd..7be6878 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,9 @@ exclude = ["benches", "tests"] readme = "README.md" [dependencies] -arrow2 = { version = "0.17", features = ["io_ipc", "io_ipc_compression", "io_json" ] } +arrow = "52" +arrow-buffer = "52" +arrow-ipc = "52" byteorder = "1" encoding_rs = "0.8" log = "0.4" @@ -22,7 +24,6 @@ thiserror = "1.0" xxhash-rust = { version = "0.8", features = ["xxh3"] } [dev-dependencies] -arrow2 = { version = "0.17", features = ["io_json"] } criterion = "0.5" iai-callgrind = "0.11" pretty_assertions = "1.3" diff --git a/gen/resources/preamble/immutable/mod.rs b/gen/resources/preamble/immutable/mod.rs index 4c2838a..08a667c 100644 --- a/gen/resources/preamble/immutable/mod.rs +++ b/gen/resources/preamble/immutable/mod.rs @@ -4,7 +4,7 @@ //! [`crate::io::peppi::read`]. //! //! These arrays can be shared, and cloning them is `O(1)`. See the -//! [arrow2 docs](https://docs.rs/arrow2/latest/arrow2/array/index.html) for more. +//! [arrow_array docs](https://docs.rs/arrow-array/latest/arrow_array/index.html) for more. #![allow(unused_variables)] @@ -13,11 +13,12 @@ mod slippi; use std::fmt; -use arrow2::{ - array::PrimitiveArray, - bitmap::Bitmap, - buffer::Buffer, - offset::OffsetsBuffer, +use arrow::{ + array::{ + types::{Float32Type, Int8Type, Int16Type, Int32Type, UInt8Type, UInt16Type, UInt32Type}, + PrimitiveArray, + }, + buffer::{NullBuffer, OffsetBuffer}, }; use crate::{ @@ -31,7 +32,7 @@ use crate::{ pub struct Data { pub pre: Pre, pub post: Post, - pub validity: Option, + pub validity: Option, } impl Data { @@ -43,15 +44,15 @@ impl Data { } } -impl From for Data { - fn from(d: mutable::Data) -> Self { - Self { - pre: d.pre.into(), - post: d.post.into(), - validity: d.validity.map(|v| v.into()), - } - } -} +//impl From for Data { +// fn from(d: mutable::Data) -> Self { +// Self { +// pre: d.pre.finish(), +// post: d.post.finish(), +// validity: d.validity.map(|v| v.into()), +// } +// } +//} /// Frame data for a single port. #[derive(Debug)] @@ -72,20 +73,20 @@ impl PortData { } } -impl From for PortData { - fn from(p: mutable::PortData) -> Self { - Self { - port: p.port, - leader: p.leader.into(), - follower: p.follower.map(|f| f.into()), - } - } -} +//impl From for PortData { +// fn from(p: mutable::PortData) -> Self { +// Self { +// port: p.port, +// leader: p.leader.finish(), +// follower: p.follower.map(|f| f.finish()), +// } +// } +//} /// All frame data for a single game, in struct-of-arrays format. pub struct Frame { /// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks - pub id: PrimitiveArray, + pub id: PrimitiveArray, /// Port-specific data pub ports: Vec, /// Start-of-frame data @@ -93,7 +94,7 @@ pub struct Frame { /// End-of-frame data pub end: Option, /// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array - pub item_offset: Option>, + pub item_offset: Option>, /// Item data pub item: Option, } @@ -135,7 +136,7 @@ impl Frame { fn rollbacks_<'a>(&self, ids: impl Iterator) -> Vec { let mut result = vec![false; self.len()]; - let unique_id_count = self.id.values_iter().max().map_or(0, |idx| { + let unique_id_count = arrow::compute::kernels::aggregate::max(&self.id).map_or(0, |idx| { 1 + usize::try_from(idx - frame::FIRST_INDEX).unwrap() }); let mut seen = vec![false; unique_id_count]; @@ -152,20 +153,20 @@ impl Frame { } } -impl From for Frame { - fn from(f: mutable::Frame) -> Self { - Self { - id: f.id.into(), - ports: f.ports.into_iter().map(|p| p.into()).collect(), - start: f.start.map(|x| x.into()), - end: f.end.map(|x| x.into()), - item_offset: f.item_offset.map(|x| - OffsetsBuffer::try_from(Buffer::from(x.into_inner())).unwrap() - ), - item: f.item.map(|x| x.into()), - } - } -} +//impl From for Frame { +// fn from(f: mutable::Frame) -> Self { +// Self { +// id: f.id.into(), +// ports: f.ports.into_iter().map(|p| p.into()).collect(), +// start: f.start.map(|x| x.into()), +// end: f.end.map(|x| x.into()), +// item_offset: f.item_offset.map(|x| +// OffsetBuffer::try_from(Buffer::from(x.into_inner())).unwrap() +// ), +// item: f.item.map(|x| x.into()), +// } +// } +//} impl fmt::Debug for Frame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> { diff --git a/gen/resources/preamble/immutable/peppi.rs b/gen/resources/preamble/immutable/peppi.rs index b89441f..69cf561 100644 --- a/gen/resources/preamble/immutable/peppi.rs +++ b/gen/resources/preamble/immutable/peppi.rs @@ -1,8 +1,13 @@ #![allow(unused_variables)] -use arrow2::{ - array::{ListArray, PrimitiveArray, StructArray}, - datatypes::{DataType, Field}, +use std::sync::Arc; + +use arrow::{ + array::{ + types::{Float32Type, Int8Type, Int16Type, Int32Type, UInt8Type, UInt16Type, UInt32Type}, + Array, ArrayRef, ListArray, PrimitiveArray, StructArray, + }, + datatypes::{DataType, Field, Fields}, }; use crate::{ @@ -15,23 +20,29 @@ use crate::{ }; impl Data { + fn fields(version: Version) -> Fields { + Fields::from( + vec![ + Field::new("pre", Pre::data_type(version).clone(), false), + Field::new("post", Post::data_type(version).clone(), false), + ] + ) + } + fn data_type(version: Version) -> DataType { - DataType::Struct(vec![ - Field::new("pre", Pre::data_type(version).clone(), false), - Field::new("post", Post::data_type(version).clone(), false), - ]) + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let values = vec![ - self.pre.into_struct_array(version).boxed(), - self.post.into_struct_array(version).boxed(), + Arc::new(self.pre.into_struct_array(version)) as ArrayRef, + Arc::new(self.post.into_struct_array(version)) as ArrayRef, ]; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { pre: Pre::from_struct_array( values[0] @@ -55,7 +66,7 @@ impl Data { } impl PortData { - fn data_type(version: Version, port: PortOccupancy) -> DataType { + fn fields(version: Version, port: PortOccupancy) -> Fields { let mut fields = vec![Field::new( "leader", Data::data_type(version).clone(), @@ -68,21 +79,25 @@ impl PortData { false, )); } - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version, port: PortOccupancy) -> DataType { + DataType::Struct(Self::fields(version, port)) } fn into_struct_array(self, version: Version, port: PortOccupancy) -> StructArray { - let mut values = vec![self.leader.into_struct_array(version).boxed()]; + let mut values = vec![Arc::new(self.leader.into_struct_array(version)) as Arc]; if let Some(follower) = self.follower { - values.push(follower.into_struct_array(version).boxed()); + values.push(Arc::new(follower.into_struct_array(version)) as Arc); } - StructArray::new(Self::data_type(version, port), values, None) + StructArray::new(Self::fields(version, port), values, None) } fn from_struct_array(array: StructArray, version: Version, port: Port) -> Self { - let (fields, values, _) = array.into_data(); - assert_eq!("leader", fields[0].name); - fields.get(1).map(|f| assert_eq!("follower", f.name)); + let (fields, values, _) = array.into_parts(); + assert_eq!("leader", fields[0].name()); + fields.get(1).map(|f| assert_eq!("follower", f.name())); Self { port: port, leader: Data::from_struct_array( @@ -104,31 +119,41 @@ impl PortData { } impl Frame { - fn port_data_type(version: Version, ports: &[PortOccupancy]) -> DataType { - DataType::Struct( - ports.iter().map(|p| { - Field::new( - format!("{}", p.port), - PortData::data_type(version, *p).clone(), - false, - ) - }) - .collect(), + fn port_fields(version: Version, ports: &[PortOccupancy]) -> Fields { + Fields::from( + ports + .iter() + .map(|p| { + Field::new( + format!("{}", p.port), + PortData::data_type(version, *p).clone(), + false, + ) + }) + .collect(), ) } - fn item_data_type(version: Version) -> DataType { - DataType::List(Box::new(Field::new( + fn port_data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + DataType::Struct(Self::port_fields(version, ports)) + } + + fn item_field(version: Version) -> Field { + Field::new( "item", Item::data_type(version), false, - ))) + ) } - fn data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + fn item_data_type(version: Version) -> DataType { + DataType::List(Arc::new(Self::item_field(version))) + } + + fn fields(version: Version, ports: &[PortOccupancy]) -> Fields { let mut fields = vec![ Field::new("id", DataType::Int32, false), - Field::new("ports", Self::port_data_type(version, ports).clone(), false), + Field::new("ports", Self::port_data_type(version, ports), false), ]; if version.gte(2, 2) { fields.push(Field::new("start", Start::data_type(version).clone(), false)); @@ -137,45 +162,49 @@ impl Frame { fields.push(Field::new("item", Self::item_data_type(version).clone(), false)); } } - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + DataType::Struct(Self::fields(version, ports)) } pub fn into_struct_array(self, version: Version, ports: &[PortOccupancy]) -> StructArray { let values: Vec<_> = std::iter::zip(ports, self.ports) - .map(|(occupancy, data)| data.into_struct_array(version, *occupancy).boxed()) + .map(|(occupancy, data)| Arc::new(data.into_struct_array(version, *occupancy)) as Arc) .collect(); let mut arrays = vec![ - self.id.boxed(), - StructArray::new(Self::port_data_type(version, ports), values, None).boxed(), + Arc::new(self.id) as Arc, + Arc::new(StructArray::new(Self::port_fields(version, ports), values, None)) as Arc, ]; if version.gte(2, 2) { - arrays.push(self.start.unwrap().into_struct_array(version).boxed()); + arrays.push(Arc::new(self.start.unwrap().into_struct_array(version)) as Arc); if version.gte(3, 0) { - arrays.push(self.end.unwrap().into_struct_array(version).boxed()); - let item_values = self.item.unwrap().into_struct_array(version).boxed(); - arrays.push(ListArray::new( - Self::item_data_type(version), + arrays.push(Arc::new(self.end.unwrap().into_struct_array(version)) as Arc); + let item_values = Arc::new(self.item.unwrap().into_struct_array(version)); + arrays.push(Arc::new(ListArray::new( + Arc::new(Self::item_field(version)), self.item_offset.unwrap(), item_values, None, - ).boxed()); + )) as Arc); } } - StructArray::new(Self::data_type(version, ports), arrays, None) + StructArray::new(Self::fields(version, ports), arrays, None) } fn port_data_from_struct_array(array: StructArray, version: Version) -> Vec { - let (fields, values, _) = array.into_data(); + let (fields, values, _) = array.into_parts(); let mut ports = vec![]; for i in 0 .. NUM_PORTS { if let Some(a) = values.get(i as usize) { ports.push(PortData::from_struct_array( a.as_any().downcast_ref::().unwrap().clone(), version, - Port::parse(&fields[i as usize].name).unwrap(), + Port::parse(&fields[i as usize].name()).unwrap(), )); } } @@ -183,20 +212,20 @@ impl Frame { } pub fn from_struct_array(array: StructArray, version: Version) -> Self { - let (fields, values, _) = array.into_data(); - assert_eq!("id", fields[0].name); - assert_eq!("ports", fields[1].name); + let (fields, values, _) = array.into_parts(); + assert_eq!("id", fields[0].name()); + assert_eq!("ports", fields[1].name()); if version.gte(2, 2) { - assert_eq!("start", fields[2].name); + assert_eq!("start", fields[2].name()); if version.gte(3, 0) { - assert_eq!("end", fields[3].name); - assert_eq!("item", fields[4].name); + assert_eq!("end", fields[3].name()); + assert_eq!("item", fields[4].name()); } } let (item, item_offset) = values.get(4).map_or((None, None), |v| { let arrays = v.as_any() - .downcast_ref::>() + .downcast_ref::() .unwrap() .clone(); let item_offset = arrays.offsets().clone(); @@ -214,7 +243,7 @@ impl Frame { Self { id: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), ports: Self::port_data_from_struct_array( diff --git a/gen/resources/preamble/immutable/slippi.rs b/gen/resources/preamble/immutable/slippi.rs index 5080e26..34795d9 100644 --- a/gen/resources/preamble/immutable/slippi.rs +++ b/gen/resources/preamble/immutable/slippi.rs @@ -26,7 +26,7 @@ impl Data { frame_id: i32, port: PortOccupancy, ) -> Result<()> { - if self.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if self.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { w.write_u8(Event::FramePre as u8)?; w.write_i32::(frame_id)?; w.write_u8(port.port as u8)?; @@ -47,7 +47,7 @@ impl Data { frame_id: i32, port: PortOccupancy, ) -> Result<()> { - if self.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if self.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { w.write_u8(Event::FramePost as u8)?; w.write_i32::(frame_id)?; w.write_u8(port.port as u8)?; @@ -82,7 +82,7 @@ impl PortData { self.follower .as_ref() .map_or(Ok(()), |f| { - if f.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if f.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { f.write_pre( w, version, @@ -119,7 +119,7 @@ impl PortData { self.follower .as_ref() .map_or(Ok(()), |f| { - if f.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if f.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { f.write_post( w, version, diff --git a/gen/resources/preamble/mutable.rs b/gen/resources/preamble/mutable.rs index d5ac252..2194ffc 100644 --- a/gen/resources/preamble/mutable.rs +++ b/gen/resources/preamble/mutable.rs @@ -5,18 +5,18 @@ #![allow(unused_variables)] #![allow(dead_code)] -use arrow2::{ - array::{MutableArray, MutablePrimitiveArray}, - bitmap::MutableBitmap, - offset::Offsets, +use arrow::array::{ + types::{Float32Type, Int8Type, Int16Type, Int32Type, UInt8Type, UInt16Type, UInt32Type}, + ArrayBuilder, PrimitiveBuilder, }; +use arrow_buffer::builder::{NullBufferBuilder, OffsetBufferBuilder}; use byteorder::ReadBytesExt; use std::io::Result; use crate::{ io::slippi::Version, - frame::{transpose, PortOccupancy}, + frame::{immutable, transpose, PortOccupancy}, game::Port, }; @@ -26,7 +26,7 @@ type BE = byteorder::BigEndian; pub struct Data { pub pre: Pre, pub post: Post, - pub validity: Option, + pub validity: NullBufferBuilder, } impl Data { @@ -34,7 +34,7 @@ impl Data { Self { pre: Pre::with_capacity(capacity, version), post: Post::with_capacity(capacity, version), - validity: None, + validity: NullBufferBuilder::new(capacity), } } @@ -44,9 +44,7 @@ impl Data { pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); + self.validity.append(false); self.pre.push_null(version); self.post.push_null(version); } @@ -57,6 +55,14 @@ impl Data { post: self.post.transpose_one(i, version), } } + + pub fn finish(&mut self) -> immutable::Data { + immutable::Data { + pre: self.pre.finish(), + post: self.post.finish(), + validity: self.validity.finish(), + } + } } /// Frame data for a single port. @@ -90,35 +96,43 @@ impl PortData { follower: self.follower.as_ref().map(|f| f.transpose_one(i, version)), } } + + pub fn finish(&mut self) -> immutable::PortData { + immutable::PortData { + port: self.port, + leader: self.leader.finish(), + follower: self.follower.as_mut().map(|f| f.finish()), + } + } } /// All frame data for a single game, in struct-of-arrays format. pub struct Frame { /// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks - pub id: MutablePrimitiveArray, + pub id: PrimitiveBuilder, /// Port-specific data pub ports: Vec, /// Start-of-frame data pub start: Option, /// End-of-frame data pub end: Option, - /// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array - pub item_offset: Option>, - /// Item data + /// Item data. Logically, each frame has its own array of items. But we represent all item data in a flat array, with `item_offset` indicating the start of each frame's sub-array pub item: Option, + /// Item array offsets (see `item`) + pub item_offset: Option>, } impl Frame { pub fn with_capacity(capacity: usize, version: Version, ports: &[PortOccupancy]) -> Self { Self { - id: MutablePrimitiveArray::::with_capacity(capacity), + id: PrimitiveBuilder::with_capacity(capacity), ports: ports .iter() .map(|p| PortData::with_capacity(capacity, version, *p)) .collect(), start: version.gte(2, 2).then(|| Start::with_capacity(capacity, version)), end: version.gte(3, 0).then(|| End::with_capacity(capacity, version)), - item_offset: version.gte(3, 0).then(|| Offsets::::with_capacity(capacity)), + item_offset: version.gte(3, 0).then(|| OffsetBufferBuilder::::new(capacity)), item: version.gte(3, 0).then(|| Item::with_capacity(0, version)), } } @@ -129,7 +143,7 @@ impl Frame { pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Frame { transpose::Frame { - id: self.id.values()[i], + id: self.id.values_slice()[i], ports: self.ports.iter().map(|p| p.transpose_one(i, version)).collect(), start: version.gte(2, 2).then(|| self.start.as_ref().unwrap().transpose_one(i, version)), end: version.gte(3, 0).then(|| self.end.as_ref().unwrap().transpose_one(i, version)), @@ -141,4 +155,27 @@ impl Frame { }), } } + + pub fn finish(mut self) -> immutable::Frame { + immutable::Frame { + id: self.id.finish(), + ports: vec![], //self.ports.iter_mut().map(|p| p.finish()).collect(), + start: match &mut self.start { + Some(x) => Some(x.finish()), + _ => None, + }, + end: match &mut self.end { + Some(x) => Some(x.finish()), + _ => None, + }, + item: match &mut self.item { + Some(x) => Some(x.finish()), + _ => None, + }, + item_offset: match &mut self.item_offset { + Some(x) => Some(x.finish()), + _ => None, + }, + } + } } diff --git a/gen/src/peppi_codegen/common.clj b/gen/src/peppi_codegen/common.clj index bb5e7a9..58cb1e9 100644 --- a/gen/src/peppi_codegen/common.clj +++ b/gen/src/peppi_codegen/common.clj @@ -191,6 +191,10 @@ [_ op lhs rhs] (format "%s %s %s" (emit-expr lhs) op (emit-expr rhs))) +(defmethod emit-expr* :cast + [_ expr ty] + (format "%s as %s" (emit-expr expr) (emit-type ty))) + (defmethod emit-expr* :subscript [_ target idx] (format "%s[%s]" (emit-expr target) idx)) diff --git a/gen/src/peppi_codegen/frame/common.clj b/gen/src/peppi_codegen/frame/common.clj index e192bd6..40cb7b7 100644 --- a/gen/src/peppi_codegen/frame/common.clj +++ b/gen/src/peppi_codegen/frame/common.clj @@ -3,6 +3,17 @@ [peppi-codegen.common :refer [read-json]] [clojure.string :as str])) +(defn arrow-type + [ty] + (case ty + "i8" "Int8Type" + "u8" "UInt8Type" + "i16" "Int16Type" + "u16" "UInt16Type" + "i32" "Int32Type" + "u32" "UInt32Type" + "f32" "Float32Type")) + (defn field-docstring [desc ver] (some-> desc diff --git a/gen/src/peppi_codegen/frame/immutable/mod.clj b/gen/src/peppi_codegen/frame/immutable/mod.clj index 3e7cc24..b4cb400 100644 --- a/gen/src/peppi_codegen/frame/immutable/mod.clj +++ b/gen/src/peppi_codegen/frame/immutable/mod.clj @@ -7,7 +7,7 @@ (defn array-type [ty] (cond - (primitive-types ty) ["PrimitiveArray" ty] + (primitive-types ty) ["PrimitiveArray" (arrow-type ty)] (nil? ty) "NullArray" :else ty)) @@ -52,17 +52,6 @@ (filterv :type) (mapv (juxt :name transpose-one-field-init)))]]])) -(defn into-immutable - [{idx :index, nm :name, ver :version}] - (let [target [:field-get "x" (or nm idx)]] - (if ver - (wrap-map target "x" [:method-call "x" "into" []]) - [:method-call target "into" []]))) - -(defn mutable - [ty] - (list "mutable" ty)) - (defmulti struct-decl (fn [[nm {:keys [fields]}]] (named? fields))) @@ -77,7 +66,7 @@ (append [:struct-field {:docstring "Indicates which indexes are valid (`None` means \"all valid\"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games)"} "validity" - ["Option" "Bitmap"]]))]) + ["Option" "NullBuffer"]]))]) (defmethod struct-decl false [[nm {:keys [description fields]}]] @@ -91,7 +80,18 @@ [[nm {:keys [fields]}]] [:impl nm [(transpose-one-fn nm fields)]]) -(defn struct-from-impl +#_(defn mutable + [ty] + (list "mutable" ty)) + +#_(defn into-immutable + [{idx :index, nm :name, ver :version}] + (let [target [:field-get "x" (or nm idx)]] + (if ver + (wrap-map target "x" [:method-call "x" "finish" []]) + [:method-call target "finish" []]))) + +#_(defn struct-from-impl [[nm {:keys [fields]}]] [:impl {:for nm} @@ -101,15 +101,16 @@ "from" [["x" (mutable nm)]] [:block - [:struct-init "Self" (cond->> (mapv (juxt :name into-immutable) fields) - (named? fields) (append ["validity" - [:method-call - [:field-get "x" "validity"] - "map" - [[:closure - [["v"]] - [[:method-call "v" "into" []]]]]]]))]]]]]) + [:struct-init + "Self" + (cond->> (mapv (juxt :name into-immutable) fields) + (named? fields) + (append ["validity" + [:method-call + [:field-get "x" "validity"] + "finish" + []]]))]]]]]) (defn -main [] - (doseq [decl (mapcat (juxt struct-decl struct-impl struct-from-impl) (read-structs))] + (doseq [decl (mapcat (juxt struct-decl struct-impl) (read-structs))] (println (emit-expr decl) "\n"))) diff --git a/gen/src/peppi_codegen/frame/immutable/peppi.clj b/gen/src/peppi_codegen/frame/immutable/peppi.clj index 614fff3..df555d6 100644 --- a/gen/src/peppi_codegen/frame/immutable/peppi.clj +++ b/gen/src/peppi_codegen/frame/immutable/peppi.clj @@ -17,11 +17,11 @@ (types ty [:fn-call ty "data_type" ["version"]]) "false"]]) -(defn data-type-fn +(defn fields-fn [fields] [:fn - {:ret "DataType"} - "data_type" + {:ret "Fields"} + "fields" [["version" "Version"]] [:block [:let @@ -36,22 +36,31 @@ "push" [(arrow-field f)]])) (into [:block])) + [:fn-call "Fields" "from" ["fields"]]]]) + +(defn data-type-fn + [] + [:fn + {:ret "DataType"} + "data_type" + [["version" "Version"]] + [:block [:struct-init (list "DataType" "Struct") - [[nil "fields"]]]]]) + [[nil [:fn-call "Self" "fields" ["version"]]]]]]]) + +(defn into-struct-array + [target] + [:method-call target "into_struct_array" ["version"]]) (defn arrow-values [{nm :name, ty :type, idx :index, ver :version}] (let [target (cond-> [:field-get "self" (or nm idx)] - ver (#(vector :method-call % "unwrap")))] - (if (types ty) - [:method-call - target - "boxed"] - [:method-call - [:method-call target "into_struct_array" ["version"]] - "boxed" - []]))) + ver (#(vector :method-call % "unwrap")) + (not (types ty)) into-struct-array)] + [:cast + [:fn-call "Arc" "new" [target]] + ["Arc" "dyn Array"]])) (defn push-call [field] @@ -66,7 +75,7 @@ struct-init [:fn-call "StructArray" "new" - [[:fn-call "Self" "data_type" ["version"]] + [[:fn-call "Self" "fields" ["version"]] "values" (if (named? fields) "self.validity" "None")]]] [:fn @@ -99,7 +108,7 @@ ver-target (if ver "x" target) body (cond (primitive-types ty) - (downcast-clone ver-target ["PrimitiveArray" ty]) + (downcast-clone ver-target ["PrimitiveArray" (arrow-type ty)]) (nil? ty) (downcast-clone ver-target "NullArray") @@ -123,7 +132,7 @@ [:block [:let ["_" "values" "validity"] - [:method-call "array" "into_data"]] + [:method-call "array" "into_parts"]] [:struct-init "Self" (cond->> (mapv (juxt :name from-struct-array) fields) @@ -131,7 +140,8 @@ (defn struct-impl [[nm {:keys [fields]}]] - [:impl nm [(data-type-fn fields) + [:impl nm [(fields-fn fields) + (data-type-fn) (into-struct-array-fn fields) (from-struct-array-fn fields)]]) diff --git a/gen/src/peppi_codegen/frame/mutable.clj b/gen/src/peppi_codegen/frame/mutable.clj index b8fcea9..63f1c8e 100644 --- a/gen/src/peppi_codegen/frame/mutable.clj +++ b/gen/src/peppi_codegen/frame/mutable.clj @@ -11,14 +11,14 @@ (defn array-type [ty] (cond - (primitive-types ty) ["MutablePrimitiveArray" ty] - (nil? ty) "MutableNullArray" + (primitive-types ty) ["PrimitiveBuilder" (arrow-type ty)] + (nil? ty) "NullBuilder" :else ty)) (defn with-capacity-arrow - [arrow-type] + [array-type] [:fn-call - arrow-type + array-type "with_capacity" ["capacity"]]) @@ -31,10 +31,7 @@ (defn with-capacity-null [] - [:fn-call - "MutableNullArray" - "new" - ["DataType::Null" 0]]) + [:fn-call "NullBuilder" "new" []]) (defn with-capacity [{ty :type, ver :version :as m}] @@ -51,29 +48,21 @@ (defn with-capacity-fn [fields] - (let [bitmap-init [:fn-call "MutableBitmap" "with_capacity" ["capacity"]]] - [:fn - {:ret "Self"} - "with_capacity" - [["capacity" "usize"] - ["version" "Version"]] - [:block - [:struct-init - "Self" - (cond->> (mapv (juxt :name with-capacity) fields) - (named? fields) (append ["validity" - (if (every? :version fields) - [:method-call - [:method-call "version" "lt" (:version (first fields))] - "then" - [[:closure - [] - [[:fn-call "MutableBitmap" "with_capacity" ["capacity"]]]]]] - "None")]))]]])) + [:fn + {:ret "Self"} + "with_capacity" + [["capacity" "usize"] + ["version" "Version"]] + [:block + [:struct-init + "Self" + (cond->> (mapv (juxt :name with-capacity) fields) + (named? fields) (append ["validity" + [:fn-call "NullBufferBuilder" "new" ["capacity"]]]))]]]) (defn push-null-primitive [target] - [:method-call target "push_null"]) + [:method-call target "append_null"]) (defn push-null-composite [target] @@ -102,17 +91,11 @@ (cond-> [:block] (named? fields) (conj [:let "len" [:method-call "self" "len"]]) (named? fields) (conj [:method-call - [:method-call - [:field-get "self" "validity"] - "get_or_insert_with" - [[:closure - [] - [[:fn-call "MutableBitmap" "from_len_set" ["len"]]]]]] - "push" + [:field-get "self" "validity"] + "append" ["false"]]) true (into (nested-version-ifs push-null fields)))]) - (defn read-push-primitive [target ty] [:method-call @@ -124,10 +107,7 @@ "map" [[:closure [["x"]] - [[:method-call - target - "push" - [[:struct-init "Some" [[nil "x"]]]]]]]]]) + [[:method-call target "append_value" ["x"]]]]]]) (defn read-push-composite [target] @@ -151,32 +131,16 @@ :else (read-push-null target)))) (defn len-fn - [[{nm :name, idx :index} :as fields]] + [] [:fn {:visibility "pub" :ret "usize"} "len" [["&self"]] [:block - (if (every? :version fields) - [:method-call - [:method-call - [:method-call - [:field-get "self" "validity"] - "as_ref"] - "map" - [[:closure [["v"]] [[:method-call "v" "len"]]]]] - "unwrap_or_else" - [[:closure - [] - [[:method-call - [:method-call - [:method-call - [:field-get "self" (or nm idx)] - "as_ref"] - "unwrap"] - "len"]]]]] - [:method-call [:field-get "self" (or nm idx)] "len"])]]) + [:method-call + [:field-get "self" "validity"] + "len"]]]) (defn read-push-fn [fields] @@ -189,15 +153,39 @@ ["version" "Version"]] (cond->> (into [:block] (nested-version-ifs read-push fields)) (named? fields) (append [:method-call - [:method-call - [:field-get "self" "validity"] - "as_mut"] - "map" - [[:closure - [["v"]] - [[:method-call "v" "push" ["true"]]]]]]) + [:field-get "self" "validity"] + "append" + ["true"]]) true (append [:struct-init "Ok" [[nil [:unit]]]]))]) +(defn immutable + [ty] + (list "immutable" ty)) + +(defn finish + [{idx :index, nm :name, ver :version}] + (let [target [:field-get "self" (or nm idx)]] + (if ver + (wrap-map (as-mut target) "x" [:method-call "x" "finish" []]) + [:method-call target "finish" []]))) + +(defn finish-fn + [ty fields] + [:fn + {:ret (immutable ty)} + "finish" + [["&mut self"]] + [:block + [:struct-init + (immutable ty) + (cond->> (mapv (juxt :name finish) fields) + (named? fields) + (append ["validity" + [:method-call + [:field-get "self" "validity"] + "finish" + []]]))]]]) + (defn struct-field [{nm :name, ty :type, ver :version, desc :description}] [:struct-field @@ -225,7 +213,7 @@ (append [:struct-field {:docstring "Indicates which indexes are valid (`None` means \"all valid\"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games)"} "validity" - ["Option" "MutableBitmap"]]))]) + "NullBufferBuilder"]))]) (defmethod struct-decl false [[nm {:keys [description fields]}]] @@ -237,10 +225,11 @@ (defn struct-impl [[nm {:keys [fields]}]] [:impl nm [(with-capacity-fn fields) - (len-fn fields) + (len-fn) (push-null-fn fields) (read-push-fn fields) - (immutable/transpose-one-fn nm fields)]]) + (finish-fn nm fields) + #_(immutable/transpose-one-fn nm fields)]]) (defn -main [] (doseq [decl (mapcat (juxt struct-decl struct-impl) (read-structs))] diff --git a/src/frame/immutable/mod.rs b/src/frame/immutable/mod.rs index 19a14d2..646363c 100644 --- a/src/frame/immutable/mod.rs +++ b/src/frame/immutable/mod.rs @@ -6,7 +6,7 @@ //! [`crate::io::peppi::read`]. //! //! These arrays can be shared, and cloning them is `O(1)`. See the -//! [arrow2 docs](https://docs.rs/arrow2/latest/arrow2/array/index.html) for more. +//! [arrow_array docs](https://docs.rs/arrow-array/latest/arrow_array/index.html) for more. #![allow(unused_variables)] @@ -15,7 +15,13 @@ mod slippi; use std::fmt; -use arrow2::{array::PrimitiveArray, bitmap::Bitmap, buffer::Buffer, offset::OffsetsBuffer}; +use arrow::{ + array::{ + types::{Float32Type, Int16Type, Int32Type, Int8Type, UInt16Type, UInt32Type, UInt8Type}, + PrimitiveArray, + }, + buffer::{NullBuffer, OffsetBuffer}, +}; use crate::{ frame::{self, mutable, transpose, Rollbacks}, @@ -28,7 +34,7 @@ use crate::{ pub struct Data { pub pre: Pre, pub post: Post, - pub validity: Option, + pub validity: Option, } impl Data { @@ -40,15 +46,15 @@ impl Data { } } -impl From for Data { - fn from(d: mutable::Data) -> Self { - Self { - pre: d.pre.into(), - post: d.post.into(), - validity: d.validity.map(|v| v.into()), - } - } -} +//impl From for Data { +// fn from(d: mutable::Data) -> Self { +// Self { +// pre: d.pre.finish(), +// post: d.post.finish(), +// validity: d.validity.map(|v| v.into()), +// } +// } +//} /// Frame data for a single port. #[derive(Debug)] @@ -69,20 +75,20 @@ impl PortData { } } -impl From for PortData { - fn from(p: mutable::PortData) -> Self { - Self { - port: p.port, - leader: p.leader.into(), - follower: p.follower.map(|f| f.into()), - } - } -} +//impl From for PortData { +// fn from(p: mutable::PortData) -> Self { +// Self { +// port: p.port, +// leader: p.leader.finish(), +// follower: p.follower.map(|f| f.finish()), +// } +// } +//} /// All frame data for a single game, in struct-of-arrays format. pub struct Frame { /// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks - pub id: PrimitiveArray, + pub id: PrimitiveArray, /// Port-specific data pub ports: Vec, /// Start-of-frame data @@ -90,7 +96,7 @@ pub struct Frame { /// End-of-frame data pub end: Option, /// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array - pub item_offset: Option>, + pub item_offset: Option>, /// Item data pub item: Option, } @@ -136,7 +142,7 @@ impl Frame { fn rollbacks_<'a>(&self, ids: impl Iterator) -> Vec { let mut result = vec![false; self.len()]; - let unique_id_count = self.id.values_iter().max().map_or(0, |idx| { + let unique_id_count = arrow::compute::kernels::aggregate::max(&self.id).map_or(0, |idx| { 1 + usize::try_from(idx - frame::FIRST_INDEX).unwrap() }); let mut seen = vec![false; unique_id_count]; @@ -153,20 +159,20 @@ impl Frame { } } -impl From for Frame { - fn from(f: mutable::Frame) -> Self { - Self { - id: f.id.into(), - ports: f.ports.into_iter().map(|p| p.into()).collect(), - start: f.start.map(|x| x.into()), - end: f.end.map(|x| x.into()), - item_offset: f - .item_offset - .map(|x| OffsetsBuffer::try_from(Buffer::from(x.into_inner())).unwrap()), - item: f.item.map(|x| x.into()), - } - } -} +//impl From for Frame { +// fn from(f: mutable::Frame) -> Self { +// Self { +// id: f.id.into(), +// ports: f.ports.into_iter().map(|p| p.into()).collect(), +// start: f.start.map(|x| x.into()), +// end: f.end.map(|x| x.into()), +// item_offset: f.item_offset.map(|x| +// OffsetBuffer::try_from(Buffer::from(x.into_inner())).unwrap() +// ), +// item: f.item.map(|x| x.into()), +// } +// } +//} impl fmt::Debug for Frame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> { @@ -178,9 +184,9 @@ impl fmt::Debug for Frame { #[derive(Debug)] pub struct End { /// *Added: v3.7* Index of the latest frame which is guaranteed not to happen again (rollback) - pub latest_finalized_frame: Option>, + pub latest_finalized_frame: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl End { @@ -191,42 +197,33 @@ impl End { } } -impl From for End { - fn from(x: mutable::End) -> Self { - Self { - latest_finalized_frame: x.latest_finalized_frame.map(|x| x.into()), - validity: x.validity.map(|v| v.into()), - } - } -} - /// An active item (includes projectiles). #[derive(Debug)] pub struct Item { /// Item type - pub r#type: PrimitiveArray, + pub r#type: PrimitiveArray, /// Item’s action state - pub state: PrimitiveArray, + pub state: PrimitiveArray, /// Direction item is facing - pub direction: PrimitiveArray, + pub direction: PrimitiveArray, /// Item’s velocity pub velocity: Velocity, /// Item’s position pub position: Position, /// Amount of damage item has taken - pub damage: PrimitiveArray, + pub damage: PrimitiveArray, /// Frames remaining until item expires - pub timer: PrimitiveArray, + pub timer: PrimitiveArray, /// Unique, serial ID per item spawned - pub id: PrimitiveArray, + pub id: PrimitiveArray, /// *Added: v3.2* Miscellaneous item state pub misc: Option, /// *Added: v3.6* Port that owns the item (-1 when unowned) - pub owner: Option>, + pub owner: Option>, /// *Added: v3.16* Inherited instance ID of the owner (0 when unowned) - pub instance_id: Option>, + pub instance_id: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Item { @@ -247,32 +244,13 @@ impl Item { } } -impl From for Item { - fn from(x: mutable::Item) -> Self { - Self { - r#type: x.r#type.into(), - state: x.state.into(), - direction: x.direction.into(), - velocity: x.velocity.into(), - position: x.position.into(), - damage: x.damage.into(), - timer: x.timer.into(), - id: x.id.into(), - misc: x.misc.map(|x| x.into()), - owner: x.owner.map(|x| x.into()), - instance_id: x.instance_id.map(|x| x.into()), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Miscellaneous item state. #[derive(Debug)] pub struct ItemMisc( - pub PrimitiveArray, - pub PrimitiveArray, - pub PrimitiveArray, - pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, ); impl ItemMisc { @@ -286,19 +264,13 @@ impl ItemMisc { } } -impl From for ItemMisc { - fn from(x: mutable::ItemMisc) -> Self { - Self(x.0.into(), x.1.into(), x.2.into(), x.3.into()) - } -} - /// 2D position. #[derive(Debug)] pub struct Position { - pub x: PrimitiveArray, - pub y: PrimitiveArray, + pub x: PrimitiveArray, + pub y: PrimitiveArray, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Position { @@ -310,69 +282,59 @@ impl Position { } } -impl From for Position { - fn from(x: mutable::Position) -> Self { - Self { - x: x.x.into(), - y: x.y.into(), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Post-frame update data, for making decisions about game states (such as computing stats). /// /// Information is collected at the end of collision detection, which is the last consideration of the game engine. #[derive(Debug)] pub struct Post { /// In-game character (can only change for Zelda/Sheik) - pub character: PrimitiveArray, + pub character: PrimitiveArray, /// Character’s action state - pub state: PrimitiveArray, + pub state: PrimitiveArray, /// Character’s position pub position: Position, /// Direction the character is facing - pub direction: PrimitiveArray, + pub direction: PrimitiveArray, /// Damage taken (percent) - pub percent: PrimitiveArray, + pub percent: PrimitiveArray, /// Size/health of shield - pub shield: PrimitiveArray, + pub shield: PrimitiveArray, /// Last attack ID that this character landed - pub last_attack_landed: PrimitiveArray, + pub last_attack_landed: PrimitiveArray, /// Combo count (as defined by the game) - pub combo_count: PrimitiveArray, + pub combo_count: PrimitiveArray, /// Port that last hit this player. Bugged in Melee: will be set to `6` in certain situations - pub last_hit_by: PrimitiveArray, + pub last_hit_by: PrimitiveArray, /// Number of stocks remaining - pub stocks: PrimitiveArray, + pub stocks: PrimitiveArray, /// *Added: v0.2* Number of frames action state has been active. Can have a fractional component - pub state_age: Option>, + pub state_age: Option>, /// *Added: v2.0* State flags pub state_flags: Option, /// *Added: v2.0* Used for different things. While in hitstun, contains hitstun frames remaining - pub misc_as: Option>, + pub misc_as: Option>, /// *Added: v2.0* Is the character airborne? - pub airborne: Option>, + pub airborne: Option>, /// *Added: v2.0* Ground ID the character last touched - pub ground: Option>, + pub ground: Option>, /// *Added: v2.0* Number of jumps remaining - pub jumps: Option>, + pub jumps: Option>, /// *Added: v2.0* L-cancel status (0 = none, 1 = successful, 2 = unsuccessful) - pub l_cancel: Option>, + pub l_cancel: Option>, /// *Added: v2.1* Hurtbox state (0 = vulnerable, 1 = invulnerable, 2 = intangible) - pub hurtbox_state: Option>, + pub hurtbox_state: Option>, /// *Added: v3.5* Self-induced and knockback velocities pub velocities: Option, /// *Added: v3.8* Hitlag frames remaining - pub hitlag: Option>, + pub hitlag: Option>, /// *Added: v3.11* Animation the character is in - pub animation_index: Option>, + pub animation_index: Option>, /// *Added: v3.16* Instance ID of the player/item that last hit this player - pub last_hit_by_instance: Option>, + pub last_hit_by_instance: Option>, /// *Added: v3.16* Unique, serial ID for each new action state across all characters. Resets to 0 on death - pub instance_id: Option>, + pub instance_id: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Post { @@ -411,70 +373,39 @@ impl Post { } } -impl From for Post { - fn from(x: mutable::Post) -> Self { - Self { - character: x.character.into(), - state: x.state.into(), - position: x.position.into(), - direction: x.direction.into(), - percent: x.percent.into(), - shield: x.shield.into(), - last_attack_landed: x.last_attack_landed.into(), - combo_count: x.combo_count.into(), - last_hit_by: x.last_hit_by.into(), - stocks: x.stocks.into(), - state_age: x.state_age.map(|x| x.into()), - state_flags: x.state_flags.map(|x| x.into()), - misc_as: x.misc_as.map(|x| x.into()), - airborne: x.airborne.map(|x| x.into()), - ground: x.ground.map(|x| x.into()), - jumps: x.jumps.map(|x| x.into()), - l_cancel: x.l_cancel.map(|x| x.into()), - hurtbox_state: x.hurtbox_state.map(|x| x.into()), - velocities: x.velocities.map(|x| x.into()), - hitlag: x.hitlag.map(|x| x.into()), - animation_index: x.animation_index.map(|x| x.into()), - last_hit_by_instance: x.last_hit_by_instance.map(|x| x.into()), - instance_id: x.instance_id.map(|x| x.into()), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Pre-frame update data, required to reconstruct a replay. /// /// Information is collected right before controller inputs are used to figure out the character’s next action. #[derive(Debug)] pub struct Pre { /// Random seed - pub random_seed: PrimitiveArray, + pub random_seed: PrimitiveArray, /// Character’s action state - pub state: PrimitiveArray, + pub state: PrimitiveArray, /// Character’s position pub position: Position, /// Direction the character is facing - pub direction: PrimitiveArray, + pub direction: PrimitiveArray, /// Processed analog joystick position pub joystick: Position, /// Processed analog c-stick position pub cstick: Position, /// Processed analog trigger position - pub triggers: PrimitiveArray, + pub triggers: PrimitiveArray, /// Processed button-state bitmask - pub buttons: PrimitiveArray, + pub buttons: PrimitiveArray, /// Physical button-state bitmask - pub buttons_physical: PrimitiveArray, + pub buttons_physical: PrimitiveArray, /// Physical analog trigger positions (useful for IPM) pub triggers_physical: TriggersPhysical, /// *Added: v1.2* Raw joystick x-position - pub raw_analog_x: Option>, + pub raw_analog_x: Option>, /// *Added: v1.4* Damage taken (percent) - pub percent: Option>, + pub percent: Option>, /// *Added: v3.15* Raw joystick y-position - pub raw_analog_y: Option>, + pub raw_analog_y: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Pre { @@ -497,36 +428,15 @@ impl Pre { } } -impl From for Pre { - fn from(x: mutable::Pre) -> Self { - Self { - random_seed: x.random_seed.into(), - state: x.state.into(), - position: x.position.into(), - direction: x.direction.into(), - joystick: x.joystick.into(), - cstick: x.cstick.into(), - triggers: x.triggers.into(), - buttons: x.buttons.into(), - buttons_physical: x.buttons_physical.into(), - triggers_physical: x.triggers_physical.into(), - raw_analog_x: x.raw_analog_x.map(|x| x.into()), - percent: x.percent.map(|x| x.into()), - raw_analog_y: x.raw_analog_y.map(|x| x.into()), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Initialization data such as game mode, settings, characters & stage. #[derive(Debug)] pub struct Start { /// Random seed - pub random_seed: PrimitiveArray, + pub random_seed: PrimitiveArray, /// *Added: v3.10* Scene frame counter. Starts at 0, and increments every frame (even when paused) - pub scene_frame_counter: Option>, + pub scene_frame_counter: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Start { @@ -538,24 +448,14 @@ impl Start { } } -impl From for Start { - fn from(x: mutable::Start) -> Self { - Self { - random_seed: x.random_seed.into(), - scene_frame_counter: x.scene_frame_counter.map(|x| x.into()), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Miscellaneous state flags. #[derive(Debug)] pub struct StateFlags( - pub PrimitiveArray, - pub PrimitiveArray, - pub PrimitiveArray, - pub PrimitiveArray, - pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, + pub PrimitiveArray, ); impl StateFlags { @@ -570,19 +470,13 @@ impl StateFlags { } } -impl From for StateFlags { - fn from(x: mutable::StateFlags) -> Self { - Self(x.0.into(), x.1.into(), x.2.into(), x.3.into(), x.4.into()) - } -} - /// Trigger state. #[derive(Debug)] pub struct TriggersPhysical { - pub l: PrimitiveArray, - pub r: PrimitiveArray, + pub l: PrimitiveArray, + pub r: PrimitiveArray, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl TriggersPhysical { @@ -594,31 +488,21 @@ impl TriggersPhysical { } } -impl From for TriggersPhysical { - fn from(x: mutable::TriggersPhysical) -> Self { - Self { - l: x.l.into(), - r: x.r.into(), - validity: x.validity.map(|v| v.into()), - } - } -} - /// Self-induced and knockback velocities. #[derive(Debug)] pub struct Velocities { /// Self-induced x-velocity (airborne) - pub self_x_air: PrimitiveArray, + pub self_x_air: PrimitiveArray, /// Self-induced y-velocity - pub self_y: PrimitiveArray, + pub self_y: PrimitiveArray, /// Knockback-induced x-velocity - pub knockback_x: PrimitiveArray, + pub knockback_x: PrimitiveArray, /// Knockback-induced y-velocity - pub knockback_y: PrimitiveArray, + pub knockback_y: PrimitiveArray, /// Self-induced x-velocity (grounded) - pub self_x_ground: PrimitiveArray, + pub self_x_ground: PrimitiveArray, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Velocities { @@ -633,26 +517,13 @@ impl Velocities { } } -impl From for Velocities { - fn from(x: mutable::Velocities) -> Self { - Self { - self_x_air: x.self_x_air.into(), - self_y: x.self_y.into(), - knockback_x: x.knockback_x.into(), - knockback_y: x.knockback_y.into(), - self_x_ground: x.self_x_ground.into(), - validity: x.validity.map(|v| v.into()), - } - } -} - /// 2D velocity. #[derive(Debug)] pub struct Velocity { - pub x: PrimitiveArray, - pub y: PrimitiveArray, + pub x: PrimitiveArray, + pub y: PrimitiveArray, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: Option, } impl Velocity { @@ -663,13 +534,3 @@ impl Velocity { } } } - -impl From for Velocity { - fn from(x: mutable::Velocity) -> Self { - Self { - x: x.x.into(), - y: x.y.into(), - validity: x.validity.map(|v| v.into()), - } - } -} diff --git a/src/frame/immutable/peppi.rs b/src/frame/immutable/peppi.rs index e909da9..162b679 100644 --- a/src/frame/immutable/peppi.rs +++ b/src/frame/immutable/peppi.rs @@ -2,9 +2,14 @@ #![allow(unused_variables)] -use arrow2::{ - array::{ListArray, PrimitiveArray, StructArray}, - datatypes::{DataType, Field}, +use std::sync::Arc; + +use arrow::{ + array::{ + types::{Float32Type, Int16Type, Int32Type, Int8Type, UInt16Type, UInt32Type, UInt8Type}, + Array, ArrayRef, ListArray, PrimitiveArray, StructArray, + }, + datatypes::{DataType, Field, Fields}, }; use crate::{ @@ -17,23 +22,27 @@ use crate::{ }; impl Data { - fn data_type(version: Version) -> DataType { - DataType::Struct(vec![ + fn fields(version: Version) -> Fields { + Fields::from(vec![ Field::new("pre", Pre::data_type(version).clone(), false), Field::new("post", Post::data_type(version).clone(), false), ]) } + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) + } + fn into_struct_array(self, version: Version) -> StructArray { let values = vec![ - self.pre.into_struct_array(version).boxed(), - self.post.into_struct_array(version).boxed(), + Arc::new(self.pre.into_struct_array(version)) as ArrayRef, + Arc::new(self.post.into_struct_array(version)) as ArrayRef, ]; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { pre: Pre::from_struct_array( values[0] @@ -57,7 +66,7 @@ impl Data { } impl PortData { - fn data_type(version: Version, port: PortOccupancy) -> DataType { + fn fields(version: Version, port: PortOccupancy) -> Fields { let mut fields = vec![Field::new( "leader", Data::data_type(version).clone(), @@ -70,21 +79,25 @@ impl PortData { false, )); } - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version, port: PortOccupancy) -> DataType { + DataType::Struct(Self::fields(version, port)) } fn into_struct_array(self, version: Version, port: PortOccupancy) -> StructArray { - let mut values = vec![self.leader.into_struct_array(version).boxed()]; + let mut values = vec![Arc::new(self.leader.into_struct_array(version)) as Arc]; if let Some(follower) = self.follower { - values.push(follower.into_struct_array(version).boxed()); + values.push(Arc::new(follower.into_struct_array(version)) as Arc); } - StructArray::new(Self::data_type(version, port), values, None) + StructArray::new(Self::fields(version, port), values, None) } fn from_struct_array(array: StructArray, version: Version, port: Port) -> Self { - let (fields, values, _) = array.into_data(); - assert_eq!("leader", fields[0].name); - fields.get(1).map(|f| assert_eq!("follower", f.name)); + let (fields, values, _) = array.into_parts(); + assert_eq!("leader", fields[0].name()); + fields.get(1).map(|f| assert_eq!("follower", f.name())); Self { port: port, leader: Data::from_struct_array( @@ -106,8 +119,8 @@ impl PortData { } impl Frame { - fn port_data_type(version: Version, ports: &[PortOccupancy]) -> DataType { - DataType::Struct( + fn port_fields(version: Version, ports: &[PortOccupancy]) -> Fields { + Fields::from( ports .iter() .map(|p| { @@ -121,18 +134,22 @@ impl Frame { ) } + fn port_data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + DataType::Struct(Self::port_fields(version, ports)) + } + + fn item_field(version: Version) -> Field { + Field::new("item", Item::data_type(version), false) + } + fn item_data_type(version: Version) -> DataType { - DataType::List(Box::new(Field::new( - "item", - Item::data_type(version), - false, - ))) + DataType::List(Arc::new(Self::item_field(version))) } - fn data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + fn fields(version: Version, ports: &[PortOccupancy]) -> Fields { let mut fields = vec![ Field::new("id", DataType::Int32, false), - Field::new("ports", Self::port_data_type(version, ports).clone(), false), + Field::new("ports", Self::port_data_type(version, ports), false), ]; if version.gte(2, 2) { fields.push(Field::new( @@ -149,48 +166,56 @@ impl Frame { )); } } - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version, ports: &[PortOccupancy]) -> DataType { + DataType::Struct(Self::fields(version, ports)) } pub fn into_struct_array(self, version: Version, ports: &[PortOccupancy]) -> StructArray { let values: Vec<_> = std::iter::zip(ports, self.ports) - .map(|(occupancy, data)| data.into_struct_array(version, *occupancy).boxed()) + .map(|(occupancy, data)| { + Arc::new(data.into_struct_array(version, *occupancy)) as Arc + }) .collect(); let mut arrays = vec![ - self.id.boxed(), - StructArray::new(Self::port_data_type(version, ports), values, None).boxed(), + Arc::new(self.id) as Arc, + Arc::new(StructArray::new( + Self::port_fields(version, ports), + values, + None, + )) as Arc, ]; if version.gte(2, 2) { - arrays.push(self.start.unwrap().into_struct_array(version).boxed()); + arrays.push(Arc::new(self.start.unwrap().into_struct_array(version)) as Arc); if version.gte(3, 0) { - arrays.push(self.end.unwrap().into_struct_array(version).boxed()); - let item_values = self.item.unwrap().into_struct_array(version).boxed(); - arrays.push( - ListArray::new( - Self::item_data_type(version), - self.item_offset.unwrap(), - item_values, - None, - ) - .boxed(), - ); + arrays + .push(Arc::new(self.end.unwrap().into_struct_array(version)) as Arc); + let item_values = Arc::new(self.item.unwrap().into_struct_array(version)); + arrays.push(Arc::new(ListArray::new( + Arc::new(Self::item_field(version)), + self.item_offset.unwrap(), + item_values, + None, + )) as Arc); } } - StructArray::new(Self::data_type(version, ports), arrays, None) + StructArray::new(Self::fields(version, ports), arrays, None) } fn port_data_from_struct_array(array: StructArray, version: Version) -> Vec { - let (fields, values, _) = array.into_data(); + let (fields, values, _) = array.into_parts(); let mut ports = vec![]; for i in 0..NUM_PORTS { if let Some(a) = values.get(i as usize) { ports.push(PortData::from_struct_array( a.as_any().downcast_ref::().unwrap().clone(), version, - Port::parse(&fields[i as usize].name).unwrap(), + Port::parse(&fields[i as usize].name()).unwrap(), )); } } @@ -198,19 +223,19 @@ impl Frame { } pub fn from_struct_array(array: StructArray, version: Version) -> Self { - let (fields, values, _) = array.into_data(); - assert_eq!("id", fields[0].name); - assert_eq!("ports", fields[1].name); + let (fields, values, _) = array.into_parts(); + assert_eq!("id", fields[0].name()); + assert_eq!("ports", fields[1].name()); if version.gte(2, 2) { - assert_eq!("start", fields[2].name); + assert_eq!("start", fields[2].name()); if version.gte(3, 0) { - assert_eq!("end", fields[3].name); - assert_eq!("item", fields[4].name); + assert_eq!("end", fields[3].name()); + assert_eq!("item", fields[4].name()); } } let (item, item_offset) = values.get(4).map_or((None, None), |v| { - let arrays = v.as_any().downcast_ref::>().unwrap().clone(); + let arrays = v.as_any().downcast_ref::().unwrap().clone(); let item_offset = arrays.offsets().clone(); let item = Item::from_struct_array( arrays @@ -227,7 +252,7 @@ impl Frame { Self { id: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), ports: Self::port_data_from_struct_array( @@ -259,30 +284,34 @@ impl Frame { use crate::frame::immutable::End; impl End { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { if version.gte(3, 7) { fields.push(Field::new("latest_finalized_frame", DataType::Int32, false)) } }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; if version.gte(3, 7) { - values.push(self.latest_finalized_frame.unwrap().boxed()) + values.push(Arc::new(self.latest_finalized_frame.unwrap()) as Arc) }; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { latest_finalized_frame: values.get(0).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -294,7 +323,7 @@ impl End { use crate::frame::immutable::Item; impl Item { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("type", DataType::UInt16, false)); @@ -315,47 +344,51 @@ impl Item { } } }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.r#type.boxed()); - values.push(self.state.boxed()); - values.push(self.direction.boxed()); - values.push(self.velocity.into_struct_array(version).boxed()); - values.push(self.position.into_struct_array(version).boxed()); - values.push(self.damage.boxed()); - values.push(self.timer.boxed()); - values.push(self.id.boxed()); + values.push(Arc::new(self.r#type) as Arc); + values.push(Arc::new(self.state) as Arc); + values.push(Arc::new(self.direction) as Arc); + values.push(Arc::new(self.velocity.into_struct_array(version)) as Arc); + values.push(Arc::new(self.position.into_struct_array(version)) as Arc); + values.push(Arc::new(self.damage) as Arc); + values.push(Arc::new(self.timer) as Arc); + values.push(Arc::new(self.id) as Arc); if version.gte(3, 2) { - values.push(self.misc.unwrap().into_struct_array(version).boxed()); + values.push(Arc::new(self.misc.unwrap().into_struct_array(version)) as Arc); if version.gte(3, 6) { - values.push(self.owner.unwrap().boxed()); + values.push(Arc::new(self.owner.unwrap()) as Arc); if version.gte(3, 16) { - values.push(self.instance_id.unwrap().boxed()) + values.push(Arc::new(self.instance_id.unwrap()) as Arc) } } }; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { r#type: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), state: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), direction: values[2] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), velocity: Velocity::from_struct_array( @@ -376,17 +409,17 @@ impl Item { ), damage: values[5] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), timer: values[6] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), id: values[7] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), misc: values.get(8).map(|x| { @@ -397,13 +430,13 @@ impl Item { }), owner: values.get(9).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), instance_id: values.get(10).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -415,7 +448,7 @@ impl Item { use crate::frame::immutable::ItemMisc; impl ItemMisc { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("0", DataType::UInt8, false)); @@ -423,39 +456,43 @@ impl ItemMisc { fields.push(Field::new("2", DataType::UInt8, false)); fields.push(Field::new("3", DataType::UInt8, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.0.boxed()); - values.push(self.1.boxed()); - values.push(self.2.boxed()); - values.push(self.3.boxed()); - StructArray::new(Self::data_type(version), values, None) + values.push(Arc::new(self.0) as Arc); + values.push(Arc::new(self.1) as Arc); + values.push(Arc::new(self.2) as Arc); + values.push(Arc::new(self.3) as Arc); + StructArray::new(Self::fields(version), values, None) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self( values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[2] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[3] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), ) @@ -465,33 +502,37 @@ impl ItemMisc { use crate::frame::immutable::Position; impl Position { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("x", DataType::Float32, false)); fields.push(Field::new("y", DataType::Float32, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.x.boxed()); - values.push(self.y.boxed()); - StructArray::new(Self::data_type(version), values, self.validity) + values.push(Arc::new(self.x) as Arc); + values.push(Arc::new(self.y) as Arc); + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { x: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), y: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), validity: validity, @@ -502,7 +543,7 @@ impl Position { use crate::frame::immutable::Post; impl Post { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("character", DataType::UInt8, false)); @@ -563,41 +604,54 @@ impl Post { } } }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.character.boxed()); - values.push(self.state.boxed()); - values.push(self.position.into_struct_array(version).boxed()); - values.push(self.direction.boxed()); - values.push(self.percent.boxed()); - values.push(self.shield.boxed()); - values.push(self.last_attack_landed.boxed()); - values.push(self.combo_count.boxed()); - values.push(self.last_hit_by.boxed()); - values.push(self.stocks.boxed()); + values.push(Arc::new(self.character) as Arc); + values.push(Arc::new(self.state) as Arc); + values.push(Arc::new(self.position.into_struct_array(version)) as Arc); + values.push(Arc::new(self.direction) as Arc); + values.push(Arc::new(self.percent) as Arc); + values.push(Arc::new(self.shield) as Arc); + values.push(Arc::new(self.last_attack_landed) as Arc); + values.push(Arc::new(self.combo_count) as Arc); + values.push(Arc::new(self.last_hit_by) as Arc); + values.push(Arc::new(self.stocks) as Arc); if version.gte(0, 2) { - values.push(self.state_age.unwrap().boxed()); + values.push(Arc::new(self.state_age.unwrap()) as Arc); if version.gte(2, 0) { - values.push(self.state_flags.unwrap().into_struct_array(version).boxed()); - values.push(self.misc_as.unwrap().boxed()); - values.push(self.airborne.unwrap().boxed()); - values.push(self.ground.unwrap().boxed()); - values.push(self.jumps.unwrap().boxed()); - values.push(self.l_cancel.unwrap().boxed()); + values.push( + Arc::new(self.state_flags.unwrap().into_struct_array(version)) + as Arc, + ); + values.push(Arc::new(self.misc_as.unwrap()) as Arc); + values.push(Arc::new(self.airborne.unwrap()) as Arc); + values.push(Arc::new(self.ground.unwrap()) as Arc); + values.push(Arc::new(self.jumps.unwrap()) as Arc); + values.push(Arc::new(self.l_cancel.unwrap()) as Arc); if version.gte(2, 1) { - values.push(self.hurtbox_state.unwrap().boxed()); + values.push(Arc::new(self.hurtbox_state.unwrap()) as Arc); if version.gte(3, 5) { - values.push(self.velocities.unwrap().into_struct_array(version).boxed()); + values.push( + Arc::new(self.velocities.unwrap().into_struct_array(version)) + as Arc, + ); if version.gte(3, 8) { - values.push(self.hitlag.unwrap().boxed()); + values.push(Arc::new(self.hitlag.unwrap()) as Arc); if version.gte(3, 11) { - values.push(self.animation_index.unwrap().boxed()); + values + .push(Arc::new(self.animation_index.unwrap()) as Arc); if version.gte(3, 16) { - values.push(self.last_hit_by_instance.unwrap().boxed()); - values.push(self.instance_id.unwrap().boxed()) + values.push(Arc::new(self.last_hit_by_instance.unwrap()) + as Arc); + values + .push(Arc::new(self.instance_id.unwrap()) as Arc) } } } @@ -605,20 +659,20 @@ impl Post { } } }; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { character: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), state: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), position: Position::from_struct_array( @@ -631,42 +685,42 @@ impl Post { ), direction: values[3] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), percent: values[4] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), shield: values[5] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), last_attack_landed: values[6] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), combo_count: values[7] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), last_hit_by: values[8] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), stocks: values[9] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), state_age: values.get(10).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -678,37 +732,37 @@ impl Post { }), misc_as: values.get(12).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), airborne: values.get(13).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), ground: values.get(14).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), jumps: values.get(15).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), l_cancel: values.get(16).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), hurtbox_state: values.get(17).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -720,25 +774,25 @@ impl Post { }), hitlag: values.get(19).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), animation_index: values.get(20).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), last_hit_by_instance: values.get(21).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), instance_id: values.get(22).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -750,7 +804,7 @@ impl Post { use crate::frame::immutable::Pre; impl Pre { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("random_seed", DataType::UInt32, false)); @@ -777,44 +831,48 @@ impl Pre { } } }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.random_seed.boxed()); - values.push(self.state.boxed()); - values.push(self.position.into_struct_array(version).boxed()); - values.push(self.direction.boxed()); - values.push(self.joystick.into_struct_array(version).boxed()); - values.push(self.cstick.into_struct_array(version).boxed()); - values.push(self.triggers.boxed()); - values.push(self.buttons.boxed()); - values.push(self.buttons_physical.boxed()); - values.push(self.triggers_physical.into_struct_array(version).boxed()); + values.push(Arc::new(self.random_seed) as Arc); + values.push(Arc::new(self.state) as Arc); + values.push(Arc::new(self.position.into_struct_array(version)) as Arc); + values.push(Arc::new(self.direction) as Arc); + values.push(Arc::new(self.joystick.into_struct_array(version)) as Arc); + values.push(Arc::new(self.cstick.into_struct_array(version)) as Arc); + values.push(Arc::new(self.triggers) as Arc); + values.push(Arc::new(self.buttons) as Arc); + values.push(Arc::new(self.buttons_physical) as Arc); + values.push(Arc::new(self.triggers_physical.into_struct_array(version)) as Arc); if version.gte(1, 2) { - values.push(self.raw_analog_x.unwrap().boxed()); + values.push(Arc::new(self.raw_analog_x.unwrap()) as Arc); if version.gte(1, 4) { - values.push(self.percent.unwrap().boxed()); + values.push(Arc::new(self.percent.unwrap()) as Arc); if version.gte(3, 15) { - values.push(self.raw_analog_y.unwrap().boxed()) + values.push(Arc::new(self.raw_analog_y.unwrap()) as Arc) } } }; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { random_seed: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), state: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), position: Position::from_struct_array( @@ -827,7 +885,7 @@ impl Pre { ), direction: values[3] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), joystick: Position::from_struct_array( @@ -848,17 +906,17 @@ impl Pre { ), triggers: values[6] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), buttons: values[7] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), buttons_physical: values[8] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), triggers_physical: TriggersPhysical::from_struct_array( @@ -871,19 +929,19 @@ impl Pre { ), raw_analog_x: values.get(10).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), percent: values.get(11).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), raw_analog_y: values.get(12).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -895,7 +953,7 @@ impl Pre { use crate::frame::immutable::Start; impl Start { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("random_seed", DataType::UInt32, false)); @@ -903,29 +961,33 @@ impl Start { fields.push(Field::new("scene_frame_counter", DataType::UInt32, false)) } }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.random_seed.boxed()); + values.push(Arc::new(self.random_seed) as Arc); if version.gte(3, 10) { - values.push(self.scene_frame_counter.unwrap().boxed()) + values.push(Arc::new(self.scene_frame_counter.unwrap()) as Arc) }; - StructArray::new(Self::data_type(version), values, self.validity) + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { random_seed: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), scene_frame_counter: values.get(1).map(|x| { x.as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone() }), @@ -937,7 +999,7 @@ impl Start { use crate::frame::immutable::StateFlags; impl StateFlags { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("0", DataType::UInt8, false)); @@ -946,45 +1008,49 @@ impl StateFlags { fields.push(Field::new("3", DataType::UInt8, false)); fields.push(Field::new("4", DataType::UInt8, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.0.boxed()); - values.push(self.1.boxed()); - values.push(self.2.boxed()); - values.push(self.3.boxed()); - values.push(self.4.boxed()); - StructArray::new(Self::data_type(version), values, None) + values.push(Arc::new(self.0) as Arc); + values.push(Arc::new(self.1) as Arc); + values.push(Arc::new(self.2) as Arc); + values.push(Arc::new(self.3) as Arc); + values.push(Arc::new(self.4) as Arc); + StructArray::new(Self::fields(version), values, None) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self( values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[2] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[3] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), values[4] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), ) @@ -994,33 +1060,37 @@ impl StateFlags { use crate::frame::immutable::TriggersPhysical; impl TriggersPhysical { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("l", DataType::Float32, false)); fields.push(Field::new("r", DataType::Float32, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.l.boxed()); - values.push(self.r.boxed()); - StructArray::new(Self::data_type(version), values, self.validity) + values.push(Arc::new(self.l) as Arc); + values.push(Arc::new(self.r) as Arc); + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { l: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), r: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), validity: validity, @@ -1031,7 +1101,7 @@ impl TriggersPhysical { use crate::frame::immutable::Velocities; impl Velocities { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("self_x_air", DataType::Float32, false)); @@ -1040,45 +1110,49 @@ impl Velocities { fields.push(Field::new("knockback_y", DataType::Float32, false)); fields.push(Field::new("self_x_ground", DataType::Float32, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.self_x_air.boxed()); - values.push(self.self_y.boxed()); - values.push(self.knockback_x.boxed()); - values.push(self.knockback_y.boxed()); - values.push(self.self_x_ground.boxed()); - StructArray::new(Self::data_type(version), values, self.validity) + values.push(Arc::new(self.self_x_air) as Arc); + values.push(Arc::new(self.self_y) as Arc); + values.push(Arc::new(self.knockback_x) as Arc); + values.push(Arc::new(self.knockback_y) as Arc); + values.push(Arc::new(self.self_x_ground) as Arc); + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { self_x_air: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), self_y: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), knockback_x: values[2] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), knockback_y: values[3] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), self_x_ground: values[4] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), validity: validity, @@ -1089,33 +1163,37 @@ impl Velocities { use crate::frame::immutable::Velocity; impl Velocity { - fn data_type(version: Version) -> DataType { + fn fields(version: Version) -> Fields { let mut fields = vec![]; { fields.push(Field::new("x", DataType::Float32, false)); fields.push(Field::new("y", DataType::Float32, false)) }; - DataType::Struct(fields) + Fields::from(fields) + } + + fn data_type(version: Version) -> DataType { + DataType::Struct(Self::fields(version)) } fn into_struct_array(self, version: Version) -> StructArray { let mut values = vec![]; - values.push(self.x.boxed()); - values.push(self.y.boxed()); - StructArray::new(Self::data_type(version), values, self.validity) + values.push(Arc::new(self.x) as Arc); + values.push(Arc::new(self.y) as Arc); + StructArray::new(Self::fields(version), values, self.validity) } fn from_struct_array(array: StructArray, version: Version) -> Self { - let (_, values, validity) = array.into_data(); + let (_, values, validity) = array.into_parts(); Self { x: values[0] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), y: values[1] .as_any() - .downcast_ref::>() + .downcast_ref::>() .unwrap() .clone(), validity: validity, diff --git a/src/frame/immutable/slippi.rs b/src/frame/immutable/slippi.rs index 9e0f2a4..6c93b91 100644 --- a/src/frame/immutable/slippi.rs +++ b/src/frame/immutable/slippi.rs @@ -28,7 +28,7 @@ impl Data { frame_id: i32, port: PortOccupancy, ) -> Result<()> { - if self.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if self.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { w.write_u8(Event::FramePre as u8)?; w.write_i32::(frame_id)?; w.write_u8(port.port as u8)?; @@ -49,7 +49,7 @@ impl Data { frame_id: i32, port: PortOccupancy, ) -> Result<()> { - if self.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if self.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { w.write_u8(Event::FramePost as u8)?; w.write_i32::(frame_id)?; w.write_u8(port.port as u8)?; @@ -82,7 +82,7 @@ impl PortData { }, )?; self.follower.as_ref().map_or(Ok(()), |f| { - if f.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if f.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { f.write_pre( w, version, @@ -117,7 +117,7 @@ impl PortData { }, )?; self.follower.as_ref().map_or(Ok(()), |f| { - if f.validity.as_ref().map_or(true, |v| v.get_bit(idx)) { + if f.validity.as_ref().map_or(true, |v| v.is_valid(idx)) { f.write_post( w, version, diff --git a/src/frame/mutable.rs b/src/frame/mutable.rs index c94b9fc..1a2c5d4 100644 --- a/src/frame/mutable.rs +++ b/src/frame/mutable.rs @@ -7,17 +7,17 @@ #![allow(unused_variables)] #![allow(dead_code)] -use arrow2::{ - array::{MutableArray, MutablePrimitiveArray}, - bitmap::MutableBitmap, - offset::Offsets, +use arrow::array::{ + types::{Float32Type, Int16Type, Int32Type, Int8Type, UInt16Type, UInt32Type, UInt8Type}, + ArrayBuilder, PrimitiveBuilder, }; +use arrow_buffer::builder::{NullBufferBuilder, OffsetBufferBuilder}; use byteorder::ReadBytesExt; use std::io::Result; use crate::{ - frame::{transpose, PortOccupancy}, + frame::{immutable, transpose, PortOccupancy}, game::Port, io::slippi::Version, }; @@ -28,7 +28,7 @@ type BE = byteorder::BigEndian; pub struct Data { pub pre: Pre, pub post: Post, - pub validity: Option, + pub validity: NullBufferBuilder, } impl Data { @@ -36,7 +36,7 @@ impl Data { Self { pre: Pre::with_capacity(capacity, version), post: Post::with_capacity(capacity, version), - validity: None, + validity: NullBufferBuilder::new(capacity), } } @@ -46,9 +46,7 @@ impl Data { pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); + self.validity.append(false); self.pre.push_null(version); self.post.push_null(version); } @@ -59,6 +57,14 @@ impl Data { post: self.post.transpose_one(i, version), } } + + pub fn finish(&mut self) -> immutable::Data { + immutable::Data { + pre: self.pre.finish(), + post: self.post.finish(), + validity: self.validity.finish(), + } + } } /// Frame data for a single port. @@ -92,28 +98,36 @@ impl PortData { follower: self.follower.as_ref().map(|f| f.transpose_one(i, version)), } } + + pub fn finish(&mut self) -> immutable::PortData { + immutable::PortData { + port: self.port, + leader: self.leader.finish(), + follower: self.follower.as_mut().map(|f| f.finish()), + } + } } /// All frame data for a single game, in struct-of-arrays format. pub struct Frame { /// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks - pub id: MutablePrimitiveArray, + pub id: PrimitiveBuilder, /// Port-specific data pub ports: Vec, /// Start-of-frame data pub start: Option, /// End-of-frame data pub end: Option, - /// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array - pub item_offset: Option>, /// Item data pub item: Option, + /// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array + pub item_offset: Option>, } impl Frame { pub fn with_capacity(capacity: usize, version: Version, ports: &[PortOccupancy]) -> Self { Self { - id: MutablePrimitiveArray::::with_capacity(capacity), + id: PrimitiveBuilder::with_capacity(capacity), ports: ports .iter() .map(|p| PortData::with_capacity(capacity, version, *p)) @@ -126,7 +140,7 @@ impl Frame { .then(|| End::with_capacity(capacity, version)), item_offset: version .gte(3, 0) - .then(|| Offsets::::with_capacity(capacity)), + .then(|| OffsetBufferBuilder::::new(capacity)), item: version.gte(3, 0).then(|| Item::with_capacity(0, version)), } } @@ -137,7 +151,7 @@ impl Frame { pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Frame { transpose::Frame { - id: self.id.values()[i], + id: self.id.values_slice()[i], ports: self .ports .iter() @@ -157,15 +171,38 @@ impl Frame { }), } } + + pub fn finish(mut self) -> immutable::Frame { + immutable::Frame { + id: self.id.finish(), + ports: vec![], //self.ports.iter_mut().map(|p| p.finish()).collect(), + start: match &mut self.start { + Some(x) => Some(x.finish()), + _ => None, + }, + end: match &mut self.end { + Some(x) => Some(x.finish()), + _ => None, + }, + item: match &mut self.item { + Some(x) => Some(x.finish()), + _ => None, + }, + item_offset: match &mut self.item_offset { + Some(x) => Some(x.finish()), + _ => None, + }, + } + } } /// Information about the end of the game. pub struct End { /// *Added: v3.7* Index of the latest frame which is guaranteed not to happen again (rollback) - pub latest_finalized_frame: Option>, + pub latest_finalized_frame: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl End { @@ -173,42 +210,40 @@ impl End { Self { latest_finalized_frame: version .gte(3, 7) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), - validity: version - .lt(3, 7) - .then(|| MutableBitmap::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.validity - .as_ref() - .map(|v| v.len()) - .unwrap_or_else(|| self.latest_finalized_frame.as_ref().unwrap().len()) + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); + self.validity.append(false); if version.gte(3, 7) { - self.latest_finalized_frame.as_mut().unwrap().push_null() + self.latest_finalized_frame.as_mut().unwrap().append_null() } } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { if version.gte(3, 7) { - r.read_i32::() - .map(|x| self.latest_finalized_frame.as_mut().unwrap().push(Some(x)))? + r.read_i32::().map(|x| { + self.latest_finalized_frame + .as_mut() + .unwrap() + .append_value(x) + })? }; - self.validity.as_mut().map(|v| v.push(true)); + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::End { - transpose::End { - latest_finalized_frame: self.latest_finalized_frame.as_ref().map(|x| x.values()[i]), + fn finish(&mut self) -> immutable::End { + immutable::End { + latest_finalized_frame: self.latest_finalized_frame.as_mut().map(|x| x.finish()), + validity: self.validity.finish(), } } } @@ -217,120 +252,119 @@ impl End { pub struct Item { /// Item type - pub r#type: MutablePrimitiveArray, + pub r#type: PrimitiveBuilder, /// Item’s action state - pub state: MutablePrimitiveArray, + pub state: PrimitiveBuilder, /// Direction item is facing - pub direction: MutablePrimitiveArray, + pub direction: PrimitiveBuilder, /// Item’s velocity pub velocity: Velocity, /// Item’s position pub position: Position, /// Amount of damage item has taken - pub damage: MutablePrimitiveArray, + pub damage: PrimitiveBuilder, /// Frames remaining until item expires - pub timer: MutablePrimitiveArray, + pub timer: PrimitiveBuilder, /// Unique, serial ID per item spawned - pub id: MutablePrimitiveArray, + pub id: PrimitiveBuilder, /// *Added: v3.2* Miscellaneous item state pub misc: Option, /// *Added: v3.6* Port that owns the item (-1 when unowned) - pub owner: Option>, + pub owner: Option>, /// *Added: v3.16* Inherited instance ID of the owner (0 when unowned) - pub instance_id: Option>, + pub instance_id: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Item { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - r#type: MutablePrimitiveArray::::with_capacity(capacity), - state: MutablePrimitiveArray::::with_capacity(capacity), - direction: MutablePrimitiveArray::::with_capacity(capacity), + r#type: PrimitiveBuilder::::with_capacity(capacity), + state: PrimitiveBuilder::::with_capacity(capacity), + direction: PrimitiveBuilder::::with_capacity(capacity), velocity: Velocity::with_capacity(capacity, version), position: Position::with_capacity(capacity, version), - damage: MutablePrimitiveArray::::with_capacity(capacity), - timer: MutablePrimitiveArray::::with_capacity(capacity), - id: MutablePrimitiveArray::::with_capacity(capacity), + damage: PrimitiveBuilder::::with_capacity(capacity), + timer: PrimitiveBuilder::::with_capacity(capacity), + id: PrimitiveBuilder::::with_capacity(capacity), misc: version .gte(3, 2) .then(|| ItemMisc::with_capacity(capacity, version)), owner: version .gte(3, 6) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), instance_id: version .gte(3, 16) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), - validity: None, + .then(|| PrimitiveBuilder::::with_capacity(capacity)), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.r#type.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.r#type.push_null(); - self.state.push_null(); - self.direction.push_null(); + self.validity.append(false); + self.r#type.append_null(); + self.state.append_null(); + self.direction.append_null(); self.velocity.push_null(version); self.position.push_null(version); - self.damage.push_null(); - self.timer.push_null(); - self.id.push_null(); + self.damage.append_null(); + self.timer.append_null(); + self.id.append_null(); if version.gte(3, 2) { self.misc.as_mut().unwrap().push_null(version); if version.gte(3, 6) { - self.owner.as_mut().unwrap().push_null(); + self.owner.as_mut().unwrap().append_null(); if version.gte(3, 16) { - self.instance_id.as_mut().unwrap().push_null() + self.instance_id.as_mut().unwrap().append_null() } } } } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u16::().map(|x| self.r#type.push(Some(x)))?; - r.read_u8().map(|x| self.state.push(Some(x)))?; - r.read_f32::().map(|x| self.direction.push(Some(x)))?; + r.read_u16::().map(|x| self.r#type.append_value(x))?; + r.read_u8().map(|x| self.state.append_value(x))?; + r.read_f32::().map(|x| self.direction.append_value(x))?; self.velocity.read_push(r, version)?; self.position.read_push(r, version)?; - r.read_u16::().map(|x| self.damage.push(Some(x)))?; - r.read_f32::().map(|x| self.timer.push(Some(x)))?; - r.read_u32::().map(|x| self.id.push(Some(x)))?; + r.read_u16::().map(|x| self.damage.append_value(x))?; + r.read_f32::().map(|x| self.timer.append_value(x))?; + r.read_u32::().map(|x| self.id.append_value(x))?; if version.gte(3, 2) { self.misc.as_mut().unwrap().read_push(r, version)?; if version.gte(3, 6) { r.read_i8() - .map(|x| self.owner.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.owner.as_mut().unwrap().append_value(x))?; if version.gte(3, 16) { r.read_u16::() - .map(|x| self.instance_id.as_mut().unwrap().push(Some(x)))? + .map(|x| self.instance_id.as_mut().unwrap().append_value(x))? } } }; - self.validity.as_mut().map(|v| v.push(true)); + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Item { - transpose::Item { - r#type: self.r#type.values()[i], - state: self.state.values()[i], - direction: self.direction.values()[i], - velocity: self.velocity.transpose_one(i, version), - position: self.position.transpose_one(i, version), - damage: self.damage.values()[i], - timer: self.timer.values()[i], - id: self.id.values()[i], - misc: self.misc.as_ref().map(|x| x.transpose_one(i, version)), - owner: self.owner.as_ref().map(|x| x.values()[i]), - instance_id: self.instance_id.as_ref().map(|x| x.values()[i]), + fn finish(&mut self) -> immutable::Item { + immutable::Item { + r#type: self.r#type.finish(), + state: self.state.finish(), + direction: self.direction.finish(), + velocity: self.velocity.finish(), + position: self.position.finish(), + damage: self.damage.finish(), + timer: self.timer.finish(), + id: self.id.finish(), + misc: self.misc.as_mut().map(|x| x.finish()), + owner: self.owner.as_mut().map(|x| x.finish()), + instance_id: self.instance_id.as_mut().map(|x| x.finish()), + validity: self.validity.finish(), } } } @@ -338,47 +372,47 @@ impl Item { /// Miscellaneous item state. pub struct ItemMisc( - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, + pub PrimitiveBuilder, + pub PrimitiveBuilder, + pub PrimitiveBuilder, + pub PrimitiveBuilder, ); impl ItemMisc { fn with_capacity(capacity: usize, version: Version) -> Self { Self( - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), ) } pub fn len(&self) -> usize { - self.0.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { - self.0.push_null(); - self.1.push_null(); - self.2.push_null(); - self.3.push_null() + self.0.append_null(); + self.1.append_null(); + self.2.append_null(); + self.3.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u8().map(|x| self.0.push(Some(x)))?; - r.read_u8().map(|x| self.1.push(Some(x)))?; - r.read_u8().map(|x| self.2.push(Some(x)))?; - r.read_u8().map(|x| self.3.push(Some(x)))?; + r.read_u8().map(|x| self.0.append_value(x))?; + r.read_u8().map(|x| self.1.append_value(x))?; + r.read_u8().map(|x| self.2.append_value(x))?; + r.read_u8().map(|x| self.3.append_value(x))?; Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::ItemMisc { - transpose::ItemMisc( - self.0.values()[i], - self.1.values()[i], - self.2.values()[i], - self.3.values()[i], + fn finish(&mut self) -> immutable::ItemMisc { + immutable::ItemMisc( + self.0.finish(), + self.1.finish(), + self.2.finish(), + self.3.finish(), ) } } @@ -386,45 +420,44 @@ impl ItemMisc { /// 2D position. pub struct Position { - pub x: MutablePrimitiveArray, - pub y: MutablePrimitiveArray, + pub x: PrimitiveBuilder, + pub y: PrimitiveBuilder, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Position { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - x: MutablePrimitiveArray::::with_capacity(capacity), - y: MutablePrimitiveArray::::with_capacity(capacity), - validity: None, + x: PrimitiveBuilder::::with_capacity(capacity), + y: PrimitiveBuilder::::with_capacity(capacity), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.x.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.x.push_null(); - self.y.push_null() + self.validity.append(false); + self.x.append_null(); + self.y.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_f32::().map(|x| self.x.push(Some(x)))?; - r.read_f32::().map(|x| self.y.push(Some(x)))?; - self.validity.as_mut().map(|v| v.push(true)); + r.read_f32::().map(|x| self.x.append_value(x))?; + r.read_f32::().map(|x| self.y.append_value(x))?; + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Position { - transpose::Position { - x: self.x.values()[i], - y: self.y.values()[i], + fn finish(&mut self) -> immutable::Position { + immutable::Position { + x: self.x.finish(), + y: self.y.finish(), + validity: self.validity.finish(), } } } @@ -435,150 +468,148 @@ impl Position { pub struct Post { /// In-game character (can only change for Zelda/Sheik) - pub character: MutablePrimitiveArray, + pub character: PrimitiveBuilder, /// Character’s action state - pub state: MutablePrimitiveArray, + pub state: PrimitiveBuilder, /// Character’s position pub position: Position, /// Direction the character is facing - pub direction: MutablePrimitiveArray, + pub direction: PrimitiveBuilder, /// Damage taken (percent) - pub percent: MutablePrimitiveArray, + pub percent: PrimitiveBuilder, /// Size/health of shield - pub shield: MutablePrimitiveArray, + pub shield: PrimitiveBuilder, /// Last attack ID that this character landed - pub last_attack_landed: MutablePrimitiveArray, + pub last_attack_landed: PrimitiveBuilder, /// Combo count (as defined by the game) - pub combo_count: MutablePrimitiveArray, + pub combo_count: PrimitiveBuilder, /// Port that last hit this player. Bugged in Melee: will be set to `6` in certain situations - pub last_hit_by: MutablePrimitiveArray, + pub last_hit_by: PrimitiveBuilder, /// Number of stocks remaining - pub stocks: MutablePrimitiveArray, + pub stocks: PrimitiveBuilder, /// *Added: v0.2* Number of frames action state has been active. Can have a fractional component - pub state_age: Option>, + pub state_age: Option>, /// *Added: v2.0* State flags pub state_flags: Option, /// *Added: v2.0* Used for different things. While in hitstun, contains hitstun frames remaining - pub misc_as: Option>, + pub misc_as: Option>, /// *Added: v2.0* Is the character airborne? - pub airborne: Option>, + pub airborne: Option>, /// *Added: v2.0* Ground ID the character last touched - pub ground: Option>, + pub ground: Option>, /// *Added: v2.0* Number of jumps remaining - pub jumps: Option>, + pub jumps: Option>, /// *Added: v2.0* L-cancel status (0 = none, 1 = successful, 2 = unsuccessful) - pub l_cancel: Option>, + pub l_cancel: Option>, /// *Added: v2.1* Hurtbox state (0 = vulnerable, 1 = invulnerable, 2 = intangible) - pub hurtbox_state: Option>, + pub hurtbox_state: Option>, /// *Added: v3.5* Self-induced and knockback velocities pub velocities: Option, /// *Added: v3.8* Hitlag frames remaining - pub hitlag: Option>, + pub hitlag: Option>, /// *Added: v3.11* Animation the character is in - pub animation_index: Option>, + pub animation_index: Option>, /// *Added: v3.16* Instance ID of the player/item that last hit this player - pub last_hit_by_instance: Option>, + pub last_hit_by_instance: Option>, /// *Added: v3.16* Unique, serial ID for each new action state across all characters. Resets to 0 on death - pub instance_id: Option>, + pub instance_id: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Post { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - character: MutablePrimitiveArray::::with_capacity(capacity), - state: MutablePrimitiveArray::::with_capacity(capacity), + character: PrimitiveBuilder::::with_capacity(capacity), + state: PrimitiveBuilder::::with_capacity(capacity), position: Position::with_capacity(capacity, version), - direction: MutablePrimitiveArray::::with_capacity(capacity), - percent: MutablePrimitiveArray::::with_capacity(capacity), - shield: MutablePrimitiveArray::::with_capacity(capacity), - last_attack_landed: MutablePrimitiveArray::::with_capacity(capacity), - combo_count: MutablePrimitiveArray::::with_capacity(capacity), - last_hit_by: MutablePrimitiveArray::::with_capacity(capacity), - stocks: MutablePrimitiveArray::::with_capacity(capacity), + direction: PrimitiveBuilder::::with_capacity(capacity), + percent: PrimitiveBuilder::::with_capacity(capacity), + shield: PrimitiveBuilder::::with_capacity(capacity), + last_attack_landed: PrimitiveBuilder::::with_capacity(capacity), + combo_count: PrimitiveBuilder::::with_capacity(capacity), + last_hit_by: PrimitiveBuilder::::with_capacity(capacity), + stocks: PrimitiveBuilder::::with_capacity(capacity), state_age: version .gte(0, 2) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), state_flags: version .gte(2, 0) .then(|| StateFlags::with_capacity(capacity, version)), misc_as: version .gte(2, 0) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), airborne: version .gte(2, 0) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), ground: version .gte(2, 0) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), jumps: version .gte(2, 0) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), l_cancel: version .gte(2, 0) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), hurtbox_state: version .gte(2, 1) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), velocities: version .gte(3, 5) .then(|| Velocities::with_capacity(capacity, version)), hitlag: version .gte(3, 8) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), animation_index: version .gte(3, 11) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), last_hit_by_instance: version .gte(3, 16) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), instance_id: version .gte(3, 16) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), - validity: None, + .then(|| PrimitiveBuilder::::with_capacity(capacity)), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.character.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.character.push_null(); - self.state.push_null(); + self.validity.append(false); + self.character.append_null(); + self.state.append_null(); self.position.push_null(version); - self.direction.push_null(); - self.percent.push_null(); - self.shield.push_null(); - self.last_attack_landed.push_null(); - self.combo_count.push_null(); - self.last_hit_by.push_null(); - self.stocks.push_null(); + self.direction.append_null(); + self.percent.append_null(); + self.shield.append_null(); + self.last_attack_landed.append_null(); + self.combo_count.append_null(); + self.last_hit_by.append_null(); + self.stocks.append_null(); if version.gte(0, 2) { - self.state_age.as_mut().unwrap().push_null(); + self.state_age.as_mut().unwrap().append_null(); if version.gte(2, 0) { self.state_flags.as_mut().unwrap().push_null(version); - self.misc_as.as_mut().unwrap().push_null(); - self.airborne.as_mut().unwrap().push_null(); - self.ground.as_mut().unwrap().push_null(); - self.jumps.as_mut().unwrap().push_null(); - self.l_cancel.as_mut().unwrap().push_null(); + self.misc_as.as_mut().unwrap().append_null(); + self.airborne.as_mut().unwrap().append_null(); + self.ground.as_mut().unwrap().append_null(); + self.jumps.as_mut().unwrap().append_null(); + self.l_cancel.as_mut().unwrap().append_null(); if version.gte(2, 1) { - self.hurtbox_state.as_mut().unwrap().push_null(); + self.hurtbox_state.as_mut().unwrap().append_null(); if version.gte(3, 5) { self.velocities.as_mut().unwrap().push_null(version); if version.gte(3, 8) { - self.hitlag.as_mut().unwrap().push_null(); + self.hitlag.as_mut().unwrap().append_null(); if version.gte(3, 11) { - self.animation_index.as_mut().unwrap().push_null(); + self.animation_index.as_mut().unwrap().append_null(); if version.gte(3, 16) { - self.last_hit_by_instance.as_mut().unwrap().push_null(); - self.instance_id.as_mut().unwrap().push_null() + self.last_hit_by_instance.as_mut().unwrap().append_null(); + self.instance_id.as_mut().unwrap().append_null() } } } @@ -589,49 +620,51 @@ impl Post { } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u8().map(|x| self.character.push(Some(x)))?; - r.read_u16::().map(|x| self.state.push(Some(x)))?; + r.read_u8().map(|x| self.character.append_value(x))?; + r.read_u16::().map(|x| self.state.append_value(x))?; self.position.read_push(r, version)?; - r.read_f32::().map(|x| self.direction.push(Some(x)))?; - r.read_f32::().map(|x| self.percent.push(Some(x)))?; - r.read_f32::().map(|x| self.shield.push(Some(x)))?; - r.read_u8().map(|x| self.last_attack_landed.push(Some(x)))?; - r.read_u8().map(|x| self.combo_count.push(Some(x)))?; - r.read_u8().map(|x| self.last_hit_by.push(Some(x)))?; - r.read_u8().map(|x| self.stocks.push(Some(x)))?; + r.read_f32::().map(|x| self.direction.append_value(x))?; + r.read_f32::().map(|x| self.percent.append_value(x))?; + r.read_f32::().map(|x| self.shield.append_value(x))?; + r.read_u8() + .map(|x| self.last_attack_landed.append_value(x))?; + r.read_u8().map(|x| self.combo_count.append_value(x))?; + r.read_u8().map(|x| self.last_hit_by.append_value(x))?; + r.read_u8().map(|x| self.stocks.append_value(x))?; if version.gte(0, 2) { r.read_f32::() - .map(|x| self.state_age.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.state_age.as_mut().unwrap().append_value(x))?; if version.gte(2, 0) { self.state_flags.as_mut().unwrap().read_push(r, version)?; r.read_f32::() - .map(|x| self.misc_as.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.misc_as.as_mut().unwrap().append_value(x))?; r.read_u8() - .map(|x| self.airborne.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.airborne.as_mut().unwrap().append_value(x))?; r.read_u16::() - .map(|x| self.ground.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.ground.as_mut().unwrap().append_value(x))?; r.read_u8() - .map(|x| self.jumps.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.jumps.as_mut().unwrap().append_value(x))?; r.read_u8() - .map(|x| self.l_cancel.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.l_cancel.as_mut().unwrap().append_value(x))?; if version.gte(2, 1) { r.read_u8() - .map(|x| self.hurtbox_state.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.hurtbox_state.as_mut().unwrap().append_value(x))?; if version.gte(3, 5) { self.velocities.as_mut().unwrap().read_push(r, version)?; if version.gte(3, 8) { r.read_f32::() - .map(|x| self.hitlag.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.hitlag.as_mut().unwrap().append_value(x))?; if version.gte(3, 11) { r.read_u32::().map(|x| { - self.animation_index.as_mut().unwrap().push(Some(x)) + self.animation_index.as_mut().unwrap().append_value(x) })?; if version.gte(3, 16) { r.read_u16::().map(|x| { - self.last_hit_by_instance.as_mut().unwrap().push(Some(x)) + self.last_hit_by_instance.as_mut().unwrap().append_value(x) })?; - r.read_u16::() - .map(|x| self.instance_id.as_mut().unwrap().push(Some(x)))? + r.read_u16::().map(|x| { + self.instance_id.as_mut().unwrap().append_value(x) + })? } } } @@ -639,41 +672,36 @@ impl Post { } } }; - self.validity.as_mut().map(|v| v.push(true)); + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Post { - transpose::Post { - character: self.character.values()[i], - state: self.state.values()[i], - position: self.position.transpose_one(i, version), - direction: self.direction.values()[i], - percent: self.percent.values()[i], - shield: self.shield.values()[i], - last_attack_landed: self.last_attack_landed.values()[i], - combo_count: self.combo_count.values()[i], - last_hit_by: self.last_hit_by.values()[i], - stocks: self.stocks.values()[i], - state_age: self.state_age.as_ref().map(|x| x.values()[i]), - state_flags: self - .state_flags - .as_ref() - .map(|x| x.transpose_one(i, version)), - misc_as: self.misc_as.as_ref().map(|x| x.values()[i]), - airborne: self.airborne.as_ref().map(|x| x.values()[i]), - ground: self.ground.as_ref().map(|x| x.values()[i]), - jumps: self.jumps.as_ref().map(|x| x.values()[i]), - l_cancel: self.l_cancel.as_ref().map(|x| x.values()[i]), - hurtbox_state: self.hurtbox_state.as_ref().map(|x| x.values()[i]), - velocities: self - .velocities - .as_ref() - .map(|x| x.transpose_one(i, version)), - hitlag: self.hitlag.as_ref().map(|x| x.values()[i]), - animation_index: self.animation_index.as_ref().map(|x| x.values()[i]), - last_hit_by_instance: self.last_hit_by_instance.as_ref().map(|x| x.values()[i]), - instance_id: self.instance_id.as_ref().map(|x| x.values()[i]), + fn finish(&mut self) -> immutable::Post { + immutable::Post { + character: self.character.finish(), + state: self.state.finish(), + position: self.position.finish(), + direction: self.direction.finish(), + percent: self.percent.finish(), + shield: self.shield.finish(), + last_attack_landed: self.last_attack_landed.finish(), + combo_count: self.combo_count.finish(), + last_hit_by: self.last_hit_by.finish(), + stocks: self.stocks.finish(), + state_age: self.state_age.as_mut().map(|x| x.finish()), + state_flags: self.state_flags.as_mut().map(|x| x.finish()), + misc_as: self.misc_as.as_mut().map(|x| x.finish()), + airborne: self.airborne.as_mut().map(|x| x.finish()), + ground: self.ground.as_mut().map(|x| x.finish()), + jumps: self.jumps.as_mut().map(|x| x.finish()), + l_cancel: self.l_cancel.as_mut().map(|x| x.finish()), + hurtbox_state: self.hurtbox_state.as_mut().map(|x| x.finish()), + velocities: self.velocities.as_mut().map(|x| x.finish()), + hitlag: self.hitlag.as_mut().map(|x| x.finish()), + animation_index: self.animation_index.as_mut().map(|x| x.finish()), + last_hit_by_instance: self.last_hit_by_instance.as_mut().map(|x| x.finish()), + instance_id: self.instance_id.as_mut().map(|x| x.finish()), + validity: self.validity.finish(), } } } @@ -684,134 +712,134 @@ impl Post { pub struct Pre { /// Random seed - pub random_seed: MutablePrimitiveArray, + pub random_seed: PrimitiveBuilder, /// Character’s action state - pub state: MutablePrimitiveArray, + pub state: PrimitiveBuilder, /// Character’s position pub position: Position, /// Direction the character is facing - pub direction: MutablePrimitiveArray, + pub direction: PrimitiveBuilder, /// Processed analog joystick position pub joystick: Position, /// Processed analog c-stick position pub cstick: Position, /// Processed analog trigger position - pub triggers: MutablePrimitiveArray, + pub triggers: PrimitiveBuilder, /// Processed button-state bitmask - pub buttons: MutablePrimitiveArray, + pub buttons: PrimitiveBuilder, /// Physical button-state bitmask - pub buttons_physical: MutablePrimitiveArray, + pub buttons_physical: PrimitiveBuilder, /// Physical analog trigger positions (useful for IPM) pub triggers_physical: TriggersPhysical, /// *Added: v1.2* Raw joystick x-position - pub raw_analog_x: Option>, + pub raw_analog_x: Option>, /// *Added: v1.4* Damage taken (percent) - pub percent: Option>, + pub percent: Option>, /// *Added: v3.15* Raw joystick y-position - pub raw_analog_y: Option>, + pub raw_analog_y: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Pre { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - random_seed: MutablePrimitiveArray::::with_capacity(capacity), - state: MutablePrimitiveArray::::with_capacity(capacity), + random_seed: PrimitiveBuilder::::with_capacity(capacity), + state: PrimitiveBuilder::::with_capacity(capacity), position: Position::with_capacity(capacity, version), - direction: MutablePrimitiveArray::::with_capacity(capacity), + direction: PrimitiveBuilder::::with_capacity(capacity), joystick: Position::with_capacity(capacity, version), cstick: Position::with_capacity(capacity, version), - triggers: MutablePrimitiveArray::::with_capacity(capacity), - buttons: MutablePrimitiveArray::::with_capacity(capacity), - buttons_physical: MutablePrimitiveArray::::with_capacity(capacity), + triggers: PrimitiveBuilder::::with_capacity(capacity), + buttons: PrimitiveBuilder::::with_capacity(capacity), + buttons_physical: PrimitiveBuilder::::with_capacity(capacity), triggers_physical: TriggersPhysical::with_capacity(capacity, version), raw_analog_x: version .gte(1, 2) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), percent: version .gte(1, 4) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), + .then(|| PrimitiveBuilder::::with_capacity(capacity)), raw_analog_y: version .gte(3, 15) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), - validity: None, + .then(|| PrimitiveBuilder::::with_capacity(capacity)), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.random_seed.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.random_seed.push_null(); - self.state.push_null(); + self.validity.append(false); + self.random_seed.append_null(); + self.state.append_null(); self.position.push_null(version); - self.direction.push_null(); + self.direction.append_null(); self.joystick.push_null(version); self.cstick.push_null(version); - self.triggers.push_null(); - self.buttons.push_null(); - self.buttons_physical.push_null(); + self.triggers.append_null(); + self.buttons.append_null(); + self.buttons_physical.append_null(); self.triggers_physical.push_null(version); if version.gte(1, 2) { - self.raw_analog_x.as_mut().unwrap().push_null(); + self.raw_analog_x.as_mut().unwrap().append_null(); if version.gte(1, 4) { - self.percent.as_mut().unwrap().push_null(); + self.percent.as_mut().unwrap().append_null(); if version.gte(3, 15) { - self.raw_analog_y.as_mut().unwrap().push_null() + self.raw_analog_y.as_mut().unwrap().append_null() } } } } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u32::().map(|x| self.random_seed.push(Some(x)))?; - r.read_u16::().map(|x| self.state.push(Some(x)))?; + r.read_u32::() + .map(|x| self.random_seed.append_value(x))?; + r.read_u16::().map(|x| self.state.append_value(x))?; self.position.read_push(r, version)?; - r.read_f32::().map(|x| self.direction.push(Some(x)))?; + r.read_f32::().map(|x| self.direction.append_value(x))?; self.joystick.read_push(r, version)?; self.cstick.read_push(r, version)?; - r.read_f32::().map(|x| self.triggers.push(Some(x)))?; - r.read_u32::().map(|x| self.buttons.push(Some(x)))?; + r.read_f32::().map(|x| self.triggers.append_value(x))?; + r.read_u32::().map(|x| self.buttons.append_value(x))?; r.read_u16::() - .map(|x| self.buttons_physical.push(Some(x)))?; + .map(|x| self.buttons_physical.append_value(x))?; self.triggers_physical.read_push(r, version)?; if version.gte(1, 2) { r.read_i8() - .map(|x| self.raw_analog_x.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.raw_analog_x.as_mut().unwrap().append_value(x))?; if version.gte(1, 4) { r.read_f32::() - .map(|x| self.percent.as_mut().unwrap().push(Some(x)))?; + .map(|x| self.percent.as_mut().unwrap().append_value(x))?; if version.gte(3, 15) { r.read_i8() - .map(|x| self.raw_analog_y.as_mut().unwrap().push(Some(x)))? + .map(|x| self.raw_analog_y.as_mut().unwrap().append_value(x))? } } }; - self.validity.as_mut().map(|v| v.push(true)); + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Pre { - transpose::Pre { - random_seed: self.random_seed.values()[i], - state: self.state.values()[i], - position: self.position.transpose_one(i, version), - direction: self.direction.values()[i], - joystick: self.joystick.transpose_one(i, version), - cstick: self.cstick.transpose_one(i, version), - triggers: self.triggers.values()[i], - buttons: self.buttons.values()[i], - buttons_physical: self.buttons_physical.values()[i], - triggers_physical: self.triggers_physical.transpose_one(i, version), - raw_analog_x: self.raw_analog_x.as_ref().map(|x| x.values()[i]), - percent: self.percent.as_ref().map(|x| x.values()[i]), - raw_analog_y: self.raw_analog_y.as_ref().map(|x| x.values()[i]), + fn finish(&mut self) -> immutable::Pre { + immutable::Pre { + random_seed: self.random_seed.finish(), + state: self.state.finish(), + position: self.position.finish(), + direction: self.direction.finish(), + joystick: self.joystick.finish(), + cstick: self.cstick.finish(), + triggers: self.triggers.finish(), + buttons: self.buttons.finish(), + buttons_physical: self.buttons_physical.finish(), + triggers_physical: self.triggers_physical.finish(), + raw_analog_x: self.raw_analog_x.as_mut().map(|x| x.finish()), + percent: self.percent.as_mut().map(|x| x.finish()), + raw_analog_y: self.raw_analog_y.as_mut().map(|x| x.finish()), + validity: self.validity.finish(), } } } @@ -820,53 +848,53 @@ impl Pre { pub struct Start { /// Random seed - pub random_seed: MutablePrimitiveArray, + pub random_seed: PrimitiveBuilder, /// *Added: v3.10* Scene frame counter. Starts at 0, and increments every frame (even when paused) - pub scene_frame_counter: Option>, + pub scene_frame_counter: Option>, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Start { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - random_seed: MutablePrimitiveArray::::with_capacity(capacity), + random_seed: PrimitiveBuilder::::with_capacity(capacity), scene_frame_counter: version .gte(3, 10) - .then(|| MutablePrimitiveArray::::with_capacity(capacity)), - validity: None, + .then(|| PrimitiveBuilder::::with_capacity(capacity)), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.random_seed.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.random_seed.push_null(); + self.validity.append(false); + self.random_seed.append_null(); if version.gte(3, 10) { - self.scene_frame_counter.as_mut().unwrap().push_null() + self.scene_frame_counter.as_mut().unwrap().append_null() } } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u32::().map(|x| self.random_seed.push(Some(x)))?; + r.read_u32::() + .map(|x| self.random_seed.append_value(x))?; if version.gte(3, 10) { r.read_u32::() - .map(|x| self.scene_frame_counter.as_mut().unwrap().push(Some(x)))? + .map(|x| self.scene_frame_counter.as_mut().unwrap().append_value(x))? }; - self.validity.as_mut().map(|v| v.push(true)); + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Start { - transpose::Start { - random_seed: self.random_seed.values()[i], - scene_frame_counter: self.scene_frame_counter.as_ref().map(|x| x.values()[i]), + fn finish(&mut self) -> immutable::Start { + immutable::Start { + random_seed: self.random_seed.finish(), + scene_frame_counter: self.scene_frame_counter.as_mut().map(|x| x.finish()), + validity: self.validity.finish(), } } } @@ -874,52 +902,52 @@ impl Start { /// Miscellaneous state flags. pub struct StateFlags( - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, - pub MutablePrimitiveArray, + pub PrimitiveBuilder, + pub PrimitiveBuilder, + pub PrimitiveBuilder, + pub PrimitiveBuilder, + pub PrimitiveBuilder, ); impl StateFlags { fn with_capacity(capacity: usize, version: Version) -> Self { Self( - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), - MutablePrimitiveArray::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), + PrimitiveBuilder::::with_capacity(capacity), ) } pub fn len(&self) -> usize { - self.0.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { - self.0.push_null(); - self.1.push_null(); - self.2.push_null(); - self.3.push_null(); - self.4.push_null() + self.0.append_null(); + self.1.append_null(); + self.2.append_null(); + self.3.append_null(); + self.4.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_u8().map(|x| self.0.push(Some(x)))?; - r.read_u8().map(|x| self.1.push(Some(x)))?; - r.read_u8().map(|x| self.2.push(Some(x)))?; - r.read_u8().map(|x| self.3.push(Some(x)))?; - r.read_u8().map(|x| self.4.push(Some(x)))?; + r.read_u8().map(|x| self.0.append_value(x))?; + r.read_u8().map(|x| self.1.append_value(x))?; + r.read_u8().map(|x| self.2.append_value(x))?; + r.read_u8().map(|x| self.3.append_value(x))?; + r.read_u8().map(|x| self.4.append_value(x))?; Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::StateFlags { - transpose::StateFlags( - self.0.values()[i], - self.1.values()[i], - self.2.values()[i], - self.3.values()[i], - self.4.values()[i], + fn finish(&mut self) -> immutable::StateFlags { + immutable::StateFlags( + self.0.finish(), + self.1.finish(), + self.2.finish(), + self.3.finish(), + self.4.finish(), ) } } @@ -927,45 +955,44 @@ impl StateFlags { /// Trigger state. pub struct TriggersPhysical { - pub l: MutablePrimitiveArray, - pub r: MutablePrimitiveArray, + pub l: PrimitiveBuilder, + pub r: PrimitiveBuilder, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl TriggersPhysical { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - l: MutablePrimitiveArray::::with_capacity(capacity), - r: MutablePrimitiveArray::::with_capacity(capacity), - validity: None, + l: PrimitiveBuilder::::with_capacity(capacity), + r: PrimitiveBuilder::::with_capacity(capacity), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.l.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.l.push_null(); - self.r.push_null() + self.validity.append(false); + self.l.append_null(); + self.r.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_f32::().map(|x| self.l.push(Some(x)))?; - r.read_f32::().map(|x| self.r.push(Some(x)))?; - self.validity.as_mut().map(|v| v.push(true)); + r.read_f32::().map(|x| self.l.append_value(x))?; + r.read_f32::().map(|x| self.r.append_value(x))?; + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::TriggersPhysical { - transpose::TriggersPhysical { - l: self.l.values()[i], - r: self.r.values()[i], + fn finish(&mut self) -> immutable::TriggersPhysical { + immutable::TriggersPhysical { + l: self.l.finish(), + r: self.r.finish(), + validity: self.validity.finish(), } } } @@ -974,65 +1001,67 @@ impl TriggersPhysical { pub struct Velocities { /// Self-induced x-velocity (airborne) - pub self_x_air: MutablePrimitiveArray, + pub self_x_air: PrimitiveBuilder, /// Self-induced y-velocity - pub self_y: MutablePrimitiveArray, + pub self_y: PrimitiveBuilder, /// Knockback-induced x-velocity - pub knockback_x: MutablePrimitiveArray, + pub knockback_x: PrimitiveBuilder, /// Knockback-induced y-velocity - pub knockback_y: MutablePrimitiveArray, + pub knockback_y: PrimitiveBuilder, /// Self-induced x-velocity (grounded) - pub self_x_ground: MutablePrimitiveArray, + pub self_x_ground: PrimitiveBuilder, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Velocities { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - self_x_air: MutablePrimitiveArray::::with_capacity(capacity), - self_y: MutablePrimitiveArray::::with_capacity(capacity), - knockback_x: MutablePrimitiveArray::::with_capacity(capacity), - knockback_y: MutablePrimitiveArray::::with_capacity(capacity), - self_x_ground: MutablePrimitiveArray::::with_capacity(capacity), - validity: None, + self_x_air: PrimitiveBuilder::::with_capacity(capacity), + self_y: PrimitiveBuilder::::with_capacity(capacity), + knockback_x: PrimitiveBuilder::::with_capacity(capacity), + knockback_y: PrimitiveBuilder::::with_capacity(capacity), + self_x_ground: PrimitiveBuilder::::with_capacity(capacity), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.self_x_air.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.self_x_air.push_null(); - self.self_y.push_null(); - self.knockback_x.push_null(); - self.knockback_y.push_null(); - self.self_x_ground.push_null() + self.validity.append(false); + self.self_x_air.append_null(); + self.self_y.append_null(); + self.knockback_x.append_null(); + self.knockback_y.append_null(); + self.self_x_ground.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_f32::().map(|x| self.self_x_air.push(Some(x)))?; - r.read_f32::().map(|x| self.self_y.push(Some(x)))?; - r.read_f32::().map(|x| self.knockback_x.push(Some(x)))?; - r.read_f32::().map(|x| self.knockback_y.push(Some(x)))?; r.read_f32::() - .map(|x| self.self_x_ground.push(Some(x)))?; - self.validity.as_mut().map(|v| v.push(true)); + .map(|x| self.self_x_air.append_value(x))?; + r.read_f32::().map(|x| self.self_y.append_value(x))?; + r.read_f32::() + .map(|x| self.knockback_x.append_value(x))?; + r.read_f32::() + .map(|x| self.knockback_y.append_value(x))?; + r.read_f32::() + .map(|x| self.self_x_ground.append_value(x))?; + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Velocities { - transpose::Velocities { - self_x_air: self.self_x_air.values()[i], - self_y: self.self_y.values()[i], - knockback_x: self.knockback_x.values()[i], - knockback_y: self.knockback_y.values()[i], - self_x_ground: self.self_x_ground.values()[i], + fn finish(&mut self) -> immutable::Velocities { + immutable::Velocities { + self_x_air: self.self_x_air.finish(), + self_y: self.self_y.finish(), + knockback_x: self.knockback_x.finish(), + knockback_y: self.knockback_y.finish(), + self_x_ground: self.self_x_ground.finish(), + validity: self.validity.finish(), } } } @@ -1040,45 +1069,44 @@ impl Velocities { /// 2D velocity. pub struct Velocity { - pub x: MutablePrimitiveArray, - pub y: MutablePrimitiveArray, + pub x: PrimitiveBuilder, + pub y: PrimitiveBuilder, /// Indicates which indexes are valid (`None` means "all valid"). Invalid indexes can occur on frames where a character is absent (ICs or 2v2 games) - pub validity: Option, + pub validity: NullBufferBuilder, } impl Velocity { fn with_capacity(capacity: usize, version: Version) -> Self { Self { - x: MutablePrimitiveArray::::with_capacity(capacity), - y: MutablePrimitiveArray::::with_capacity(capacity), - validity: None, + x: PrimitiveBuilder::::with_capacity(capacity), + y: PrimitiveBuilder::::with_capacity(capacity), + validity: NullBufferBuilder::new(capacity), } } pub fn len(&self) -> usize { - self.x.len() + self.validity.len() } pub fn push_null(&mut self, version: Version) { let len = self.len(); - self.validity - .get_or_insert_with(|| MutableBitmap::from_len_set(len)) - .push(false); - self.x.push_null(); - self.y.push_null() + self.validity.append(false); + self.x.append_null(); + self.y.append_null() } pub fn read_push(&mut self, r: &mut &[u8], version: Version) -> Result<()> { - r.read_f32::().map(|x| self.x.push(Some(x)))?; - r.read_f32::().map(|x| self.y.push(Some(x)))?; - self.validity.as_mut().map(|v| v.push(true)); + r.read_f32::().map(|x| self.x.append_value(x))?; + r.read_f32::().map(|x| self.y.append_value(x))?; + self.validity.append(true); Ok(()) } - pub fn transpose_one(&self, i: usize, version: Version) -> transpose::Velocity { - transpose::Velocity { - x: self.x.values()[i], - y: self.y.values()[i], + fn finish(&mut self) -> immutable::Velocity { + immutable::Velocity { + x: self.x.finish(), + y: self.y.finish(), + validity: self.validity.finish(), } } } diff --git a/src/io/peppi/de.rs b/src/io/peppi/de.rs index c22405d..860a860 100644 --- a/src/io/peppi/de.rs +++ b/src/io/peppi/de.rs @@ -113,7 +113,7 @@ pub fn read(r: R, opts: Option<&Opts>) -> Result { true => { let start = start.as_ref().ok_or(err!("missing start"))?; MutableFrame::with_capacity(0, start.slippi.version, &port_occupancy(start)) - .into() + .finish() } _ => read_arrow_frames(file, version)?, }); diff --git a/src/io/peppi/ser.rs b/src/io/peppi/ser.rs index 0626fd1..1edf02f 100644 --- a/src/io/peppi/ser.rs +++ b/src/io/peppi/ser.rs @@ -1,11 +1,16 @@ use std::{error::Error, io::Write, path::Path}; -use arrow2::{ - array::Array, - chunk::Chunk, - datatypes::{Field, Schema}, - io::ipc::write::{Compression, FileWriter, WriteOptions}, -}; +use arrow::array::Array; +use arrow::datatypes::{Field, Schema}; +use arrow_ipc::gen::Message::CompressionType; +use arrow_ipc::writer::{FileWriter, IpcWriteOptions}; + +//use arrow2::{ +// array::Array, +// chunk::Chunk, +// datatypes::{Field, Schema}, +// io::ipc::write::{Compression, FileWriter, WriteOptions}, +//}; use crate::{ game::{immutable::Game, port_occupancy}, @@ -19,7 +24,7 @@ pub struct Opts { /// /// Use this to maximize read speed while saving some disk space (e.g. for machine learning). /// If you just want maximum compression, compress the entire `.slpp` file instead. - pub compression: Option, + pub compression: Option, } fn tar_append>( @@ -72,27 +77,23 @@ pub fn write(w: W, game: Game, opts: Option<&Opts>) -> Result<(), Box< if game.frames.id.len() > 0 { let ports = port_occupancy(&game.start); - let batch = game + let array = game .frames .into_struct_array(game.start.slippi.version, &ports); - let schema = Schema::from(vec![Field { - name: "frame".to_string(), - data_type: batch.data_type().clone(), - is_nullable: false, - metadata: Default::default(), - }]); + let schema = Schema::new(vec![Field::new( + "frame".to_string(), + array.data_type().clone(), + false, + )]); - let chunk = Chunk::new(vec![Box::new(batch) as Box]); + let batch = RecordBatch::try_new(schema, vec![Box::new(array) as Box])?; let mut buf = Vec::new(); - let mut writer = FileWriter::try_new( + let mut writer = FileWriter::try_new_with_options( &mut buf, - schema, - None, - WriteOptions { - compression: opts.map_or(None, |o| o.compression), - }, + &schema, + IpcWriteOptions::default().try_with_compression(opts.and_then(|o| o.compression))?, )?; - writer.write(&chunk, None)?; + writer.write(&batch)?; writer.finish()?; tar_append(&mut tar, &buf, "frames.arrow")?; } diff --git a/src/io/slippi/de.rs b/src/io/slippi/de.rs index 278efbd..fa7ca95 100644 --- a/src/io/slippi/de.rs +++ b/src/io/slippi/de.rs @@ -7,7 +7,7 @@ use std::{ path::PathBuf, }; -use arrow2::array::MutableArray; +use arrow::array::ArrayBuilder; use byteorder::ReadBytesExt; use log::{debug, info, trace, warn}; @@ -133,11 +133,11 @@ impl ParseState { } fn last_id(&self) -> Option { - self.game.frames.id.values().last().map(|id| *id) + self.game.frames.id.values_slice().last().map(|id| *id) } fn frame_open(&mut self, id: i32) { - self.game.frames.id.push(Some(id)); + self.game.frames.id.append_value(id); } fn frame_close(&mut self) { @@ -708,8 +708,7 @@ pub fn parse_event(mut r: R, state: &mut ParseState, opts: Option<&Opts .as_mut() .unwrap() .validity - .as_mut() - .map(|v| v.push(true)); + .append(true); state.game.frames.ports[port_index] .follower .as_mut() @@ -720,8 +719,7 @@ pub fn parse_event(mut r: R, state: &mut ParseState, opts: Option<&Opts state.game.frames.ports[port_index] .leader .validity - .as_mut() - .map(|v| v.push(true)); + .append(true); state.game.frames.ports[port_index] .leader .pre @@ -753,25 +751,22 @@ pub fn parse_event(mut r: R, state: &mut ParseState, opts: Option<&Opts let id = r.read_i32::()?; trace!("Frame end: {}", id); assert_eq!(id, state.last_id().unwrap()); - let old_len = *state.game.frames.item_offset.as_ref().unwrap().last(); - let new_len: i32 = state + let old_len = *state.game.frames.item_offset.as_ref().unwrap().last().unwrap(); + let new_len: usize = state .game .frames .item .as_ref() .unwrap() .r#type - .len() - .try_into() - .unwrap(); + .len(); state .game .frames .item_offset .as_mut() .unwrap() - .try_push(new_len.checked_sub(old_len).unwrap()) - .unwrap(); + .push_length(new_len.checked_sub(old_len.try_into().unwrap()).unwrap()); state .game .frames diff --git a/src/io/slippi/ser.rs b/src/io/slippi/ser.rs index 29596d1..5a8db38 100644 --- a/src/io/slippi/ser.rs +++ b/src/io/slippi/ser.rs @@ -126,9 +126,9 @@ fn frame_counts(frames: &Frame) -> FrameCounts { .ports .iter() .map(|p| { - len - p.leader.validity.as_ref().map_or(0, |v| v.unset_bits()) + len - p.leader.validity.as_ref().map_or(0, |v| v.null_count()) + p.follower.as_ref().map_or(0, |f| { - len - f.validity.as_ref().map_or(0, |v| v.unset_bits()) + len - f.validity.as_ref().map_or(0, |v| v.null_count()) }) }) .sum::()