Skip to content

Commit

Permalink
[WIP] Switch back to arrow-rs
Browse files Browse the repository at this point in the history
  • Loading branch information
hohav committed Aug 19, 2024
1 parent d0a45cb commit 3c1ca50
Show file tree
Hide file tree
Showing 18 changed files with 1,224 additions and 1,178 deletions.
5 changes: 3 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ exclude = ["benches", "tests"]
readme = "README.md"

[dependencies]
arrow2 = { version = "0.17", features = ["io_ipc", "io_ipc_compression", "io_json" ] }
arrow = "52"
arrow-buffer = "52"
arrow-ipc = "52"
byteorder = "1"
encoding_rs = "0.8"
log = "0.4"
Expand All @@ -22,7 +24,6 @@ thiserror = "1.0"
xxhash-rust = { version = "0.8", features = ["xxh3"] }

[dev-dependencies]
arrow2 = { version = "0.17", features = ["io_json"] }
criterion = "0.5"
iai-callgrind = "0.11"
pretty_assertions = "1.3"
Expand Down
85 changes: 43 additions & 42 deletions gen/resources/preamble/immutable/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! [`crate::io::peppi::read`].
//!
//! These arrays can be shared, and cloning them is `O(1)`. See the
//! [arrow2 docs](https://docs.rs/arrow2/latest/arrow2/array/index.html) for more.
//! [arrow_array docs](https://docs.rs/arrow-array/latest/arrow_array/index.html) for more.
#![allow(unused_variables)]

Expand All @@ -13,11 +13,12 @@ mod slippi;

use std::fmt;

use arrow2::{
array::PrimitiveArray,
bitmap::Bitmap,
buffer::Buffer,
offset::OffsetsBuffer,
use arrow::{
array::{
types::{Float32Type, Int8Type, Int16Type, Int32Type, UInt8Type, UInt16Type, UInt32Type},
PrimitiveArray,
},
buffer::{NullBuffer, OffsetBuffer},
};

use crate::{
Expand All @@ -31,7 +32,7 @@ use crate::{
pub struct Data {
pub pre: Pre,
pub post: Post,
pub validity: Option<Bitmap>,
pub validity: Option<NullBuffer>,
}

impl Data {
Expand All @@ -43,15 +44,15 @@ impl Data {
}
}

impl From<mutable::Data> for Data {
fn from(d: mutable::Data) -> Self {
Self {
pre: d.pre.into(),
post: d.post.into(),
validity: d.validity.map(|v| v.into()),
}
}
}
//impl From<mutable::Data> for Data {
// fn from(d: mutable::Data) -> Self {
// Self {
// pre: d.pre.finish(),
// post: d.post.finish(),
// validity: d.validity.map(|v| v.into()),
// }
// }
//}

/// Frame data for a single port.
#[derive(Debug)]
Expand All @@ -72,28 +73,28 @@ impl PortData {
}
}

impl From<mutable::PortData> for PortData {
fn from(p: mutable::PortData) -> Self {
Self {
port: p.port,
leader: p.leader.into(),
follower: p.follower.map(|f| f.into()),
}
}
}
//impl From<mutable::PortData> for PortData {
// fn from(p: mutable::PortData) -> Self {
// Self {
// port: p.port,
// leader: p.leader.finish(),
// follower: p.follower.map(|f| f.finish()),
// }
// }
//}

/// All frame data for a single game, in struct-of-arrays format.
pub struct Frame {
/// Frame IDs start at `-123` and increment each frame. May repeat in case of rollbacks
pub id: PrimitiveArray<i32>,
pub id: PrimitiveArray<Int32Type>,
/// Port-specific data
pub ports: Vec<PortData>,
/// Start-of-frame data
pub start: Option<Start>,
/// End-of-frame data
pub end: Option<End>,
/// Logically, each frame has its own array of items. But we represent all item data in a flat array, with this field indicating the start of each sub-array
pub item_offset: Option<OffsetsBuffer<i32>>,
pub item_offset: Option<OffsetBuffer<i32>>,
/// Item data
pub item: Option<Item>,
}
Expand Down Expand Up @@ -135,7 +136,7 @@ impl Frame {

fn rollbacks_<'a>(&self, ids: impl Iterator<Item = (usize, &'a i32)>) -> Vec<bool> {
let mut result = vec![false; self.len()];
let unique_id_count = self.id.values_iter().max().map_or(0, |idx| {
let unique_id_count = arrow::compute::kernels::aggregate::max(&self.id).map_or(0, |idx| {
1 + usize::try_from(idx - frame::FIRST_INDEX).unwrap()
});
let mut seen = vec![false; unique_id_count];
Expand All @@ -152,20 +153,20 @@ impl Frame {
}
}

impl From<mutable::Frame> for Frame {
fn from(f: mutable::Frame) -> Self {
Self {
id: f.id.into(),
ports: f.ports.into_iter().map(|p| p.into()).collect(),
start: f.start.map(|x| x.into()),
end: f.end.map(|x| x.into()),
item_offset: f.item_offset.map(|x|
OffsetsBuffer::try_from(Buffer::from(x.into_inner())).unwrap()
),
item: f.item.map(|x| x.into()),
}
}
}
//impl From<mutable::Frame> for Frame {
// fn from(f: mutable::Frame) -> Self {
// Self {
// id: f.id.into(),
// ports: f.ports.into_iter().map(|p| p.into()).collect(),
// start: f.start.map(|x| x.into()),
// end: f.end.map(|x| x.into()),
// item_offset: f.item_offset.map(|x|
// OffsetBuffer::try_from(Buffer::from(x.into_inner())).unwrap()
// ),
// item: f.item.map(|x| x.into()),
// }
// }
//}

impl fmt::Debug for Frame {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::result::Result<(), fmt::Error> {
Expand Down
Loading

0 comments on commit 3c1ca50

Please sign in to comment.