WIP: Several changes

.gitignore

@@ -5,5 +5,4 @@
 *tar.gz
 *.clang_complete
 *.root
-
 .idea/

alice-open-data/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "alice-open-data"
-version = "0.5.0"
+version = "0.5.1"
 authors = ["cbourjau <c.bourjau@posteo.de>"]
 description = "Tools to download and manage the publicly released ALICE open data"
 repository = "https://github.com/cbourjau/alice-rs"

examples/convert_to_msgpack/Cargo.toml

@@ -8,6 +8,6 @@ edition = "2018"
 alice-open-data = { version="0.5.0", path="../../alice-open-data" }
 malice = { version="0.3.0", path="../../malice" }
 rmp-serde = "0.13.7"
-root-io = { version="0.3.0", path="../../root-io" }
+root-io = { version="0.4", path="../../root-io" }
 serde = "1.0"
 serde_derive = "1.0"

histogram/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "histogram"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["cbourjau <c.bourjau@posteo.de>"]
 edition = "2018"
 

malice/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "malice"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["cbourjau <c.bourjau@posteo.de>"]
 exclude = ["benches/*"]
 description = "A small framwork providing sensible defaults to analyse ALICE's open data"
@@ -19,8 +19,8 @@ cpp = ["alice-sys"]
 bitflags = "1"
 failure = "0.1"
 futures = "0.3"
-nom = "^5"
-root-io = { version="0.3", path="../root-io" }
+nom = "7"
+root-io = { version="0.4", path="../root-io" }
 # Optional dependencies
 alice-sys = { version="0.1", optional = true }
 itertools = "0.10"

malice/src/event.rs

@@ -4,10 +4,12 @@ use std::fmt::Debug;
 use failure::Error;
 use futures::prelude::*;
 use itertools::izip;
-use nom::{combinator::map, error::ParseError, number::complete::*, sequence::tuple, IResult};
+use nom::{combinator::map, number::complete::*, sequence::tuple};
 use wasm_bindgen::prelude::*;
 
-use root_io::core::parsers::{parse_custom_mantissa, parse_tobjarray_of_tnameds};
+use root_io::core::parsers::{
+    parse_custom_mantissa, parse_tobjarray_of_tnameds, RResult, RootError, Span,
+};
 use root_io::stream_zip;
 use root_io::tree_reader::Tree;
 
@@ -241,9 +243,9 @@ fn string_to_mask(s: &str, run_number: i32) -> TriggerMask {
     }
 }
 
-fn parse_pid_probabilities<'s, E>(input: &'s [u8]) -> IResult<&'s [u8], PidProbabilities, E>
+fn parse_pid_probabilities<'s, E>(input: Span<'s>) -> RResult<'s, PidProbabilities, E>
 where
-    E: ParseError<&'s [u8]> + Debug,
+    E: RootError<Span<'s>>,
 {
     let (input, electron) = parse_custom_mantissa(input, 8)?;
     let (input, muon) = parse_custom_mantissa(input, 8)?;

root-io/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "root-io"
-version = "0.3.0"
+version = "0.4.0"
 authors = ["cbourjau <c.bourjau@posteo.de>"]
 exclude = ["*test_data/", "*.root"]
 description = "Reading of `.root` binary files which are commonly used in particle physics"
@@ -13,31 +13,32 @@ edition = "2018"
 
 
 [dependencies]
-alice-open-data = { version="0.5", path="../alice-open-data/" }
-bitflags = "1.0.0"
-failure = "0.1.5"
+alice-open-data = { version = "0.5", path = "../alice-open-data/" }
+bitflags = "1.3"
 flate2 = "^1"
 futures = "0.3"
-lzma-rs = "0.1.1"
-quote = "0.3.15"
-reqwest = "0.10"
+lzma-rs = "0.2"
+quote = "0.3"
+reqwest = "0.11"
 uuid = "0.8.2"
 lz4-compress = "0.1.1"
-
-[dependencies.nom]
-version = "^5"
+nom = "7"
+nom-supreme = "0.6"
+nom_locate = "4.0.0"
+thiserror = "1"
+ouroboros = "0.14"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-wasm-bindgen-futures = "0.4.10"
+wasm-bindgen-futures = "0.4"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 criterion = "0.3"
-tokio = { version = "0.2", features = ["macros"] }
+tokio = { version = "1.15", features = ["macros", "rt-multi-thread"] }
 
 [target.'cfg(target_arch = "wasm32")'.dev-dependencies]
-wasm-bindgen = "0.2.60"
-wasm-bindgen-test = "0.3.10"
-web-sys = {"version" = "0.3.4", "features" = [ 'console' ]}
+wasm-bindgen = "0.2"
+wasm-bindgen-test = "0.3"
+web-sys = { "version" = "0.3", "features" = ['console'] }
 
 # Run with cargo bench --bench iter_branch -- --baseline pathbuf-in-container --color never
 [[bench]]

root-io/README.md

@@ -2,18 +2,21 @@
 
 [![Crates.io Version](https://img.shields.io/crates/v/root-io.svg)](https://crates.io/crates/root-io)
 
-
 ## Documentation
+
 https://docs.rs/root-io
 
-`root-io` provides basic support for reading data stored in binary `.root` files commonly used in particle physics experiments. This crates provides:
+`root-io` provides basic support for reading data stored in binary `.root` files commonly used in particle physics
+experiments. This crates provides:
 
-  - Core types and parsers to read the layout description of custom classes contained in a given file
-  - Tools to generate `yaml` describing the streamed objects (aka. `TStreamerInfo`)
-  - Tools to generate (buggy) `Rust` code as a starting point for a new parser
-  - Set of types and parsers needed to read so-called `TTree`s
-
-The majority of the exposed API serves the latter point; striving to enable an easy iteration over data stored in `TTree`s. In particular, `root-io` supports reading `TBranches` (i.e. akin to "columns" of a database) with a variable number of elements in each entry (i.e. `TBranches` of `TClonesArray`).
+- Core types and parsers to read the layout description of custom classes contained in a given file
+- Tools to generate `yaml` describing the streamed objects (aka. `TStreamerInfo`)
+- Tools to generate (buggy) `Rust` code as a starting point for a new parser
+- Set of types and parsers needed to read so-called `TTree`s
+
+The majority of the exposed API serves the latter point; striving to enable an easy iteration over data stored
+in `TTree`s. In particular, `root-io` supports reading `TBranches` (i.e. akin to "columns" of a database) with a
+variable number of elements in each entry (i.e. `TBranches` of `TClonesArray`).
 
 The `root-ls` crate utilizes this crate to in a CLI to inspect a given root file and to deploy the code-gen tools.
 

root-io/benches/iter_branch.rs

@@ -3,10 +3,9 @@ extern crate criterion;
 extern crate nom;
 extern crate root_io;
 
-use nom::number::complete::{be_f32, be_i32, be_u32};
-
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use futures::StreamExt;
+use nom::number::complete::{be_f32, be_i32, be_u32};
 use tokio::runtime::Runtime;
 
 use root_io::RootFile;

root-io/src/core/compression.rs

@@ -0,0 +1,54 @@
+use flate2::bufread::ZlibDecoder;
+use lzma_rs::xz_decompress;
+use thiserror::Error;
+
+use std::io::Read;
+use std::*;
+use DecompressionError::*;
+
+#[derive(Error, Debug)]
+pub enum DecompressionError {
+    #[error("Header too short")]
+    InsufficientData,
+    #[error("Compression algorithm '{0}' not supported")]
+    AlgorithmNotImplemented(String),
+    #[error("Failed to decompress LZMA section")]
+    LzmaFailure(#[from] lzma_rs::error::Error),
+    #[error("Failed to decompress LZ4 section")]
+    Lz4Failure,
+    #[error("Failed to decompress ZLib section")]
+    ZLibFailure(#[from] std::io::Error),
+}
+
+pub(crate) fn decompress(input: &[u8]) -> Result<Vec<u8>, DecompressionError> {
+    if input.len() < 9 {
+        return Err(InsufficientData);
+    }
+
+    // There is something in bytes 2..=8, but we haven't identified it yet
+    let magic = &input[..2];
+    let compressed = &input[9..];
+
+    let mut ret = vec![];
+
+    match magic {
+        b"ZL" => {
+            let mut decoder = ZlibDecoder::new(compressed);
+            decoder.read_to_end(&mut ret)?;
+            Ok(ret)
+        }
+        b"XZ" => {
+            let mut reader = std::io::BufReader::new(compressed);
+            xz_decompress(&mut reader, &mut ret)?;
+            Ok(ret)
+        }
+        b"L4" => {
+            // TODO checksum verification?
+            // skip leading u64
+            lz4_compress::decompress(&compressed[8..]).map_err(|_| Lz4Failure)
+        }
+        other => Err(AlgorithmNotImplemented(
+            String::from_utf8(other.to_vec()).unwrap_or(format!("Bad magic {other:?}")),
+        )),
+    }
+}

root-io/src/core/data_source.rs

@@ -1,14 +1,15 @@
+use reqwest::{
+    header::{RANGE, USER_AGENT},
+    Client, Url,
+};
+
 use std::fs::File;
 use std::io::{Read, Seek, SeekFrom};
 #[cfg(not(target_arch = "wasm32"))]
 use std::path::Path;
 use std::path::PathBuf;
 
-use failure::Error;
-use reqwest::{
-    header::{RANGE, USER_AGENT},
-    Client, Url,
-};
+use crate::core::ReadError;
 
 /// The source from where the Root file is read. Construct it using
 /// `.into()` on a `Url` or `Path`. The latter is not availible for
@@ -33,7 +34,7 @@ impl Source {
         thing.into()
     }
 
-    pub async fn fetch(&self, start: u64, len: u64) -> Result<Vec<u8>, Error> {
+    pub async fn fetch(&self, start: u64, len: u64) -> Result<Vec<u8>, ReadError> {
         match &self.0 {
             SourceInner::Local(path) => {
                 let mut f = File::open(&path)?;