Skip to content

Commit

Permalink
Hiding useless symbols and removing code. (#1522)
Browse files Browse the repository at this point in the history
  • Loading branch information
fulmicoton authored Sep 16, 2022
1 parent e029fdf commit 64f08a1
Show file tree
Hide file tree
Showing 6 changed files with 39 additions and 62 deletions.
21 changes: 7 additions & 14 deletions fastfield_codecs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,21 @@ use serialize::Header;

mod bitpacked;
mod blockwise_linear;
pub(crate) mod line;
mod line;
mod linear;
mod monotonic_mapping;

mod column;
mod gcd;
mod serialize;

pub use self::bitpacked::BitpackedCodec;
pub use self::blockwise_linear::BlockwiseLinearCodec;
use self::bitpacked::BitpackedCodec;
use self::blockwise_linear::BlockwiseLinearCodec;
pub use self::column::{monotonic_map_column, Column, VecColumn};
pub use self::linear::LinearCodec;
use self::linear::LinearCodec;
pub use self::monotonic_mapping::MonotonicallyMappableToU64;
pub use self::serialize::{estimate, serialize, serialize_and_load, NormalizedHeader};
use self::serialize::NormalizedHeader;
pub use self::serialize::{estimate, serialize, serialize_and_load};

#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
#[repr(u8)]
Expand Down Expand Up @@ -100,7 +101,7 @@ fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(

/// The FastFieldSerializerEstimate trait is required on all variants
/// of fast field compressions, to decide which one to choose.
pub trait FastFieldCodec: 'static {
trait FastFieldCodec: 'static {
/// A codex needs to provide a unique name and id, which is
/// used for debugging and de/serialization.
const CODEC_TYPE: FastFieldCodecType;
Expand Down Expand Up @@ -132,14 +133,6 @@ pub const ALL_CODEC_TYPES: [FastFieldCodecType; 3] = [
FastFieldCodecType::Linear,
];

#[derive(Debug, Clone)]
/// Statistics are used in codec detection and stored in the fast field footer.
pub struct FastFieldStats {
pub min_value: u64,
pub max_value: u64,
pub num_vals: u64,
}

#[cfg(test)]
mod tests {
use proptest::prelude::*;
Expand Down
12 changes: 1 addition & 11 deletions fastfield_codecs/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[macro_use]
extern crate prettytable;
use fastfield_codecs::{Column, FastFieldCodecType, FastFieldStats, VecColumn};
use fastfield_codecs::{Column, FastFieldCodecType, VecColumn};
use prettytable::{Cell, Row, Table};

fn main() {
Expand Down Expand Up @@ -90,13 +90,3 @@ pub fn serialize_with_codec(
let actual_compression = out.len() as f32 / (col.num_vals() * 8) as f32;
Some((estimation, actual_compression, codec_type))
}

pub fn stats_from_vec(data: &[u64]) -> FastFieldStats {
let min_value = data.iter().cloned().min().unwrap_or(0);
let max_value = data.iter().cloned().max().unwrap_or(0);
FastFieldStats {
min_value,
max_value,
num_vals: data.len() as u64,
}
}
2 changes: 1 addition & 1 deletion src/fastfield/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ pub(crate) use self::multivalued::MultivalueStartIndex;
pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
pub use self::readers::FastFieldReaders;
pub(crate) use self::readers::{type_and_cardinality, FastType};
pub use self::serializer::{Column, CompositeFastFieldSerializer, FastFieldStats};
pub use self::serializer::{Column, CompositeFastFieldSerializer};
pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
use crate::schema::{Cardinality, FieldType, Type, Value};
use crate::{DateTime, DocId};
Expand Down
2 changes: 1 addition & 1 deletion src/fastfield/serializer/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::io::{self, Write};

pub use fastfield_codecs::{Column, FastFieldStats};
pub use fastfield_codecs::Column;
use fastfield_codecs::{FastFieldCodecType, MonotonicallyMappableToU64, ALL_CODEC_TYPES};

use crate::directory::{CompositeWrite, WritePtr};
Expand Down
22 changes: 10 additions & 12 deletions src/fastfield/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ use fnv::FnvHashMap;
use tantivy_bitpacker::BlockedBitpacker;

use super::multivalued::MultiValuedFastFieldWriter;
use super::serializer::FastFieldStats;
use super::FastFieldType;
use crate::fastfield::{BytesFastFieldWriter, CompositeFastFieldSerializer};
use crate::indexer::doc_id_mapping::DocIdMapping;
Expand Down Expand Up @@ -360,16 +359,12 @@ impl IntFastFieldWriter {
(self.val_min, self.val_max)
};

let stats = FastFieldStats {
min_value: min,
max_value: max,
num_vals: self.val_count as u64,
};

let fastfield_accessor = WriterFastFieldAccessProvider {
doc_id_map,
vals: &self.vals,
stats,
min_value: min,
max_value: max,
num_vals: self.val_count as u64,
};

serializer.create_auto_detect_u64_fast_field(self.field, fastfield_accessor)?;
Expand All @@ -382,8 +377,11 @@ impl IntFastFieldWriter {
struct WriterFastFieldAccessProvider<'map, 'bitp> {
doc_id_map: Option<&'map DocIdMapping>,
vals: &'bitp BlockedBitpacker,
stats: FastFieldStats,
min_value: u64,
max_value: u64,
num_vals: u64,
}

impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
/// Return the value associated to the given doc.
///
Expand Down Expand Up @@ -417,14 +415,14 @@ impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
}

fn min_value(&self) -> u64 {
self.stats.min_value
self.min_value
}

fn max_value(&self) -> u64 {
self.stats.max_value
self.max_value
}

fn num_vals(&self) -> u64 {
self.stats.num_vals
self.num_vals
}
}
42 changes: 19 additions & 23 deletions src/indexer/merger.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ use crate::core::{Segment, SegmentReader};
use crate::docset::{DocSet, TERMINATED};
use crate::error::DataCorruption;
use crate::fastfield::{
AliveBitSet, Column, CompositeFastFieldSerializer, FastFieldStats, MultiValueLength,
MultiValuedFastFieldReader,
AliveBitSet, Column, CompositeFastFieldSerializer, MultiValueLength, MultiValuedFastFieldReader,
};
use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
use crate::indexer::doc_id_mapping::{expect_field_id_for_sort_field, SegmentDocIdMapping};
Expand Down Expand Up @@ -359,16 +358,13 @@ impl IndexMerger {
})
.collect::<Vec<_>>();

let stats = FastFieldStats {
min_value,
max_value,
num_vals: doc_id_mapping.len() as u64,
};
#[derive(Clone)]
struct SortedDocIdFieldAccessProvider<'a> {
doc_id_mapping: &'a SegmentDocIdMapping,
fast_field_readers: &'a Vec<Arc<dyn Column<u64>>>,
stats: FastFieldStats,
min_value: u64,
max_value: u64,
num_vals: u64,
}
impl<'a> Column for SortedDocIdFieldAccessProvider<'a> {
fn get_val(&self, doc: u64) -> u64 {
Expand All @@ -391,21 +387,23 @@ impl IndexMerger {
)
}
fn min_value(&self) -> u64 {
self.stats.min_value
self.min_value
}

fn max_value(&self) -> u64 {
self.stats.max_value
self.max_value
}

fn num_vals(&self) -> u64 {
self.stats.num_vals
self.num_vals
}
}
let fastfield_accessor = SortedDocIdFieldAccessProvider {
doc_id_mapping,
fast_field_readers: &fast_field_readers,
stats,
min_value,
max_value,
num_vals: doc_id_mapping.len() as u64,
};
fast_field_serializer.create_auto_detect_u64_fast_field(field, fastfield_accessor)?;

Expand Down Expand Up @@ -705,17 +703,13 @@ impl IndexMerger {
}

// We can now initialize our serializer, and push it the different values
let stats = FastFieldStats {
max_value,
num_vals: num_vals as u64,
min_value,
};

struct SortedDocIdMultiValueAccessProvider<'a> {
doc_id_mapping: &'a SegmentDocIdMapping,
fast_field_readers: &'a Vec<MultiValuedFastFieldReader<u64>>,
offsets: Vec<u64>,
stats: FastFieldStats,
min_value: u64,
max_value: u64,
num_vals: u64,
}
impl<'a> Column for SortedDocIdMultiValueAccessProvider<'a> {
fn get_val(&self, pos: u64) -> u64 {
Expand Down Expand Up @@ -757,22 +751,24 @@ impl IndexMerger {
)
}
fn min_value(&self) -> u64 {
self.stats.min_value
self.min_value
}

fn max_value(&self) -> u64 {
self.stats.max_value
self.max_value
}

fn num_vals(&self) -> u64 {
self.stats.num_vals
self.num_vals
}
}
let fastfield_accessor = SortedDocIdMultiValueAccessProvider {
doc_id_mapping,
fast_field_readers: &ff_readers,
offsets,
stats,
min_value,
max_value,
num_vals: num_vals as u64,
};
fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(
field,
Expand Down

0 comments on commit 64f08a1

Please sign in to comment.