Skip to content

Commit

Permalink
fix doc store cache docs
Browse files Browse the repository at this point in the history
addresses an issue reported in #1820
  • Loading branch information
PSeitz committed Jan 23, 2023
1 parent 2874554 commit bee3ac9
Show file tree
Hide file tree
Showing 5 changed files with 21 additions and 16 deletions.
4 changes: 2 additions & 2 deletions src/core/searcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ impl SearcherInner {
index: Index,
segment_readers: Vec<SegmentReader>,
generation: TrackedObject<SearcherGeneration>,
doc_store_cache_size: usize,
doc_store_cache_num_blocks: usize,
) -> io::Result<SearcherInner> {
assert_eq!(
&segment_readers
Expand All @@ -261,7 +261,7 @@ impl SearcherInner {
);
let store_readers: Vec<StoreReader> = segment_readers
.iter()
.map(|segment_reader| segment_reader.get_store_reader(doc_store_cache_size))
.map(|segment_reader| segment_reader.get_store_reader(doc_store_cache_num_blocks))
.collect::<io::Result<Vec<_>>>()?;

Ok(SearcherInner {
Expand Down
9 changes: 6 additions & 3 deletions src/core/segment_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,12 @@ impl SegmentReader {
&self.fieldnorm_readers
}

/// Accessor to the segment's `StoreReader`.
pub fn get_store_reader(&self, cache_size: usize) -> io::Result<StoreReader> {
StoreReader::open(self.store_file.clone(), cache_size)
/// Accessor to the segment's [`StoreReader`](crate::store::StoreReader).
///
/// `cache_num_blocks` sets the number of decompressed blocks to be cached in an LRU.
/// The size of blocks is configurable, this should be reflexted in the
pub fn get_store_reader(&self, cache_num_blocks: usize) -> io::Result<StoreReader> {
StoreReader::open(self.store_file.clone(), cache_num_blocks)
}

/// Open a new segment for reading.
Expand Down
10 changes: 5 additions & 5 deletions src/reader/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ impl TryInto<IndexReader> for IndexReaderBuilder {
}

struct InnerIndexReader {
doc_store_cache_size: usize,
doc_store_cache_num_blocks: usize,
index: Index,
warming_state: WarmingState,
searcher: arc_swap::ArcSwap<SearcherInner>,
Expand All @@ -178,7 +178,7 @@ impl InnerIndexReader {
&searcher_generation_inventory,
)?;
Ok(InnerIndexReader {
doc_store_cache_size,
doc_store_cache_num_blocks: doc_store_cache_size,
index,
warming_state,
searcher: ArcSwap::from(searcher),
Expand Down Expand Up @@ -214,7 +214,7 @@ impl InnerIndexReader {

fn create_searcher(
index: &Index,
doc_store_cache_size: usize,
doc_store_cache_num_blocks: usize,
warming_state: &WarmingState,
searcher_generation_counter: &Arc<AtomicU64>,
searcher_generation_inventory: &Inventory<SearcherGeneration>,
Expand All @@ -232,7 +232,7 @@ impl InnerIndexReader {
index.clone(),
segment_readers,
searcher_generation,
doc_store_cache_size,
doc_store_cache_num_blocks,
)?);

warming_state.warm_new_searcher_generation(&searcher.clone().into())?;
Expand All @@ -242,7 +242,7 @@ impl InnerIndexReader {
fn reload(&self) -> crate::Result<()> {
let searcher = Self::create_searcher(
&self.index,
self.doc_store_cache_size,
self.doc_store_cache_num_blocks,
&self.warming_state,
&self.searcher_generation_counter,
&self.searcher_generation_inventory,
Expand Down
5 changes: 2 additions & 3 deletions src/store/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
//! order to be handled in the `Store`.
//!
//! Internally, documents (or rather their stored fields) are serialized to a buffer.
//! When the buffer exceeds 16K, the buffer is compressed using `brotli`, `LZ4` or `snappy`
//! and the resulting block is written to disk.
//! When the buffer exceeds `block_size` (defaults to 16K), the buffer is compressed using `brotli`,
//! `LZ4` or `snappy` and the resulting block is written to disk.
//!
//! One can then request for a specific `DocId`.
//! A skip list helps navigating to the right block,
Expand All @@ -29,7 +29,6 @@
//! [`SegmentReader`'s `doc` method](../struct.SegmentReader.html#method.doc)
//! - at the index level, the [`Searcher::doc()`](crate::Searcher::doc) method
//!
//! !

mod compressors;
mod decompressors;
Expand Down
9 changes: 6 additions & 3 deletions src/store/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,10 @@ impl Sum for CacheStats {

impl StoreReader {
/// Opens a store reader
pub fn open(store_file: FileSlice, cache_size: usize) -> io::Result<StoreReader> {
///
/// `cache_num_blocks` sets the number of decompressed blocks to be cached in an LRU.
/// The size of blocks is configurable, this should be reflexted in the
pub fn open(store_file: FileSlice, cache_num_blocks: usize) -> io::Result<StoreReader> {
let (footer, data_and_offset) = DocStoreFooter::extract_footer(store_file)?;

let (data_file, offset_index_file) = data_and_offset.split(footer.offset as usize);
Expand All @@ -125,8 +128,8 @@ impl StoreReader {
decompressor: footer.decompressor,
data: data_file,
cache: BlockCache {
cache: NonZeroUsize::new(cache_size)
.map(|cache_size| Mutex::new(LruCache::new(cache_size))),
cache: NonZeroUsize::new(cache_num_blocks)
.map(|cache_num_blocks| Mutex::new(LruCache::new(cache_num_blocks))),
cache_hits: Default::default(),
cache_misses: Default::default(),
},
Expand Down

0 comments on commit bee3ac9

Please sign in to comment.