CR comments.

Rename deserialize_u64 -> deserialize_vint_u64

sstable/README.md

@@ -0,0 +1,28 @@
+# SSTable
+
+The `tantivy-sstable` crate is yet another sstable crate.
+
+It has been designed to be used in `quickwit`:
+- as an alternative to the default tantivy fst dictionary.
+- as a way to store the column index for dynamic fast fields.
+
+The benefit compared to the fst crate is locality.
+Searching a key in the fst crate requires downloading the entire dictionary.
+
+Once the sstable index is downloaded, running a `get` in the sstable
+crate only requires a single fetch.
+
+Right now, the block index and the default block size have been thought
+for quickwit, and the performance of a get is very bad.
+
+# Sorted strings?
+
+SSTable stands for Sorted String Table.
+Strings have to be insert in sorted order.
+
+That sorted order is used in different ways:
+- it makes gets and streaming ranges of keys
+possible.
+- it allows incremental encoding of the keys
+- the front compression is leveraged to optimize
+the intersection with an automaton

sstable/src/dictionary.rs

@@ -11,12 +11,25 @@ use tantivy_fst::Automaton;
 use crate::streamer::{Streamer, StreamerBuilder};
 use crate::{BlockAddr, DeltaReader, Reader, SSTable, SSTableIndex, TermOrdinal};
 
-/// The term dictionary contains all of the terms in
-/// `tantivy index` in a sorted manner.
+/// An SSTable is a sorted map that associates sorted `&[u8]` keys
+/// to any kind of typed values.
 ///
-/// The `Fst` crate is used to associate terms to their
-/// respective `TermOrdinal`. The `TermInfoStore` then makes it
-/// possible to fetch the associated `TermInfo`.
+/// The SSTable is organized in blocks.
+/// In each block, keys and values are encoded separately.
+///
+/// The keys are encoded using incremental encoding.
+/// The values on the other hand, are encoded according to a value-specific
+/// codec defined in the TSSTable generic argument.
+///
+/// Finally, an index is joined to the Dictionary to make it possible,
+/// given a key to identify which block contains this key.
+///
+/// The codec was designed in such a way that the sstable
+/// reader is not aware of block, and yet can read any sequence of blocks,
+/// as long as the slice of bytes it is given starts and stops at
+/// block boundary.
+///
+/// (See also README.md)
 pub struct Dictionary<TSSTable: SSTable> {
     pub sstable_slice: FileSlice,
     pub sstable_index: SSTableIndex,
@@ -62,6 +75,23 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
         Ok(TSSTable::delta_reader(data))
     }
 
+    /// This function returns a file slice covering a set of sstable blocks
+    /// that include the key range passed in arguments.
+    ///
+    /// It works by identifying
+    /// - `first_block`: the block containing the start boudary key
+    /// - `last_block`: the block containing the end boundary key.
+    ///
+    /// And then returning the range that spans over all blocks between.
+    /// and including first_block and last_block, aka:
+    /// `[first_block.start_offset .. last_block.end_offset)`
+    ///
+    /// Technically this function does not provide the tightest fit, as
+    /// for simplification, it treats the start bound of the `key_range`
+    /// as if it was inclusive, even if it is exclusive.
+    /// On the rare edge case where a user asks for `(start_key, end_key]`
+    /// and `start_key` happens to be the last key of a block, we return a
+    /// slice that is the first block was not necessary.
     fn file_slice_for_range(&self, key_range: impl RangeBounds<[u8]>) -> FileSlice {
         let start_bound: Bound<usize> = match key_range.start_bound() {
             Bound::Included(key) | Bound::Excluded(key) => {

sstable/src/value/mod.rs

@@ -26,7 +26,7 @@ pub trait ValueWriter: Default {
 
     /// Records a new value.
     /// This method usually just accumulates data in a `Vec`,
-    /// only to be serialized on the call to `ValueWriter::write_block`.
+    /// only to be serialized on the call to `ValueWriter::serialize_block`.
     fn write(&mut self, val: &Self::Value);
 
     /// Serializes the accumulated values into the output buffer.
@@ -37,7 +37,7 @@ pub use range::{RangeReader, RangeWriter};
 pub use u64_monotonic::{U64MonotonicReader, U64MonotonicWriter};
 pub use void::{VoidReader, VoidWriter};
 
-fn deserialize_u64(data: &mut &[u8]) -> u64 {
+fn deserialize_vint_u64(data: &mut &[u8]) -> u64 {
     let (num_bytes, val) = super::vint::deserialize_read(data);
     *data = &data[num_bytes..];
     val

sstable/src/value/range.rs

@@ -1,7 +1,7 @@
 use std::io;
 use std::ops::Range;
 
-use crate::value::{deserialize_u64, ValueReader, ValueWriter};
+use crate::value::{deserialize_vint_u64, ValueReader, ValueWriter};
 
 #[derive(Default)]
 pub struct RangeReader {
@@ -18,11 +18,11 @@ impl ValueReader for RangeReader {
     fn load(&mut self, mut data: &[u8]) -> io::Result<usize> {
         self.vals.clear();
         let original_num_bytes = data.len();
-        let len = deserialize_u64(&mut data) as usize;
+        let len = deserialize_vint_u64(&mut data) as usize;
         if len != 0 {
-            let mut prev_val = deserialize_u64(&mut data);
+            let mut prev_val = deserialize_vint_u64(&mut data);
             for _ in 1..len {
-                let next_val = prev_val + deserialize_u64(&mut data);
+                let next_val = prev_val + deserialize_vint_u64(&mut data);
                 self.vals.push(prev_val..next_val);
                 prev_val = next_val;
             }

sstable/src/value/u64_monotonic.rs

@@ -1,6 +1,6 @@
 use std::io;
 
-use crate::value::{deserialize_u64, ValueReader, ValueWriter};
+use crate::value::{deserialize_vint_u64, ValueReader, ValueWriter};
 use crate::vint;
 
 #[derive(Default)]
@@ -17,11 +17,11 @@ impl ValueReader for U64MonotonicReader {
 
     fn load(&mut self, mut data: &[u8]) -> io::Result<usize> {
         let original_num_bytes = data.len();
-        let num_vals = deserialize_u64(&mut data) as usize;
+        let num_vals = deserialize_vint_u64(&mut data) as usize;
         self.vals.clear();
         let mut prev_val = 0u64;
         for _ in 0..num_vals {
-            let delta = deserialize_u64(&mut data);
+            let delta = deserialize_vint_u64(&mut data);
             let val = prev_val + delta;
             self.vals.push(val);
             prev_val = val;