forked from quickwit-oss/tantivy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
mod.rs
212 lines (180 loc) · 6.86 KB
/
mod.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
//! The term dictionary main role is to associate the sorted [`Term`s](crate::Term) to
//! a [`TermInfo`](crate::postings::TermInfo) struct that contains some meta-information
//! about the term.
//!
//! Internally, the term dictionary relies on the `fst` crate to store
//! a sorted mapping that associate each term to its rank in the lexicographical order.
//! For instance, in a dictionary containing the sorted terms "abba", "bjork", "blur" and "donovan",
//! the [`TermOrdinal`] are respectively `0`, `1`, `2`, and `3`.
//!
//! For `u64`-terms, tantivy explicitly uses a `BigEndian` representation to ensure that the
//! lexicographical order matches the natural order of integers.
//!
//! `i64`-terms are transformed to `u64` using a continuous mapping `val ⟶ val - i64::MIN`
//! and then treated as a `u64`.
//!
//! `f64`-terms are transformed to `u64` using a mapping that preserve order, and are then treated
//! as `u64`.
//!
//! A second datastructure makes it possible to access a
//! [`TermInfo`](crate::postings::TermInfo).
#[cfg(not(feature = "quickwit"))]
mod fst_termdict;
#[cfg(not(feature = "quickwit"))]
use fst_termdict as termdict;
#[cfg(feature = "quickwit")]
mod sstable_termdict;
#[cfg(feature = "quickwit")]
use sstable_termdict as termdict;
#[cfg(test)]
mod tests;
/// Position of the term in the sorted list of terms.
pub type TermOrdinal = u64;
use std::io;
use common::file_slice::FileSlice;
use common::BinarySerializable;
use tantivy_fst::Automaton;
use self::termdict::{
TermDictionary as InnerTermDict, TermDictionaryBuilder as InnerTermDictBuilder,
TermStreamerBuilder,
};
pub use self::termdict::{TermMerger, TermStreamer};
use crate::postings::TermInfo;
#[repr(u32)]
#[allow(dead_code)]
enum DictionaryType {
Fst = 1,
SSTable = 2,
}
#[cfg(not(feature = "quickwit"))]
const CURRENT_TYPE: DictionaryType = DictionaryType::Fst;
#[cfg(feature = "quickwit")]
const CURRENT_TYPE: DictionaryType = DictionaryType::SSTable;
// TODO in the future this should become an enum of supported dictionaries
/// A TermDictionary wrapping either an FST based dictionary or a SSTable based one.
pub struct TermDictionary(InnerTermDict);
impl TermDictionary {
/// Opens a `TermDictionary`.
pub fn open(file: FileSlice) -> io::Result<Self> {
let (main_slice, dict_type) = file.split_from_end(4);
let mut dict_type = dict_type.read_bytes()?;
let dict_type = u32::deserialize(&mut dict_type)?;
if dict_type != CURRENT_TYPE as u32 {
return Err(io::Error::new(
io::ErrorKind::Other,
format!(
"Unsuported dictionary type, expected {}, found {dict_type}",
CURRENT_TYPE as u32,
),
));
}
InnerTermDict::open(main_slice).map(TermDictionary)
}
/// Creates an empty term dictionary which contains no terms.
pub fn empty() -> Self {
TermDictionary(InnerTermDict::empty())
}
/// Returns the number of terms in the dictionary.
/// Term ordinals range from 0 to `num_terms() - 1`.
pub fn num_terms(&self) -> usize {
self.0.num_terms()
}
/// Returns the ordinal associated with a given term.
pub fn term_ord<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TermOrdinal>> {
self.0.term_ord(key)
}
/// Stores the term associated with a given term ordinal in
/// a `bytes` buffer.
///
/// Term ordinals are defined as the position of the term in
/// the sorted list of terms.
///
/// Returns true if and only if the term has been found.
///
/// Regardless of whether the term is found or not,
/// the buffer may be modified.
pub fn ord_to_term(&self, ord: TermOrdinal, bytes: &mut Vec<u8>) -> io::Result<bool> {
self.0.ord_to_term(ord, bytes)
}
// this isn't used, and has different prototype in Fst and SSTable
// Returns the number of terms in the dictionary.
// pub fn term_info_from_ord(&self, term_ord: TermOrdinal) -> TermInfo {
// self.0.term_info_from_ord(term_ord)
// }
/// Lookups the value corresponding to the key.
pub fn get<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TermInfo>> {
self.0.get(key)
}
/// Returns a range builder, to stream all of the terms
/// within an interval.
pub fn range(&self) -> TermStreamerBuilder<'_> {
self.0.range()
}
/// A stream of all the sorted terms.
pub fn stream(&self) -> io::Result<TermStreamer<'_>> {
self.0.stream()
}
/// Returns a search builder, to stream all of the terms
/// within the Automaton
pub fn search<'a, A: Automaton + 'a>(&'a self, automaton: A) -> TermStreamerBuilder<'a, A>
where A::State: Clone {
self.0.search(automaton)
}
#[cfg(feature = "quickwit")]
/// Lookups the value corresponding to the key.
pub async fn get_async<K: AsRef<[u8]>>(&self, key: K) -> io::Result<Option<TermInfo>> {
self.0.get_async(key).await
}
#[cfg(feature = "quickwit")]
#[doc(hidden)]
pub async fn warm_up_dictionary(&self) -> io::Result<()> {
self.0.warm_up_dictionary().await
}
#[cfg(feature = "quickwit")]
/// Returns a file slice covering a set of sstable blocks
/// that includes the key range passed in arguments.
pub fn file_slice_for_range(
&self,
key_range: impl std::ops::RangeBounds<[u8]>,
limit: Option<u64>,
) -> FileSlice {
self.0.file_slice_for_range(key_range, limit)
}
}
/// A TermDictionaryBuilder wrapping either an FST or a SSTable dictionary builder.
pub struct TermDictionaryBuilder<W: io::Write>(InnerTermDictBuilder<W>);
impl<W: io::Write> TermDictionaryBuilder<W> {
/// Creates a new `TermDictionaryBuilder`
pub fn create(w: W) -> io::Result<Self> {
InnerTermDictBuilder::create(w).map(TermDictionaryBuilder)
}
/// Inserts a `(key, value)` pair in the term dictionary.
///
/// *Keys have to be inserted in order.*
pub fn insert<K: AsRef<[u8]>>(&mut self, key_ref: K, value: &TermInfo) -> io::Result<()> {
self.0.insert(key_ref, value)
}
/// # Warning
/// Horribly dangerous internal API
///
/// If used, it must be used by systematically alternating calls
/// to insert_key and insert_value.
///
/// Prefer using `.insert(key, value)`
pub fn insert_key(&mut self, key: &[u8]) -> io::Result<()> {
self.0.insert_key(key)
}
/// # Warning
///
/// Horribly dangerous internal API. See `.insert_key(...)`.
pub fn insert_value(&mut self, term_info: &TermInfo) -> io::Result<()> {
self.0.insert_value(term_info)
}
/// Finalize writing the builder, and returns the underlying
/// `Write` object.
pub fn finish(self) -> io::Result<W> {
let mut writer = self.0.finish()?;
(CURRENT_TYPE as u32).serialize(&mut writer)?;
Ok(writer)
}
}