Merge branch 'master' of https://github.com/quantleaf/probly-search

quantleaf · Jun 11, 2021 · a75a7f3 · a75a7f3
2 parents 41f9eb7 + da06d63
commit a75a7f3
Showing 1 changed file with 21 additions and 1 deletion.
diff --git a/src/index.rs b/src/index.rs
@@ -268,8 +268,8 @@ pub fn add_document_to_index<T: Eq + Hash + Copy, D>(
                 let mut filtered_terms_count = 0;
                 for mut term in terms {
                     term = filter(&term);
-                    all_terms.push(term.to_owned());
                     if term.as_str() != "" {
+                        all_terms.push(term.to_owned());
                         filtered_terms_count += 1;
                         let counts = term_counts.get_mut(&term);
                         match counts {
@@ -606,6 +606,26 @@ mod tests {
 
             // dont test all the properties
         }
+
+        #[test]
+        fn it_should_ignore_empty_tokens() {
+            let field_accessors: Vec<FieldAccessor<Doc>> =
+                vec![field_accessor as fn(doc: &Doc) -> Option<&str>];
+
+            let mut index = create_index::<usize>(1);
+            let doc_1 = Doc {
+                id: 1,
+                text: "a  b".to_string(), // double space could introduce empty tokens
+            };
+            add_document_to_index(
+                &mut index,
+                &field_accessors,
+                tokenizer,
+                filter,
+                doc_1.id,
+                doc_1.clone(),
+            );
+        }
     }
 
     mod delete {