Skip to content

Commit

Permalink
Merge pull request #1809 from mattheww/2022-05_searchindex
Browse files Browse the repository at this point in the history
Omit words longer than 80 characters from the search index
  • Loading branch information
Dylan-DPC authored Jun 22, 2022
2 parents 2f5e89f + 00a55b3 commit 93aee64
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 4 deletions.
13 changes: 12 additions & 1 deletion src/renderer/html_handlebars/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ use crate::utils;

use serde::Serialize;

const MAX_WORD_LENGTH_TO_INDEX: usize = 80;

/// Creates all files required for search.
pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
let mut index = Index::new(&["title", "body", "breadcrumbs"]);
Expand Down Expand Up @@ -44,6 +46,15 @@ pub fn create_files(search_config: &Search, destination: &Path, book: &Book) ->
Ok(())
}

/// Tokenizes in the same way as elasticlunr-rs (for English), but also drops long tokens.
fn tokenize(text: &str) -> Vec<String> {
text.split(|c: char| c.is_whitespace() || c == '-')
.filter(|s| !s.is_empty())
.map(|s| s.trim().to_lowercase())
.filter(|s| s.len() <= MAX_WORD_LENGTH_TO_INDEX)
.collect()
}

/// Uses the given arguments to construct a search document, then inserts it to the given index.
fn add_doc(
index: &mut Index,
Expand All @@ -62,7 +73,7 @@ fn add_doc(
doc_urls.push(url.into());

let items = items.iter().map(|&x| utils::collapse_whitespace(x.trim()));
index.add_doc(&doc_ref, items);
index.add_doc_with_tokenizer(&doc_ref, items, tokenize);
}

/// Renders markdown into flat unformatted text and adds it to the search index.
Expand Down
4 changes: 3 additions & 1 deletion tests/dummy_book/src/first/no-headers.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
Capybara capybara capybara.

Capybara capybara capybara.
Capybara capybara capybara.

ThisLongWordIsIncludedSoWeCanCheckThatSufficientlyLongWordsAreOmittedFromTheSearchIndex.
2 changes: 1 addition & 1 deletion tests/rendered_output.rs
Original file line number Diff line number Diff line change
Expand Up @@ -772,7 +772,7 @@ mod search {
);
assert_eq!(
docs[&no_headers]["body"],
"Capybara capybara capybara. Capybara capybara capybara."
"Capybara capybara capybara. Capybara capybara capybara. ThisLongWordIsIncludedSoWeCanCheckThatSufficientlyLongWordsAreOmittedFromTheSearchIndex."
);
}

Expand Down
2 changes: 1 addition & 1 deletion tests/searchindex_fixture.json
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@
"title": "Unicode stress tests"
},
"18": {
"body": "Capybara capybara capybara. Capybara capybara capybara.",
"body": "Capybara capybara capybara. Capybara capybara capybara. ThisLongWordIsIncludedSoWeCanCheckThatSufficientlyLongWordsAreOmittedFromTheSearchIndex.",
"breadcrumbs": "First Chapter » No Headers",
"id": "18",
"title": "First Chapter"
Expand Down

0 comments on commit 93aee64

Please sign in to comment.