Skip to content

Commit

Permalink
fix lint
Browse files Browse the repository at this point in the history
  • Loading branch information
wsxiaoys committed Aug 28, 2023
1 parent 7ecd737 commit 4e6cca5
Showing 1 changed file with 12 additions and 4 deletions.
16 changes: 12 additions & 4 deletions crates/ctranslate2-bindings/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ pub struct CTranslate2Engine {
engine: cxx::SharedPtr<ffi::TextInferenceEngine>,
tokenizer: Tokenizer,
stop_regex_cache: DashMap<&'static Vec<&'static str>, Regex>,
stop_words_encoding_offset: Option<usize>
stop_words_encoding_offset: Option<usize>,
}

impl CTranslate2Engine {
Expand All @@ -105,7 +105,7 @@ impl CTranslate2Engine {
engine,
stop_regex_cache: DashMap::new(),
tokenizer: Tokenizer::from_file(&options.tokenizer_path).unwrap(),
stop_words_encoding_offset: options.stop_words_encoding_offset
stop_words_encoding_offset: options.stop_words_encoding_offset,
};
}
}
Expand All @@ -127,7 +127,11 @@ impl TextGeneration for CTranslate2Engine {
if re.is_none() {
self.stop_regex_cache.insert(
options.stop_words,
create_stop_regex(&self.tokenizer, options.stop_words, self.stop_words_encoding_offset),
create_stop_regex(
&self.tokenizer,
options.stop_words,
self.stop_words_encoding_offset,
),
);
re = self.stop_regex_cache.get(options.stop_words);
}
Expand Down Expand Up @@ -182,7 +186,11 @@ fn reverse(s: String) -> String {
}
}

fn create_stop_regex(tokenizer: &Tokenizer, stop_words: &[&str], stop_words_encoding_offset: Option<usize>) -> Regex {
fn create_stop_regex(
tokenizer: &Tokenizer,
stop_words: &[&str],
stop_words_encoding_offset: Option<usize>,
) -> Regex {
let encodings = tokenizer
.encode_batch(stop_words.to_owned(), false)
.unwrap();
Expand Down

0 comments on commit 4e6cca5

Please sign in to comment.