Update tokenizers to v0.21

benbrandt · Nov 28, 2024 · 62ff28e · 62ff28e
1 parent 9b3c4c5
commit 62ff28e
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 43 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -71,7 +71,7 @@ rust_tokenizers = { version = "8", optional = true }
 strum = { version = "0.26", features = ["derive"] }
 thiserror = "2.0.3"
 tiktoken-rs = { version = "0.6", optional = true }
-tokenizers = { version = "0.20", default-features = false, optional = true }
+tokenizers = { version = "0.21", default-features = false, optional = true }
 tree-sitter = { version = "0.24", optional = true }
 unicode-segmentation = "1.12"
 
@@ -85,7 +85,7 @@ fake = "3"
 insta = { version = "1.41", features = ["glob", "yaml"] }
 more-asserts = "0.3"
 rayon = "1.10"
-tokenizers = { version = "0.20", default-features = false, features = [
+tokenizers = { version = "0.21", default-features = false, features = [
     "onig",
     "http",
 ] }

diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -23,7 +23,7 @@ text-splitter = { path = "../..", features = [
     "tokenizers",
 ] }
 tiktoken-rs = "0.6"
-tokenizers = { version = "0.20", default-features = false, features = ["onig"] }
+tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
 tree-sitter = "0.24"
 
 [lints]

diff --git a/clippy.toml b/clippy.toml
@@ -4,6 +4,7 @@ allowed-duplicate-crates = [
     "itertools",
     "thiserror",
     "thiserror-impl",
+    "unicode-width",
     "windows-sys",
     "windows-targets",
     "windows_aarch64_gnullvm",