Iterative Approach

.github/workflows/ci.yml

@@ -69,8 +69,8 @@ jobs:
     # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
     strategy:
       matrix:
-        msrv: [1.62.1]
-    name: ${{ matrix.msrv }}
+        msrv: [1.60.0]
+    name: MSRV
     steps:
       - uses: actions/checkout@v3
       - name: Install ${{ matrix.msrv }}

CHANGELOG.md

@@ -8,6 +8,11 @@
   - `huggingface -> tokenizers`
   - `tiktoken -> tiktoken-rs`
 
+### Features
+
+- Moved from recursive approach to iterative approach to avoid stack overflow issues.
+- Relax MSRV to 1.60.0
+
 ## v0.2.2
 
 Add all features to docs.rs

Cargo.toml

@@ -9,7 +9,7 @@ license = "MIT"
 keywords = ["text", "split", "tokenizer", "nlp", "ai"]
 categories = ["text-processing"]
 exclude = ["/tests/snapshots/**", "/tests/inputs/**"]
-rust-version = "1.62.1"
+rust-version = "1.60.0"
 
 [package.metadata.docs.rs]
 all-features = true
@@ -18,6 +18,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+auto_enums = "0.8.0"
 either = "1.8.1"
 itertools = "0.10.5"
 once_cell = "1.17.1"
@@ -27,7 +28,7 @@ tokenizers = { version = "0.13.3", optional = true }
 unicode-segmentation = "1.10.1"
 
 [dev-dependencies]
-fake = "2.6.0"
+fake = "2.6.1"
 insta = { version = "1.29.0", features = ["glob", "yaml"] }
 more-asserts = "0.3.1"