Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bump pyo3 from 0.20.3 to 0.21.0 #125

Merged
merged 3 commits into from
Mar 26, 2024
Merged

Bump pyo3 from 0.20.3 to 0.21.0 #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c9b2e03
Bump pyo3 from 0.20.3 to 0.21.0
dependabot[bot] Mar 26, 2024
1b6e193
Updates for v0.21
benbrandt Mar 26, 2024
ab3372e
Change order of flags
benbrandt Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/python.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
shell: bash
run: |
set -e
pip install semantic-text-splitter --find-links dist --force-reinstall
pip install --find-links dist --force-reinstall semantic-text-splitter
pip install pytest tokenizers
pytest

Expand Down Expand Up @@ -96,7 +96,7 @@ jobs:
shell: bash
run: |
set -e
pip install semantic-text-splitter --find-links dist --force-reinstall
pip install --find-links dist --force-reinstall semantic-text-splitter
pip install pytest tokenizers
pytest

Expand Down Expand Up @@ -132,7 +132,7 @@ jobs:
shell: bash
run: |
set -e
pip install semantic-text-splitter --find-links dist --force-reinstall
pip install --find-links dist --force-reinstall semantic-text-splitter
pip install pytest tokenizers
pytest

Expand Down
20 changes: 10 additions & 10 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion bindings/python/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ name = "semantic_text_splitter"
crate-type = ["cdylib"]

[dependencies]
pyo3 = { version = "0.20.3", features = ["abi3-py38"] }
pyo3 = { version = "0.21.0", features = ["abi3-py38"] }
text-splitter = { path = "../..", features = [
"markdown",
"tiktoken-rs",
Expand Down
24 changes: 15 additions & 9 deletions bindings/python/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

use std::str::FromStr;

use pyo3::{exceptions::PyException, prelude::*};
use pyo3::{exceptions::PyException, prelude::*, pybacked::PyBackedStr};
use text_splitter::{
Characters, ChunkCapacity, ChunkSize, ChunkSizer, MarkdownSplitter, TextSplitter,
};
Expand Down Expand Up @@ -57,7 +57,7 @@ impl ChunkSizer for CustomCallback {
Python::with_gil(|py| {
let size = self
.0
.call(py, (chunk,), None)
.call_bound(py, (chunk,), None)
.unwrap()
.extract::<usize>(py)
.unwrap();
Expand Down Expand Up @@ -198,11 +198,14 @@ impl PyTextSplitter {
*/
#[staticmethod]
#[pyo3(signature = (tokenizer, trim_chunks=true))]
fn from_huggingface_tokenizer(tokenizer: &PyAny, trim_chunks: bool) -> PyResult<Self> {
fn from_huggingface_tokenizer(
tokenizer: &Bound<'_, PyAny>,
trim_chunks: bool,
) -> PyResult<Self> {
// Get the json out so we can reconstruct the tokenizer on the Rust side
let json = tokenizer.call_method0("to_str")?.extract::<&str>()?;
let json = tokenizer.call_method0("to_str")?.extract::<PyBackedStr>()?;
let tokenizer =
Tokenizer::from_str(json).map_err(|e| PyException::new_err(format!("{e}")))?;
Tokenizer::from_str(&json).map_err(|e| PyException::new_err(format!("{e}")))?;

Ok(Self {
splitter: TextSplitterOptions::Huggingface(
Expand Down Expand Up @@ -489,11 +492,14 @@ impl PyMarkdownSplitter {
*/
#[staticmethod]
#[pyo3(signature = (tokenizer, trim_chunks=true))]
fn from_huggingface_tokenizer(tokenizer: &PyAny, trim_chunks: bool) -> PyResult<Self> {
fn from_huggingface_tokenizer(
tokenizer: &Bound<'_, PyAny>,
trim_chunks: bool,
) -> PyResult<Self> {
// Get the json out so we can reconstruct the tokenizer on the Rust side
let json = tokenizer.call_method0("to_str")?.extract::<&str>()?;
let json = tokenizer.call_method0("to_str")?.extract::<PyBackedStr>()?;
let tokenizer =
Tokenizer::from_str(json).map_err(|e| PyException::new_err(format!("{e}")))?;
Tokenizer::from_str(&json).map_err(|e| PyException::new_err(format!("{e}")))?;

Ok(Self {
splitter: MarkdownSplitterOptions::Huggingface(
Expand Down Expand Up @@ -665,7 +671,7 @@ impl PyMarkdownSplitter {

#[doc = include_str!("../README.md")]
#[pymodule]
fn semantic_text_splitter(_py: Python, m: &PyModule) -> PyResult<()> {
fn semantic_text_splitter(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyTextSplitter>()?;
m.add_class::<PyMarkdownSplitter>()?;
Ok(())
Expand Down
Loading