Skip to content

Commit

Permalink
auto-initialize: new feature to control initializing Python
Browse files Browse the repository at this point in the history
  • Loading branch information
davidhewitt committed Jan 1, 2021
1 parent 42ca48a commit 5eb33f7
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 49 deletions.
15 changes: 8 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,28 @@ jobs:
run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV

- name: Build docs
run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}

- name: Build without default features
- name: Build (no features)
run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}

- name: Build with default features
run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
- name: Build (all additive features)
run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}

# Run tests (except on PyPy, because no embedding API).
- if: matrix.python-version != 'pypy-3.6'
name: Test
run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

# Run tests again, but in abi3 mode
- if: matrix.python-version != 'pypy-3.6'
name: Test (abi3)
run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

# Run tests again, for abi3-py36 (the minimal Python version)
- if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
name: Test (abi3-py36)
run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

- name: Test proc-macro code
run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }}
Expand Down
24 changes: 17 additions & 7 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,36 @@ assert_approx_eq = "1.1.0"
trybuild = "1.0.23"
rustversion = "1.0"
proptest = { version = "0.10.1", default-features = false, features = ["std"] }
# features needed to run the PyO3 test suite
pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }

[features]
default = ["macros"]
macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"]
default = ["macros", "auto-initialize"]

# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]

# Use this feature when building an extension module.
# It tells the linker to keep the python symbols unresolved,
# so that the module can also be used with statically linked python interpreters.
extension-module = []

# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
abi3 = []

# With abi3, we can manually set the minimum Python version.
abi3-py36 = ["abi3-py37"]
abi3-py37 = ["abi3-py38"]
abi3-py38 = ["abi3-py39"]
abi3-py39 = ["abi3"]

# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
# Python interpreter if needed.
auto-initialize = []

# Optimizes PyObject to Vec conversion and so on.
nightly = []

# Use this feature when building an extension module.
# It tells the linker to keep the python symbols unresolved,
# so that the module can also be used with statically linked python interpreters.
extension-module = []

[workspace]
members = [
"pyo3-macros",
Expand Down
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and
use it to run Python code, add `pyo3` to your `Cargo.toml` like this:

```toml
[dependencies]
pyo3 = "0.13.0"
[dependencies.pyo3]
version = "0.13.0"
features = ["embedding", "auto-initialize"]
```

Example program displaying the value of `sys.version` and the current user name:
Expand Down
4 changes: 4 additions & 0 deletions build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> {
}
}

if interpreter_config.shared {
println!("cargo:rustc-cfg=Py_SHARED");
}

if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
println!("cargo:rustc-cfg=PyPy");
};
Expand Down
96 changes: 72 additions & 24 deletions src/gil.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@
//! Interaction with python's global interpreter lock
use crate::{ffi, internal_tricks::Unsendable, Python};
use parking_lot::{const_mutex, Mutex};
use parking_lot::{const_mutex, Mutex, Once};
use std::cell::{Cell, RefCell};
use std::{mem::ManuallyDrop, ptr::NonNull, sync};
use std::{mem::ManuallyDrop, ptr::NonNull};

static START: sync::Once = sync::Once::new();
static START: Once = Once::new();

thread_local! {
/// This is a internal counter in pyo3 monitoring whether this thread has the GIL.
Expand Down Expand Up @@ -45,16 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool {
/// If both the Python interpreter and Python threading are already initialized,
/// this function has no effect.
///
/// # Availability
///
/// This function is only available when linking against Python distributions that contain a
/// shared library.
///
/// This function is not available on PyPy.
///
/// # Panic
/// If the Python interpreter is initialized but Python threading is not,
/// a panic occurs.
/// It is not possible to safely access the Python runtime unless the main
/// thread (the thread which originally initialized Python) also initializes
/// threading.
///
/// When writing an extension module, the `#[pymodule]` macro
/// will ensure that Python threading is initialized.
///
#[cfg(all(Py_SHARED, not(PyPy)))]
pub fn prepare_freethreaded_python() {
// Protect against race conditions when Python is not yet initialized
// and multiple threads concurrently call 'prepare_freethreaded_python()'.
Expand All @@ -72,34 +76,29 @@ pub fn prepare_freethreaded_python() {
// Note that the 'main thread' notion in Python isn't documented properly;
// and running Python without one is not officially supported.

// PyPy does not support the embedding API
#[cfg(not(PyPy))]
{
ffi::Py_InitializeEx(0);

// Make sure Py_Finalize will be called before exiting.
extern "C" fn finalize() {
unsafe {
if ffi::Py_IsInitialized() != 0 {
ffi::PyGILState_Ensure();
ffi::Py_Finalize();
}
ffi::Py_InitializeEx(0);

// Make sure Py_Finalize will be called before exiting.
extern "C" fn finalize() {
unsafe {
if ffi::Py_IsInitialized() != 0 {
ffi::PyGILState_Ensure();
ffi::Py_Finalize();
}
}
libc::atexit(finalize);
}
libc::atexit(finalize);

// > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
// > to call it yourself anymore.
#[cfg(not(Py_3_7))]
if ffi::PyEval_ThreadsInitialized() == 0 {
ffi::PyEval_InitThreads();
}
// PyEval_InitThreads() will acquire the GIL,
// but we don't want to hold it at this point

// Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
// (it's not acquired in the other code paths)
// So immediately release the GIL:
#[cfg(not(PyPy))]
let _thread_state = ffi::PyEval_SaveThread();
// Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
// and will be restored by PyGILState_Ensure.
Expand Down Expand Up @@ -137,7 +136,56 @@ impl GILGuard {
/// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
/// new `GILGuard` will also contain a `GILPool`.
pub(crate) fn acquire() -> GILGuard {
prepare_freethreaded_python();
// Maybe auto-initialize the GIL:
// - If auto-initialize feature set and supported, try to initalize the interpreter.
// - If the auto-initialize feature is set but unsupported, emit hard errors only when
// the extension-module feature is not activated - extension modules don't care about
// auto-initialize so this avoids breaking existing builds.
// - Otherwise, just check the GIL is initialized.
cfg_if::cfg_if! {
if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] {
prepare_freethreaded_python();
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(rustdoc)))] {
compile_error!(concat!(
"The `auto-initialize` feature is not supported when linking Python ",
"statically instead of with a shared library.\n",
"\n",
"Please disable the `auto-initialize` feature, for example by entering the following ",
"in your cargo.toml:\n",
"\n",
" pyo3 = { version = \"0.13.0\", default-features = false }\n",
"\n",
"Alternatively, compile PyO3 using a Python distribution which contains a shared ",
"libary."
));
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(rustdoc)))] {
compile_error!(concat!(
"The `auto-initialize` feature is not supported by PyPy.\n",
"\n",
"Please disable the `auto-initialize` feature, for example by entering the following ",
"in your cargo.toml:\n",
"\n",
" pyo3 = { version = \"0.13.0\", default-features = false }\n",
));
} else {
// extension module feature enabled and PyPy or static linking
// OR auto-initialize feature not enabled
START.call_once_force(|_| unsafe {
// Use call_once_force because if there is a panic because the interpreter is not
// initialized, it's fine for the user to initialize the interpreter and retry.
assert_ne!(
ffi::Py_IsInitialized(),
0,
"The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
);
assert_ne!(
ffi::PyEval_ThreadsInitialized(),
0,
"Python threading is not initalized and the `auto-initialize` feature is not enabled."
);
});
}
}

let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL

Expand Down
9 changes: 6 additions & 3 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,9 @@
//! Add `pyo3` to your `Cargo.toml`:
//!
//! ```toml
//! [dependencies]
//! pyo3 = "0.13.0"
//! [dependencies.pyo3]
//! version = "0.13.0"
//! features = ["embedding", "auto-initialize"]
//! ```
//!
//! Example program displaying the value of `sys.version`:
Expand Down Expand Up @@ -145,12 +146,14 @@ pub use crate::conversion::{
ToBorrowedObject, ToPyObject,
};
pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
#[cfg(all(Py_SHARED, not(PyPy)))]
pub use crate::gil::prepare_freethreaded_python;
pub use crate::gil::{GILGuard, GILPool};
pub use crate::instance::{Py, PyNativeType, PyObject};
pub use crate::pycell::{PyCell, PyRef, PyRefMut};
pub use crate::pyclass::PyClass;
pub use crate::pyclass_init::PyClassInitializer;
pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo};
pub use crate::python::{Python, PythonVersionInfo};
pub use crate::type_object::{type_flags, PyTypeInfo};
// Since PyAny is as important as PyObject, we expose it to the top level.
pub use crate::types::PyAny;
Expand Down
20 changes: 14 additions & 6 deletions src/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ use std::ffi::{CStr, CString};
use std::marker::PhantomData;
use std::os::raw::{c_char, c_int};

pub use gil::prepare_freethreaded_python;

/// Represents the major, minor, and patch (if any) versions of this interpreter.
///
/// See [Python::version].
Expand Down Expand Up @@ -134,8 +132,13 @@ impl Python<'_> {
/// Acquires the global interpreter lock, which allows access to the Python runtime. The
/// provided closure F will be executed with the acquired `Python` marker token.
///
/// If the Python runtime is not already initialized, this function will initialize it.
/// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
/// If the `auto-initialize` feature is enabled and the Python runtime is not already
/// initialized, this function will initialize it. See
/// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
///
/// # Panics
/// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
/// initialized.
///
/// # Example
/// ```
Expand All @@ -158,8 +161,9 @@ impl Python<'_> {
impl<'p> Python<'p> {
/// Acquires the global interpreter lock, which allows access to the Python runtime.
///
/// If the Python runtime is not already initialized, this function will initialize it.
/// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
/// If the `auto-initialize` feature is enabled and the Python runtime is not already
/// initialized, this function will initialize it. See
/// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
///
/// Most users should not need to use this API directly, and should prefer one of two options:
/// 1. When implementing `#[pymethods]` or `#[pyfunction]` add a function argument
Expand All @@ -172,6 +176,10 @@ impl<'p> Python<'p> {
/// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order
/// to acquisition. If PyO3 detects this order is not maintained, it may be forced to begin
/// an irrecoverable panic.
///
/// # Panics
/// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
/// initialized.
#[inline]
pub fn acquire_gil() -> GILGuard {
GILGuard::acquire()
Expand Down

0 comments on commit 5eb33f7

Please sign in to comment.