1//! Fundamental properties of objects tied to the Python interpreter.
2//!
3//! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded
4//! scenarios there is a global lock, the *global interpreter lock* (hereafter referred to as *GIL*)
5//! that must be held to safely interact with Python objects. This is why in PyO3 when you acquire
6//! the GIL you get a [`Python`] marker token that carries the *lifetime* of holding the GIL and all
7//! borrowed references to Python objects carry this lifetime as well. This will statically ensure
8//! that you can never use Python objects after dropping the lock - if you mess this up it will be
9//! caught at compile time and your program will fail to compile.
10//!
11//! It also supports this pattern that many extension modules employ:
12//! - Drop the GIL, so that other Python threads can acquire it and make progress themselves
13//! - Do something independently of the Python interpreter, like IO, a long running calculation or
14//! awaiting a future
15//! - Once that is done, reacquire the GIL
16//!
17//! That API is provided by [`Python::allow_threads`] and enforced via the [`Ungil`] bound on the
18//! closure and the return type. This is done by relying on the [`Send`] auto trait. `Ungil` is
19//! defined as the following:
20//!
21//! ```rust
22//! pub unsafe trait Ungil {}
23//!
24//! unsafe impl<T: Send> Ungil for T {}
25//! ```
26//!
27//! We piggy-back off the `Send` auto trait because it is not possible to implement custom auto
28//! traits on stable Rust. This is the solution which enables it for as many types as possible while
29//! making the API usable.
30//!
31//! In practice this API works quite well, but it comes with some drawbacks:
32//!
33//! ## Drawbacks
34//!
35//! There is no reason to prevent `!Send` types like [`Rc`] from crossing the closure. After all,
36//! [`Python::allow_threads`] just lets other Python threads run - it does not itself launch a new
37//! thread.
38//!
39//! ```rust, compile_fail
40//! # #[cfg(feature = "nightly")]
41//! # compile_error!("this actually works on nightly")
42//! use pyo3::prelude::*;
43//! use std::rc::Rc;
44//!
45//! fn main() {
46//! Python::with_gil(|py| {
47//! let rc = Rc::new(5);
48//!
49//! py.allow_threads(|| {
50//! // This would actually be fine...
51//! println!("{:?}", *rc);
52//! });
53//! });
54//! }
55//! ```
56//!
57//! Because we are using `Send` for something it's not quite meant for, other code that
58//! (correctly) upholds the invariants of [`Send`] can cause problems.
59//!
60//! [`SendWrapper`] is one of those. Per its documentation:
61//!
62//! > A wrapper which allows you to move around non-Send-types between threads, as long as you
63//! > access the contained value only from within the original thread and make sure that it is
64//! > dropped from within the original thread.
65//!
66//! This will "work" to smuggle Python references across the closure, because we're not actually
67//! doing anything with threads:
68//!
69//! ```rust, no_run
70//! use pyo3::prelude::*;
71//! use pyo3::types::PyString;
72//! use send_wrapper::SendWrapper;
73//!
74//! Python::with_gil(|py| {
75//! let string = PyString::new(py, "foo");
76//!
77//! let wrapped = SendWrapper::new(string);
78//!
79//! py.allow_threads(|| {
80//! # #[cfg(not(feature = "nightly"))]
81//! # {
82//! // 💥 Unsound! 💥
83//! let smuggled: &PyString = *wrapped;
84//! println!("{:?}", smuggled);
85//! # }
86//! });
87//! });
88//! ```
89//!
90//! For now the answer to that is "don't do that".
91//!
92//! # A proper implementation using an auto trait
93//!
94//! However on nightly Rust and when PyO3's `nightly` feature is
95//! enabled, `Ungil` is defined as the following:
96//!
97//! ```rust
98//! # #[cfg(FALSE)]
99//! # {
100//! #![feature(auto_traits, negative_impls)]
101//!
102//! pub unsafe auto trait Ungil {}
103//!
104//! // It is unimplemented for the `Python` struct and Python objects.
105//! impl !Ungil for Python<'_> {}
106//! impl !Ungil for ffi::PyObject {}
107//!
108//! // `Py` wraps it in a safe api, so this is OK
109//! unsafe impl<T> Ungil for Py<T> {}
110//! # }
111//! ```
112//!
113//! With this feature enabled, the above two examples will start working and not working, respectively.
114//!
115//! [`SendWrapper`]: https://docs.rs/send_wrapper/latest/send_wrapper/struct.SendWrapper.html
116//! [`Rc`]: std::rc::Rc
117//! [`Py`]: crate::Py
118use crate::err::{self, PyDowncastError, PyErr, PyResult};
119use crate::gil::{GILGuard, GILPool, SuspendGIL};
120use crate::impl_::not_send::NotSend;
121use crate::types::{
122 PyAny, PyDict, PyEllipsis, PyModule, PyNone, PyNotImplemented, PyString, PyType,
123};
124use crate::version::PythonVersionInfo;
125use crate::{ffi, FromPyPointer, IntoPy, Py, PyNativeType, PyObject, PyTryFrom, PyTypeInfo};
126use std::ffi::{CStr, CString};
127use std::marker::PhantomData;
128use std::os::raw::c_int;
129
130/// Types that are safe to access while the GIL is not held.
131///
132/// # Safety
133///
134/// The type must not carry borrowed Python references or, if it does, not allow access to them if
135/// the GIL is not held.
136///
137/// See the [module-level documentation](self) for more information.
138///
139/// # Examples
140///
141/// This tracking is currently imprecise as it relies on the [`Send`] auto trait on stable Rust.
142/// For example, an `Rc` smart pointer should be usable without the GIL, but we currently prevent that:
143///
144/// ```compile_fail
145/// # use pyo3::prelude::*;
146/// use std::rc::Rc;
147///
148/// Python::with_gil(|py| {
149/// let rc = Rc::new(42);
150///
151/// py.allow_threads(|| {
152/// println!("{:?}", rc);
153/// });
154/// });
155/// ```
156///
157/// This also implies that the interplay between `with_gil` and `allow_threads` is unsound, for example
158/// one can circumvent this protection using the [`send_wrapper`](https://docs.rs/send_wrapper/) crate:
159///
160/// ```no_run
161/// # use pyo3::prelude::*;
162/// # use pyo3::types::PyString;
163/// use send_wrapper::SendWrapper;
164///
165/// Python::with_gil(|py| {
166/// let string = PyString::new(py, "foo");
167///
168/// let wrapped = SendWrapper::new(string);
169///
170/// py.allow_threads(|| {
171/// let sneaky: &PyString = *wrapped;
172///
173/// println!("{:?}", sneaky);
174/// });
175/// });
176/// ```
177///
178/// Fixing this loophole on stable Rust has significant ergonomic issues, but it is fixed when using
179/// nightly Rust and the `nightly` feature, c.f. [#2141](https://github.com/PyO3/pyo3/issues/2141).
180#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag
181#[cfg(not(feature = "nightly"))]
182pub unsafe trait Ungil {}
183
184#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag
185#[cfg(not(feature = "nightly"))]
186unsafe impl<T: Send> Ungil for T {}
187
188#[cfg(feature = "nightly")]
189mod nightly {
190 macro_rules! define {
191 ($($tt:tt)*) => { $($tt)* }
192 }
193
194 define! {
195 /// Types that are safe to access while the GIL is not held.
196 ///
197 /// # Safety
198 ///
199 /// The type must not carry borrowed Python references or, if it does, not allow access to them if
200 /// the GIL is not held.
201 ///
202 /// See the [module-level documentation](self) for more information.
203 ///
204 /// # Examples
205 ///
206 /// Types which are `Ungil` cannot be used in contexts where the GIL was released, e.g.
207 ///
208 /// ```compile_fail
209 /// # use pyo3::prelude::*;
210 /// # use pyo3::types::PyString;
211 /// Python::with_gil(|py| {
212 /// let string = PyString::new(py, "foo");
213 ///
214 /// py.allow_threads(|| {
215 /// println!("{:?}", string);
216 /// });
217 /// });
218 /// ```
219 ///
220 /// This applies to the GIL token `Python` itself as well, e.g.
221 ///
222 /// ```compile_fail
223 /// # use pyo3::prelude::*;
224 /// Python::with_gil(|py| {
225 /// py.allow_threads(|| {
226 /// drop(py);
227 /// });
228 /// });
229 /// ```
230 ///
231 /// On nightly Rust, this is not based on the [`Send`] auto trait and hence we are able
232 /// to prevent incorrectly circumventing it using e.g. the [`send_wrapper`](https://docs.rs/send_wrapper/) crate:
233 ///
234 /// ```compile_fail
235 /// # use pyo3::prelude::*;
236 /// # use pyo3::types::PyString;
237 /// use send_wrapper::SendWrapper;
238 ///
239 /// Python::with_gil(|py| {
240 /// let string = PyString::new(py, "foo");
241 ///
242 /// let wrapped = SendWrapper::new(string);
243 ///
244 /// py.allow_threads(|| {
245 /// let sneaky: &PyString = *wrapped;
246 ///
247 /// println!("{:?}", sneaky);
248 /// });
249 /// });
250 /// ```
251 ///
252 /// This also enables using non-[`Send`] types in `allow_threads`,
253 /// at least if they are not also bound to the GIL:
254 ///
255 /// ```rust
256 /// # use pyo3::prelude::*;
257 /// use std::rc::Rc;
258 ///
259 /// Python::with_gil(|py| {
260 /// let rc = Rc::new(42);
261 ///
262 /// py.allow_threads(|| {
263 /// println!("{:?}", rc);
264 /// });
265 /// });
266 /// ```
267 pub unsafe auto trait Ungil {}
268 }
269
270 impl !Ungil for crate::Python<'_> {}
271
272 // This means that PyString, PyList, etc all inherit !Ungil from this.
273 impl !Ungil for crate::PyAny {}
274
275 // All the borrowing wrappers
276 impl<T> !Ungil for crate::PyCell<T> {}
277 impl<T> !Ungil for crate::PyRef<'_, T> {}
278 impl<T> !Ungil for crate::PyRefMut<'_, T> {}
279
280 // FFI pointees
281 impl !Ungil for crate::ffi::PyObject {}
282 impl !Ungil for crate::ffi::PyLongObject {}
283
284 impl !Ungil for crate::ffi::PyThreadState {}
285 impl !Ungil for crate::ffi::PyInterpreterState {}
286 impl !Ungil for crate::ffi::PyWeakReference {}
287 impl !Ungil for crate::ffi::PyFrameObject {}
288 impl !Ungil for crate::ffi::PyCodeObject {}
289 #[cfg(not(Py_LIMITED_API))]
290 impl !Ungil for crate::ffi::PyDictKeysObject {}
291 #[cfg(not(any(Py_LIMITED_API, Py_3_10)))]
292 impl !Ungil for crate::ffi::PyArena {}
293}
294
295#[cfg(feature = "nightly")]
296pub use nightly::Ungil;
297
298/// A marker token that represents holding the GIL.
299///
300/// It serves three main purposes:
301/// - It provides a global API for the Python interpreter, such as [`Python::eval`].
302/// - It can be passed to functions that require a proof of holding the GIL, such as
303/// [`Py::clone_ref`].
304/// - Its lifetime represents the scope of holding the GIL which can be used to create Rust
305/// references that are bound to it, such as `&`[`PyAny`].
306///
307/// Note that there are some caveats to using it that you might need to be aware of. See the
308/// [Deadlocks](#deadlocks) and [Releasing and freeing memory](#releasing-and-freeing-memory)
309/// paragraphs for more information about that.
310///
311/// # Obtaining a Python token
312///
313/// The following are the recommended ways to obtain a [`Python`] token, in order of preference:
314/// - In a function or method annotated with [`#[pyfunction]`](crate::pyfunction) or [`#[pymethods]`](crate::pymethods) you can declare it
315/// as a parameter, and PyO3 will pass in the token when Python code calls it.
316/// - If you already have something with a lifetime bound to the GIL, such as `&`[`PyAny`], you can
317/// use its [`.py()`][PyAny::py] method to get a token.
318/// - When you need to acquire the GIL yourself, such as when calling Python code from Rust, you
319/// should call [`Python::with_gil`] to do that and pass your code as a closure to it.
320///
321/// # Deadlocks
322///
323/// Note that the GIL can be temporarily released by the Python interpreter during a function call
324/// (e.g. importing a module). In general, you don't need to worry about this because the GIL is
325/// reacquired before returning to the Rust code:
326///
327/// ```text
328/// `Python` exists |=====================================|
329/// GIL actually held |==========| |================|
330/// Rust code running |=======| |==| |======|
331/// ```
332///
333/// This behaviour can cause deadlocks when trying to lock a Rust mutex while holding the GIL:
334///
335/// * Thread 1 acquires the GIL
336/// * Thread 1 locks a mutex
337/// * Thread 1 makes a call into the Python interpreter which releases the GIL
338/// * Thread 2 acquires the GIL
339/// * Thread 2 tries to locks the mutex, blocks
340/// * Thread 1's Python interpreter call blocks trying to reacquire the GIL held by thread 2
341///
342/// To avoid deadlocking, you should release the GIL before trying to lock a mutex or `await`ing in
343/// asynchronous code, e.g. with [`Python::allow_threads`].
344///
345/// # Releasing and freeing memory
346///
347/// The [`Python`] type can be used to create references to variables owned by the Python
348/// interpreter, using functions such as [`Python::eval`] and [`PyModule::import`]. These
349/// references are tied to a [`GILPool`] whose references are not cleared until it is dropped.
350/// This can cause apparent "memory leaks" if it is kept around for a long time.
351///
352/// ```rust
353/// use pyo3::prelude::*;
354/// use pyo3::types::PyString;
355///
356/// # fn main () -> PyResult<()> {
357/// Python::with_gil(|py| -> PyResult<()> {
358/// for _ in 0..10 {
359/// let hello: &PyString = py.eval("\"Hello World!\"", None, None)?.extract()?;
360/// println!("Python says: {}", hello.to_str()?);
361/// // Normally variables in a loop scope are dropped here, but `hello` is a reference to
362/// // something owned by the Python interpreter. Dropping this reference does nothing.
363/// }
364/// Ok(())
365/// })
366/// // This is where the `hello`'s reference counts start getting decremented.
367/// # }
368/// ```
369///
370/// The variable `hello` is dropped at the end of each loop iteration, but the lifetime of the
371/// pointed-to memory is bound to [`Python::with_gil`]'s [`GILPool`] which will not be dropped until
372/// the end of [`Python::with_gil`]'s scope. Only then is each `hello`'s Python reference count
373/// decreased. This means that at the last line of the example there are 10 copies of `hello` in
374/// Python's memory, not just one at a time as we might expect from Rust's [scoping rules].
375///
376/// See the [Memory Management] chapter of the guide for more information about how PyO3 uses
377/// [`GILPool`] to manage memory.
378///
379/// [scoping rules]: https://doc.rust-lang.org/stable/book/ch04-01-what-is-ownership.html#ownership-rules
380/// [`Py::clone_ref`]: crate::Py::clone_ref
381/// [Memory Management]: https://pyo3.rs/main/memory.html#gil-bound-memory
382#[derive(Copy, Clone)]
383pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>);
384
385impl Python<'_> {
386 /// Acquires the global interpreter lock, allowing access to the Python interpreter. The
387 /// provided closure `F` will be executed with the acquired `Python` marker token.
388 ///
389 /// If implementing [`#[pymethods]`](crate::pymethods) or [`#[pyfunction]`](crate::pyfunction),
390 /// declare `py: Python` as an argument. PyO3 will pass in the token to grant access to the GIL
391 /// context in which the function is running, avoiding the need to call `with_gil`.
392 ///
393 /// If the [`auto-initialize`] feature is enabled and the Python runtime is not already
394 /// initialized, this function will initialize it. See
395 #[cfg_attr(
396 not(PyPy),
397 doc = "[`prepare_freethreaded_python`](crate::prepare_freethreaded_python)"
398 )]
399 #[cfg_attr(PyPy, doc = "`prepare_freethreaded_python`")]
400 /// for details.
401 ///
402 /// If the current thread does not yet have a Python "thread state" associated with it,
403 /// a new one will be automatically created before `F` is executed and destroyed after `F`
404 /// completes.
405 ///
406 /// # Panics
407 ///
408 /// - If the [`auto-initialize`] feature is not enabled and the Python interpreter is not
409 /// initialized.
410 ///
411 /// # Examples
412 ///
413 /// ```
414 /// use pyo3::prelude::*;
415 ///
416 /// # fn main() -> PyResult<()> {
417 /// Python::with_gil(|py| -> PyResult<()> {
418 /// let x: i32 = py.eval("5", None, None)?.extract()?;
419 /// assert_eq!(x, 5);
420 /// Ok(())
421 /// })
422 /// # }
423 /// ```
424 ///
425 /// [`auto-initialize`]: https://pyo3.rs/main/features.html#auto-initialize
426 #[inline]
427 pub fn with_gil<F, R>(f: F) -> R
428 where
429 F: for<'py> FnOnce(Python<'py>) -> R,
430 {
431 let _guard = GILGuard::acquire();
432
433 // SAFETY: Either the GIL was already acquired or we just created a new `GILGuard`.
434 f(unsafe { Python::assume_gil_acquired() })
435 }
436
437 /// Like [`Python::with_gil`] except Python interpreter state checking is skipped.
438 ///
439 /// Normally when the GIL is acquired, we check that the Python interpreter is an
440 /// appropriate state (e.g. it is fully initialized). This function skips those
441 /// checks.
442 ///
443 /// # Safety
444 ///
445 /// If [`Python::with_gil`] would succeed, it is safe to call this function.
446 ///
447 /// In most cases, you should use [`Python::with_gil`].
448 ///
449 /// A justified scenario for calling this function is during multi-phase interpreter
450 /// initialization when [`Python::with_gil`] would fail before
451 // this link is only valid on 3.8+not pypy and up.
452 #[cfg_attr(
453 all(Py_3_8, not(PyPy)),
454 doc = "[`_Py_InitializeMain`](crate::ffi::_Py_InitializeMain)"
455 )]
456 #[cfg_attr(any(not(Py_3_8), PyPy), doc = "`_Py_InitializeMain`")]
457 /// is called because the interpreter is only partially initialized.
458 ///
459 /// Behavior in other scenarios is not documented.
460 #[inline]
461 pub unsafe fn with_gil_unchecked<F, R>(f: F) -> R
462 where
463 F: for<'py> FnOnce(Python<'py>) -> R,
464 {
465 let _guard = GILGuard::acquire_unchecked();
466
467 // SAFETY: Either the GIL was already acquired or we just created a new `GILGuard`.
468 f(Python::assume_gil_acquired())
469 }
470}
471
472impl<'py> Python<'py> {
473 /// Temporarily releases the GIL, thus allowing other Python threads to run. The GIL will be
474 /// reacquired when `F`'s scope ends.
475 ///
476 /// If you don't need to touch the Python
477 /// interpreter for some time and have other Python threads around, this will let you run
478 /// Rust-only code while letting those other Python threads make progress.
479 ///
480 /// Only types that implement [`Ungil`] can cross the closure. See the
481 /// [module level documentation](self) for more information.
482 ///
483 /// If you need to pass Python objects into the closure you can use [`Py`]`<T>`to create a
484 /// reference independent of the GIL lifetime. However, you cannot do much with those without a
485 /// [`Python`] token, for which you'd need to reacquire the GIL.
486 ///
487 /// # Example: Releasing the GIL while running a computation in Rust-only code
488 ///
489 /// ```
490 /// use pyo3::prelude::*;
491 ///
492 /// #[pyfunction]
493 /// fn sum_numbers(py: Python<'_>, numbers: Vec<u32>) -> PyResult<u32> {
494 /// // We release the GIL here so any other Python threads get a chance to run.
495 /// py.allow_threads(move || {
496 /// // An example of an "expensive" Rust calculation
497 /// let sum = numbers.iter().sum();
498 ///
499 /// Ok(sum)
500 /// })
501 /// }
502 /// #
503 /// # fn main() -> PyResult<()> {
504 /// # Python::with_gil(|py| -> PyResult<()> {
505 /// # let fun = pyo3::wrap_pyfunction!(sum_numbers, py)?;
506 /// # let res = fun.call1((vec![1_u32, 2, 3],))?;
507 /// # assert_eq!(res.extract::<u32>()?, 6_u32);
508 /// # Ok(())
509 /// # })
510 /// # }
511 /// ```
512 ///
513 /// Please see the [Parallelism] chapter of the guide for a thorough discussion of using
514 /// [`Python::allow_threads`] in this manner.
515 ///
516 /// # Example: Passing borrowed Python references into the closure is not allowed
517 ///
518 /// ```compile_fail
519 /// use pyo3::prelude::*;
520 /// use pyo3::types::PyString;
521 ///
522 /// fn parallel_print(py: Python<'_>) {
523 /// let s = PyString::new(py, "This object cannot be accessed without holding the GIL >_<");
524 /// py.allow_threads(move || {
525 /// println!("{:?}", s); // This causes a compile error.
526 /// });
527 /// }
528 /// ```
529 ///
530 /// [`Py`]: crate::Py
531 /// [`PyString`]: crate::types::PyString
532 /// [auto-traits]: https://doc.rust-lang.org/nightly/unstable-book/language-features/auto-traits.html
533 /// [Parallelism]: https://pyo3.rs/main/parallelism.html
534 pub fn allow_threads<T, F>(self, f: F) -> T
535 where
536 F: Ungil + FnOnce() -> T,
537 T: Ungil,
538 {
539 // Use a guard pattern to handle reacquiring the GIL,
540 // so that the GIL will be reacquired even if `f` panics.
541 // The `Send` bound on the closure prevents the user from
542 // transferring the `Python` token into the closure.
543 let _guard = unsafe { SuspendGIL::new() };
544 f()
545 }
546
547 /// Evaluates a Python expression in the given context and returns the result.
548 ///
549 /// If `globals` is `None`, it defaults to Python module `__main__`.
550 /// If `locals` is `None`, it defaults to the value of `globals`.
551 ///
552 /// If `globals` doesn't contain `__builtins__`, default `__builtins__`
553 /// will be added automatically.
554 ///
555 /// # Examples
556 ///
557 /// ```
558 /// # use pyo3::prelude::*;
559 /// # Python::with_gil(|py| {
560 /// let result = py.eval("[i * 10 for i in range(5)]", None, None).unwrap();
561 /// let res: Vec<i64> = result.extract().unwrap();
562 /// assert_eq!(res, vec![0, 10, 20, 30, 40])
563 /// # });
564 /// ```
565 pub fn eval(
566 self,
567 code: &str,
568 globals: Option<&PyDict>,
569 locals: Option<&PyDict>,
570 ) -> PyResult<&'py PyAny> {
571 self.run_code(code, ffi::Py_eval_input, globals, locals)
572 }
573
574 /// Executes one or more Python statements in the given context.
575 ///
576 /// If `globals` is `None`, it defaults to Python module `__main__`.
577 /// If `locals` is `None`, it defaults to the value of `globals`.
578 ///
579 /// If `globals` doesn't contain `__builtins__`, default `__builtins__`
580 /// will be added automatically.
581 ///
582 /// # Examples
583 /// ```
584 /// use pyo3::{
585 /// prelude::*,
586 /// types::{PyBytes, PyDict},
587 /// };
588 /// Python::with_gil(|py| {
589 /// let locals = PyDict::new(py);
590 /// py.run(
591 /// r#"
592 /// import base64
593 /// s = 'Hello Rust!'
594 /// ret = base64.b64encode(s.encode('utf-8'))
595 /// "#,
596 /// None,
597 /// Some(locals),
598 /// )
599 /// .unwrap();
600 /// let ret = locals.get_item("ret").unwrap().unwrap();
601 /// let b64: &PyBytes = ret.downcast().unwrap();
602 /// assert_eq!(b64.as_bytes(), b"SGVsbG8gUnVzdCE=");
603 /// });
604 /// ```
605 ///
606 /// You can use [`py_run!`](macro.py_run.html) for a handy alternative of `run`
607 /// if you don't need `globals` and unwrapping is OK.
608 pub fn run(
609 self,
610 code: &str,
611 globals: Option<&PyDict>,
612 locals: Option<&PyDict>,
613 ) -> PyResult<()> {
614 let res = self.run_code(code, ffi::Py_file_input, globals, locals);
615 res.map(|obj| {
616 debug_assert!(obj.is_none());
617 })
618 }
619
620 /// Runs code in the given context.
621 ///
622 /// `start` indicates the type of input expected: one of `Py_single_input`,
623 /// `Py_file_input`, or `Py_eval_input`.
624 ///
625 /// If `globals` is `None`, it defaults to Python module `__main__`.
626 /// If `locals` is `None`, it defaults to the value of `globals`.
627 fn run_code(
628 self,
629 code: &str,
630 start: c_int,
631 globals: Option<&PyDict>,
632 locals: Option<&PyDict>,
633 ) -> PyResult<&'py PyAny> {
634 let code = CString::new(code)?;
635 unsafe {
636 let mptr = ffi::PyImport_AddModule("__main__\0".as_ptr() as *const _);
637 if mptr.is_null() {
638 return Err(PyErr::fetch(self));
639 }
640
641 let globals = globals
642 .map(|dict| dict.as_ptr())
643 .unwrap_or_else(|| ffi::PyModule_GetDict(mptr));
644 let locals = locals.map(|dict| dict.as_ptr()).unwrap_or(globals);
645
646 // If `globals` don't provide `__builtins__`, most of the code will fail if Python
647 // version is <3.10. That's probably not what user intended, so insert `__builtins__`
648 // for them.
649 //
650 // See also:
651 // - https://github.com/python/cpython/pull/24564 (the same fix in CPython 3.10)
652 // - https://github.com/PyO3/pyo3/issues/3370
653 let builtins_s = crate::intern!(self, "__builtins__").as_ptr();
654 let has_builtins = ffi::PyDict_Contains(globals, builtins_s);
655 if has_builtins == -1 {
656 return Err(PyErr::fetch(self));
657 }
658 if has_builtins == 0 {
659 // Inherit current builtins.
660 let builtins = ffi::PyEval_GetBuiltins();
661
662 // `PyDict_SetItem` doesn't take ownership of `builtins`, but `PyEval_GetBuiltins`
663 // seems to return a borrowed reference, so no leak here.
664 if ffi::PyDict_SetItem(globals, builtins_s, builtins) == -1 {
665 return Err(PyErr::fetch(self));
666 }
667 }
668
669 let code_obj = ffi::Py_CompileString(code.as_ptr(), "<string>\0".as_ptr() as _, start);
670 if code_obj.is_null() {
671 return Err(PyErr::fetch(self));
672 }
673 let res_ptr = ffi::PyEval_EvalCode(code_obj, globals, locals);
674 ffi::Py_DECREF(code_obj);
675
676 self.from_owned_ptr_or_err(res_ptr)
677 }
678 }
679
680 /// Gets the Python type object for type `T`.
681 #[inline]
682 pub fn get_type<T>(self) -> &'py PyType
683 where
684 T: PyTypeInfo,
685 {
686 T::type_object(self)
687 }
688
689 /// Imports the Python module with the specified name.
690 pub fn import<N>(self, name: N) -> PyResult<&'py PyModule>
691 where
692 N: IntoPy<Py<PyString>>,
693 {
694 PyModule::import(self, name)
695 }
696
697 /// Gets the Python builtin value `None`.
698 #[allow(non_snake_case)] // the Python keyword starts with uppercase
699 #[inline]
700 pub fn None(self) -> PyObject {
701 PyNone::get(self).into()
702 }
703
704 /// Gets the Python builtin value `Ellipsis`, or `...`.
705 #[allow(non_snake_case)] // the Python keyword starts with uppercase
706 #[inline]
707 pub fn Ellipsis(self) -> PyObject {
708 PyEllipsis::get(self).into()
709 }
710
711 /// Gets the Python builtin value `NotImplemented`.
712 #[allow(non_snake_case)] // the Python keyword starts with uppercase
713 #[inline]
714 pub fn NotImplemented(self) -> PyObject {
715 PyNotImplemented::get(self).into()
716 }
717
718 /// Gets the running Python interpreter version as a string.
719 ///
720 /// # Examples
721 /// ```rust
722 /// # use pyo3::Python;
723 /// Python::with_gil(|py| {
724 /// // The full string could be, for example:
725 /// // "3.10.0 (tags/v3.10.0:b494f59, Oct 4 2021, 19:00:18) [MSC v.1929 64 bit (AMD64)]"
726 /// assert!(py.version().starts_with("3."));
727 /// });
728 /// ```
729 pub fn version(self) -> &'py str {
730 unsafe {
731 CStr::from_ptr(ffi::Py_GetVersion())
732 .to_str()
733 .expect("Python version string not UTF-8")
734 }
735 }
736
737 /// Gets the running Python interpreter version as a struct similar to
738 /// `sys.version_info`.
739 ///
740 /// # Examples
741 /// ```rust
742 /// # use pyo3::Python;
743 /// Python::with_gil(|py| {
744 /// // PyO3 supports Python 3.7 and up.
745 /// assert!(py.version_info() >= (3, 7));
746 /// assert!(py.version_info() >= (3, 7, 0));
747 /// });
748 /// ```
749 pub fn version_info(self) -> PythonVersionInfo<'py> {
750 let version_str = self.version();
751
752 // Portion of the version string returned by Py_GetVersion up to the first space is the
753 // version number.
754 let version_number_str = version_str.split(' ').next().unwrap_or(version_str);
755
756 PythonVersionInfo::from_str(version_number_str).unwrap()
757 }
758
759 /// Registers the object in the release pool, and tries to downcast to specific type.
760 pub fn checked_cast_as<T>(self, obj: PyObject) -> Result<&'py T, PyDowncastError<'py>>
761 where
762 T: PyTryFrom<'py>,
763 {
764 let any: &PyAny = unsafe { self.from_owned_ptr(obj.into_ptr()) };
765 <T as PyTryFrom>::try_from(any)
766 }
767
768 /// Registers the object in the release pool, and does an unchecked downcast
769 /// to the specific type.
770 ///
771 /// # Safety
772 ///
773 /// Callers must ensure that ensure that the cast is valid.
774 pub unsafe fn cast_as<T>(self, obj: PyObject) -> &'py T
775 where
776 T: PyNativeType + PyTypeInfo,
777 {
778 let any: &PyAny = self.from_owned_ptr(obj.into_ptr());
779 T::unchecked_downcast(any)
780 }
781
782 /// Registers the object pointer in the release pool,
783 /// and does an unchecked downcast to the specific type.
784 ///
785 /// # Safety
786 ///
787 /// Callers must ensure that ensure that the cast is valid.
788 #[allow(clippy::wrong_self_convention)]
789 pub unsafe fn from_owned_ptr<T>(self, ptr: *mut ffi::PyObject) -> &'py T
790 where
791 T: FromPyPointer<'py>,
792 {
793 FromPyPointer::from_owned_ptr(self, ptr)
794 }
795
796 /// Registers the owned object pointer in the release pool.
797 ///
798 /// Returns `Err(PyErr)` if the pointer is NULL.
799 /// Does an unchecked downcast to the specific type.
800 ///
801 /// # Safety
802 ///
803 /// Callers must ensure that ensure that the cast is valid.
804 #[allow(clippy::wrong_self_convention)]
805 pub unsafe fn from_owned_ptr_or_err<T>(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T>
806 where
807 T: FromPyPointer<'py>,
808 {
809 FromPyPointer::from_owned_ptr_or_err(self, ptr)
810 }
811
812 /// Registers the owned object pointer in release pool.
813 ///
814 /// Returns `None` if the pointer is NULL.
815 /// Does an unchecked downcast to the specific type.
816 ///
817 /// # Safety
818 ///
819 /// Callers must ensure that ensure that the cast is valid.
820 #[allow(clippy::wrong_self_convention)]
821 pub unsafe fn from_owned_ptr_or_opt<T>(self, ptr: *mut ffi::PyObject) -> Option<&'py T>
822 where
823 T: FromPyPointer<'py>,
824 {
825 FromPyPointer::from_owned_ptr_or_opt(self, ptr)
826 }
827
828 /// Does an unchecked downcast to the specific type.
829 ///
830 /// Panics if the pointer is NULL.
831 ///
832 /// # Safety
833 ///
834 /// Callers must ensure that ensure that the cast is valid.
835 #[allow(clippy::wrong_self_convention)]
836 pub unsafe fn from_borrowed_ptr<T>(self, ptr: *mut ffi::PyObject) -> &'py T
837 where
838 T: FromPyPointer<'py>,
839 {
840 FromPyPointer::from_borrowed_ptr(self, ptr)
841 }
842
843 /// Does an unchecked downcast to the specific type.
844 ///
845 /// Returns `Err(PyErr)` if the pointer is NULL.
846 ///
847 /// # Safety
848 ///
849 /// Callers must ensure that ensure that the cast is valid.
850 #[allow(clippy::wrong_self_convention)]
851 pub unsafe fn from_borrowed_ptr_or_err<T>(self, ptr: *mut ffi::PyObject) -> PyResult<&'py T>
852 where
853 T: FromPyPointer<'py>,
854 {
855 FromPyPointer::from_borrowed_ptr_or_err(self, ptr)
856 }
857
858 /// Does an unchecked downcast to the specific type.
859 ///
860 /// Returns `None` if the pointer is NULL.
861 ///
862 /// # Safety
863 ///
864 /// Callers must ensure that ensure that the cast is valid.
865 #[allow(clippy::wrong_self_convention)]
866 pub unsafe fn from_borrowed_ptr_or_opt<T>(self, ptr: *mut ffi::PyObject) -> Option<&'py T>
867 where
868 T: FromPyPointer<'py>,
869 {
870 FromPyPointer::from_borrowed_ptr_or_opt(self, ptr)
871 }
872
873 /// Lets the Python interpreter check and handle any pending signals. This will invoke the
874 /// corresponding signal handlers registered in Python (if any).
875 ///
876 /// Returns `Err(`[`PyErr`]`)` if any signal handler raises an exception.
877 ///
878 /// These signals include `SIGINT` (normally raised by CTRL + C), which by default raises
879 /// `KeyboardInterrupt`. For this reason it is good practice to call this function regularly
880 /// as part of long-running Rust functions so that users can cancel it.
881 ///
882 /// # Example
883 ///
884 /// ```rust
885 /// # #![allow(dead_code)] // this example is quite impractical to test
886 /// use pyo3::prelude::*;
887 ///
888 /// # fn main() {
889 /// #[pyfunction]
890 /// fn loop_forever(py: Python<'_>) -> PyResult<()> {
891 /// loop {
892 /// // As this loop is infinite it should check for signals every once in a while.
893 /// // Using `?` causes any `PyErr` (potentially containing `KeyboardInterrupt`)
894 /// // to break out of the loop.
895 /// py.check_signals()?;
896 ///
897 /// // do work here
898 /// # break Ok(()) // don't actually loop forever
899 /// }
900 /// }
901 /// # }
902 /// ```
903 ///
904 /// # Note
905 ///
906 /// This function calls [`PyErr_CheckSignals()`][1] which in turn may call signal handlers.
907 /// As Python's [`signal`][2] API allows users to define custom signal handlers, calling this
908 /// function allows arbitrary Python code inside signal handlers to run.
909 ///
910 /// If the function is called from a non-main thread, or under a non-main Python interpreter,
911 /// it does nothing yet still returns `Ok(())`.
912 ///
913 /// [1]: https://docs.python.org/3/c-api/exceptions.html?highlight=pyerr_checksignals#c.PyErr_CheckSignals
914 /// [2]: https://docs.python.org/3/library/signal.html
915 pub fn check_signals(self) -> PyResult<()> {
916 err::error_on_minusone(self, unsafe { ffi::PyErr_CheckSignals() })
917 }
918
919 /// Create a new pool for managing PyO3's owned references.
920 ///
921 /// When this `GILPool` is dropped, all PyO3 owned references created after this `GILPool` will
922 /// all have their Python reference counts decremented, potentially allowing Python to drop
923 /// the corresponding Python objects.
924 ///
925 /// Typical usage of PyO3 will not need this API, as [`Python::with_gil`] automatically creates
926 /// a `GILPool` where appropriate.
927 ///
928 /// Advanced uses of PyO3 which perform long-running tasks which never free the GIL may need
929 /// to use this API to clear memory, as PyO3 usually does not clear memory until the GIL is
930 /// released.
931 ///
932 /// # Examples
933 ///
934 /// ```rust
935 /// # use pyo3::prelude::*;
936 /// Python::with_gil(|py| {
937 /// // Some long-running process like a webserver, which never releases the GIL.
938 /// loop {
939 /// // Create a new pool, so that PyO3 can clear memory at the end of the loop.
940 /// let pool = unsafe { py.new_pool() };
941 ///
942 /// // It is recommended to *always* immediately set py to the pool's Python, to help
943 /// // avoid creating references with invalid lifetimes.
944 /// let py = pool.python();
945 ///
946 /// // do stuff...
947 /// # break; // Exit the loop so that doctest terminates!
948 /// }
949 /// });
950 /// ```
951 ///
952 /// # Safety
953 ///
954 /// Extreme care must be taken when using this API, as misuse can lead to accessing invalid
955 /// memory. In addition, the caller is responsible for guaranteeing that the GIL remains held
956 /// for the entire lifetime of the returned `GILPool`.
957 ///
958 /// Two best practices are required when using this API:
959 /// - From the moment `new_pool()` is called, only the `Python` token from the returned
960 /// `GILPool` (accessible using [`.python()`]) should be used in PyO3 APIs. All other older
961 /// `Python` tokens with longer lifetimes are unsafe to use until the `GILPool` is dropped,
962 /// because they can be used to create PyO3 owned references which have lifetimes which
963 /// outlive the `GILPool`.
964 /// - Similarly, methods on existing owned references will implicitly refer back to the
965 /// `Python` token which that reference was originally created with. If the returned values
966 /// from these methods are owned references they will inherit the same lifetime. As a result,
967 /// Rust's lifetime rules may allow them to outlive the `GILPool`, even though this is not
968 /// safe for reasons discussed above. Care must be taken to never access these return values
969 /// after the `GILPool` is dropped, unless they are converted to `Py<T>` *before* the pool
970 /// is dropped.
971 ///
972 /// [`.python()`]: crate::GILPool::python
973 #[inline]
974 pub unsafe fn new_pool(self) -> GILPool {
975 GILPool::new()
976 }
977}
978
979impl Python<'_> {
980 /// Creates a scope using a new pool for managing PyO3's owned references.
981 ///
982 /// This is a safe alterantive to [`new_pool`][Self::new_pool] as
983 /// it limits the closure to using the new GIL token at the cost of
984 /// being unable to capture existing GIL-bound references.
985 ///
986 /// Note that on stable Rust, this API suffers from the same the `SendWrapper` loophole
987 /// as [`allow_threads`][Self::allow_threads], c.f. the documentation of the [`Ungil`] trait,
988 ///
989 /// # Examples
990 ///
991 /// ```rust
992 /// # use pyo3::prelude::*;
993 /// Python::with_gil(|py| {
994 /// // Some long-running process like a webserver, which never releases the GIL.
995 /// loop {
996 /// // Create a new scope, so that PyO3 can clear memory at the end of the loop.
997 /// py.with_pool(|py| {
998 /// // do stuff...
999 /// });
1000 /// # break; // Exit the loop so that doctest terminates!
1001 /// }
1002 /// });
1003 /// ```
1004 ///
1005 /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references
1006 ///
1007 /// ```compile_fail
1008 /// # use pyo3::prelude::*;
1009 /// # use pyo3::types::PyString;
1010 ///
1011 /// Python::with_gil(|py| {
1012 /// let old_str = PyString::new(py, "a message from the past");
1013 ///
1014 /// py.with_pool(|_py| {
1015 /// print!("{:?}", old_str);
1016 /// });
1017 /// });
1018 /// ```
1019 ///
1020 /// or continuing to use the old GIL token
1021 ///
1022 /// ```compile_fail
1023 /// # use pyo3::prelude::*;
1024 ///
1025 /// Python::with_gil(|old_py| {
1026 /// old_py.with_pool(|_new_py| {
1027 /// let _none = old_py.None();
1028 /// });
1029 /// });
1030 /// ```
1031 #[inline]
1032 pub fn with_pool<F, R>(&self, f: F) -> R
1033 where
1034 F: for<'py> FnOnce(Python<'py>) -> R + Ungil,
1035 {
1036 // SAFETY: The closure is `Ungil`,
1037 // i.e. it does not capture any GIL-bound references
1038 // and accesses only the newly created GIL token.
1039 let pool = unsafe { GILPool::new() };
1040
1041 f(pool.python())
1042 }
1043}
1044
1045impl<'unbound> Python<'unbound> {
1046 /// Unsafely creates a Python token with an unbounded lifetime.
1047 ///
1048 /// Many of PyO3 APIs use `Python<'_>` as proof that the GIL is held, but this function can be
1049 /// used to call them unsafely.
1050 ///
1051 /// # Safety
1052 ///
1053 /// - This token and any borrowed Python references derived from it can only be safely used
1054 /// whilst the currently executing thread is actually holding the GIL.
1055 /// - This function creates a token with an *unbounded* lifetime. Safe code can assume that
1056 /// holding a `Python<'py>` token means the GIL is and stays acquired for the lifetime `'py`.
1057 /// If you let it or borrowed Python references escape to safe code you are
1058 /// responsible for bounding the lifetime `'unbound` appropriately. For more on unbounded
1059 /// lifetimes, see the [nomicon].
1060 ///
1061 /// [nomicon]: https://doc.rust-lang.org/nomicon/unbounded-lifetimes.html
1062 #[inline]
1063 pub unsafe fn assume_gil_acquired() -> Python<'unbound> {
1064 Python(PhantomData)
1065 }
1066}
1067
1068#[cfg(test)]
1069mod tests {
1070 use super::*;
1071 use crate::types::{IntoPyDict, PyDict, PyList};
1072 use crate::Py;
1073 use std::sync::Arc;
1074
1075 #[test]
1076 fn test_eval() {
1077 Python::with_gil(|py| {
1078 // Make sure builtin names are accessible
1079 let v: i32 = py
1080 .eval("min(1, 2)", None, None)
1081 .map_err(|e| e.display(py))
1082 .unwrap()
1083 .extract()
1084 .unwrap();
1085 assert_eq!(v, 1);
1086
1087 let d = [("foo", 13)].into_py_dict(py);
1088
1089 // Inject our own global namespace
1090 let v: i32 = py
1091 .eval("foo + 29", Some(d), None)
1092 .unwrap()
1093 .extract()
1094 .unwrap();
1095 assert_eq!(v, 42);
1096
1097 // Inject our own local namespace
1098 let v: i32 = py
1099 .eval("foo + 29", None, Some(d))
1100 .unwrap()
1101 .extract()
1102 .unwrap();
1103 assert_eq!(v, 42);
1104
1105 // Make sure builtin names are still accessible when using a local namespace
1106 let v: i32 = py
1107 .eval("min(foo, 2)", None, Some(d))
1108 .unwrap()
1109 .extract()
1110 .unwrap();
1111 assert_eq!(v, 2);
1112 });
1113 }
1114
1115 #[test]
1116 #[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
1117 fn test_allow_threads_releases_and_acquires_gil() {
1118 Python::with_gil(|py| {
1119 let b = std::sync::Arc::new(std::sync::Barrier::new(2));
1120
1121 let b2 = b.clone();
1122 std::thread::spawn(move || Python::with_gil(|_| b2.wait()));
1123
1124 py.allow_threads(|| {
1125 // If allow_threads does not release the GIL, this will deadlock because
1126 // the thread spawned above will never be able to acquire the GIL.
1127 b.wait();
1128 });
1129
1130 unsafe {
1131 // If the GIL is not reacquired at the end of allow_threads, this call
1132 // will crash the Python interpreter.
1133 let tstate = ffi::PyEval_SaveThread();
1134 ffi::PyEval_RestoreThread(tstate);
1135 }
1136 });
1137 }
1138
1139 #[test]
1140 fn test_allow_threads_panics_safely() {
1141 Python::with_gil(|py| {
1142 let result = std::panic::catch_unwind(|| unsafe {
1143 let py = Python::assume_gil_acquired();
1144 py.allow_threads(|| {
1145 panic!("There was a panic!");
1146 });
1147 });
1148
1149 // Check panic was caught
1150 assert!(result.is_err());
1151
1152 // If allow_threads is implemented correctly, this thread still owns the GIL here
1153 // so the following Python calls should not cause crashes.
1154 let list = PyList::new(py, [1, 2, 3, 4]);
1155 assert_eq!(list.extract::<Vec<i32>>().unwrap(), vec![1, 2, 3, 4]);
1156 });
1157 }
1158
1159 #[test]
1160 fn test_allow_threads_pass_stuff_in() {
1161 let list: Py<PyList> = Python::with_gil(|py| {
1162 let list = PyList::new(py, vec!["foo", "bar"]);
1163 list.into()
1164 });
1165 let mut v = vec![1, 2, 3];
1166 let a = Arc::new(String::from("foo"));
1167
1168 Python::with_gil(|py| {
1169 py.allow_threads(|| {
1170 drop((list, &mut v, a));
1171 });
1172 });
1173 }
1174
1175 #[test]
1176 #[cfg(not(Py_LIMITED_API))]
1177 fn test_acquire_gil() {
1178 const GIL_NOT_HELD: c_int = 0;
1179 const GIL_HELD: c_int = 1;
1180
1181 let state = unsafe { crate::ffi::PyGILState_Check() };
1182 assert_eq!(state, GIL_NOT_HELD);
1183
1184 Python::with_gil(|_| {
1185 let state = unsafe { crate::ffi::PyGILState_Check() };
1186 assert_eq!(state, GIL_HELD);
1187 });
1188
1189 let state = unsafe { crate::ffi::PyGILState_Check() };
1190 assert_eq!(state, GIL_NOT_HELD);
1191 }
1192
1193 #[test]
1194 fn test_ellipsis() {
1195 Python::with_gil(|py| {
1196 assert_eq!(py.Ellipsis().to_string(), "Ellipsis");
1197
1198 let v = py
1199 .eval("...", None, None)
1200 .map_err(|e| e.display(py))
1201 .unwrap();
1202
1203 assert!(v.eq(py.Ellipsis()).unwrap());
1204 });
1205 }
1206
1207 #[test]
1208 fn test_py_run_inserts_globals() {
1209 Python::with_gil(|py| {
1210 let namespace = PyDict::new(py);
1211 py.run("class Foo: pass", Some(namespace), Some(namespace))
1212 .unwrap();
1213 assert!(matches!(namespace.get_item("Foo"), Ok(Some(..))));
1214 assert!(matches!(namespace.get_item("__builtins__"), Ok(Some(..))));
1215 })
1216 }
1217}
1218