Basic GraalPy Support (#3247)

* graalpy: recognize graalpy implementation when building

* graalpy: global Ellipse, None, NotImplemented, True, and False are only available as pointers

* graalpy: PyObject struct is opaque, use functions for everything

* graalpy: missing many of the same functions as pypy

* graalpy: do not have 128bit conversion functions

* graalpy: add functions for datetime accessor macros

* graalpy: add implementations for list macro functions

* graalpy: skip tuple macros

* graalpy: always use extern Py_CompileString function

* graalpy: disable assertion that does not apply to graalpy

* graalpy: floatobject structure is opaque on graalpy

* graalpy: ignore gc dependent test

* graalpy: add CI config

* graalpy: run rust fmt

* graalpy: add changelog entry

* graalpy: discover interpreter on PATH

* graalpy: interpreter id is not applicable to graalpy (just like pypy)

* graalpy: skip tests that cannot work on GraalPy

* graalpy: fix constructing normalized Err instances

Co-authored-by: David Hewitt <[email protected]>

* graalpy: correct capi library name, but skip rust tests due to missing symbols

* graalpy: no support for C extensions on windows in latest release

* graalpy: declare support versions

* graalpy: frame, code, method, and function objects access from C API is mostly missing

* graalpy: take care only to expose C structure that GraalPy allocates

* graalpy: Bail out if graalpy version is less than what we support

---------

Co-authored-by: David Hewitt <[email protected]>

Author: timfel

.github/workflows/build.yml

@@ -24,6 +24,7 @@ jobs:
   build:
     continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) || inputs.rust == 'beta' || inputs.rust == 'nightly' }}
     runs-on: ${{ inputs.os }}
+    if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }}
     steps:
       - uses: actions/checkout@v4
 
@@ -65,6 +66,7 @@ jobs:
         run: nox -s docs
 
       - name: Build (no features)
+        if: ${{ !startsWith(inputs.python-version, 'graalpy') }}
         run: cargo build --lib --tests --no-default-features
 
       # --no-default-features when used with `cargo build/test -p` doesn't seem to work!
@@ -74,7 +76,7 @@ jobs:
           cargo build --no-default-features
 
       # Run tests (except on PyPy, because no embedding API).
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test (no features)
         run: cargo test --no-default-features --lib --tests
 
@@ -85,24 +87,25 @@ jobs:
           cargo test --no-default-features
 
       - name: Build (all additive features)
+        if: ${{ !startsWith(inputs.python-version, 'graalpy') }}
         run: cargo build --lib --tests --no-default-features --features "full ${{ inputs.extra-features }}"
 
       - if: ${{ startsWith(inputs.python-version, 'pypy') }}
         name: Build PyPy (abi3-py37)
         run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}"
 
       # Run tests (except on PyPy, because no embedding API).
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test
         run: cargo test --no-default-features --features "full ${{ inputs.extra-features }}"
 
       # Run tests again, but in abi3 mode
-      - if: ${{ !startsWith(inputs.python-version, 'pypy') }}
+      - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }}
         name: Test (abi3)
         run: cargo test --no-default-features --features "abi3 full ${{ inputs.extra-features }}"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
-      - if: ${{ (!startsWith(inputs.python-version, 'pypy')) && (inputs.python-version != '3.7') }}
+      - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }}
         name: Test (abi3-py37)
         run: cargo test --no-default-features --features "abi3-py37 full ${{ inputs.extra-features }}"
 
@@ -120,7 +123,7 @@ jobs:
 
       - uses: dorny/paths-filter@v3
         # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here
-        if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }}
+        if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') }}
         id: ffi-changes
         with:
           base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }}
@@ -135,7 +138,7 @@ jobs:
       - name: Run pyo3-ffi-check
         # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor
         # is pypy 3.9 on windows
-        if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
+        if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
         run: nox -s ffi-check
 
     env:

.github/workflows/ci.yml

@@ -228,6 +228,7 @@ jobs:
           "pypy3.8",
           "pypy3.9",
           "pypy3.10",
+          "graalpy24.0",
         ]
         platform:
           [

README.md

@@ -17,7 +17,7 @@
 ## Usage
 
 PyO3 supports the following software versions:
-  - Python 3.7 and up (CPython and PyPy)
+  - Python 3.7 and up (CPython, PyPy, and GraalPy)
   - Rust 1.56 and up
 
 You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn.

newsfragments/3247.added.md

@@ -0,0 +1 @@
+Added support for running PyO3 extensions on GraalPy.

pyo3-build-config/src/impl_.rs

@@ -32,6 +32,12 @@ use crate::{
 /// Minimum Python version PyO3 supports.
 const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 };
 
+/// GraalPy may implement the same CPython version over multiple releases.
+const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion {
+    major: 24,
+    minor: 0,
+};
+
 /// Maximum Python version that can be used as minimum required Python version with abi3.
 const ABI3_MAX_MINOR: u8 = 12;
 
@@ -173,6 +179,11 @@ impl InterpreterConfig {
                     See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information."
                 ));
             }
+        } else if self.implementation.is_graalpy() {
+            println!("cargo:rustc-cfg=GraalPy");
+            if self.abi3 {
+                warn!("GraalPy does not support abi3 so the build artifacts will be version-specific.");
+            }
         } else if self.abi3 {
             out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned());
         }
@@ -197,6 +208,12 @@ import sys
 from sysconfig import get_config_var, get_platform
 
 PYPY = platform.python_implementation() == "PyPy"
+GRAALPY = platform.python_implementation() == "GraalVM"
+
+if GRAALPY:
+    graalpy_ver = map(int, __graalpython__.get_graalvm_version().split('.'));
+    print("graalpy_major", next(graalpy_ver))
+    print("graalpy_minor", next(graalpy_ver))
 
 # sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue
 # so that the version mismatch can be reported in a nicer way later.
@@ -226,7 +243,7 @@ SHARED = bool(get_config_var("Py_ENABLE_SHARED"))
 print("implementation", platform.python_implementation())
 print("version_major", sys.version_info[0])
 print("version_minor", sys.version_info[1])
-print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED)
+print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED)
 print_if_set("ld_version", get_config_var("LDVERSION"))
 print_if_set("libdir", get_config_var("LIBDIR"))
 print_if_set("base_prefix", base_prefix)
@@ -244,6 +261,23 @@ print("ext_suffix", get_config_var("EXT_SUFFIX"))
             interpreter.as_ref().display()
         );
 
+        if let Some(value) = map.get("graalpy_major") {
+            let graalpy_version = PythonVersion {
+                major: value
+                    .parse()
+                    .context("failed to parse GraalPy major version")?,
+                minor: map["graalpy_minor"]
+                    .parse()
+                    .context("failed to parse GraalPy minor version")?,
+            };
+            ensure!(
+                graalpy_version >= MINIMUM_SUPPORTED_VERSION_GRAALPY,
+                "At least GraalPy version {} needed, got {}",
+                MINIMUM_SUPPORTED_VERSION_GRAALPY,
+                graalpy_version
+            );
+        };
+
         let shared = map["shared"].as_str() == "True";
 
         let version = PythonVersion {
@@ -588,7 +622,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX"))
     /// Lowers the configured version to the abi3 version, if set.
     fn fixup_for_abi3_version(&mut self, abi3_version: Option<PythonVersion>) -> Result<()> {
         // PyPy doesn't support abi3; don't adjust the version
-        if self.implementation.is_pypy() {
+        if self.implementation.is_pypy() || self.implementation.is_graalpy() {
             return Ok(());
         }
 
@@ -647,6 +681,7 @@ impl FromStr for PythonVersion {
 pub enum PythonImplementation {
     CPython,
     PyPy,
+    GraalPy,
 }
 
 impl PythonImplementation {
@@ -655,12 +690,19 @@ impl PythonImplementation {
         self == PythonImplementation::PyPy
     }
 
+    #[doc(hidden)]
+    pub fn is_graalpy(self) -> bool {
+        self == PythonImplementation::GraalPy
+    }
+
     #[doc(hidden)]
     pub fn from_soabi(soabi: &str) -> Result<Self> {
         if soabi.starts_with("pypy") {
             Ok(PythonImplementation::PyPy)
         } else if soabi.starts_with("cpython") {
             Ok(PythonImplementation::CPython)
+        } else if soabi.starts_with("graalpy") {
+            Ok(PythonImplementation::GraalPy)
         } else {
             bail!("unsupported Python interpreter");
         }
@@ -672,6 +714,7 @@ impl Display for PythonImplementation {
         match self {
             PythonImplementation::CPython => write!(f, "CPython"),
             PythonImplementation::PyPy => write!(f, "PyPy"),
+            PythonImplementation::GraalPy => write!(f, "GraalVM"),
         }
     }
 }
@@ -682,6 +725,7 @@ impl FromStr for PythonImplementation {
         match s {
             "CPython" => Ok(PythonImplementation::CPython),
             "PyPy" => Ok(PythonImplementation::PyPy),
+            "GraalVM" => Ok(PythonImplementation::GraalPy),
             _ => bail!("unknown interpreter: {}", s),
         }
     }
@@ -760,7 +804,7 @@ pub struct CrossCompileConfig {
     /// The version of the Python library to link against.
     version: Option<PythonVersion>,
 
-    /// The target Python implementation hint (CPython or PyPy)
+    /// The target Python implementation hint (CPython, PyPy, GraalPy, ...)
     implementation: Option<PythonImplementation>,
 
     /// The compile target triple (e.g. aarch64-unknown-linux-gnu)
@@ -1264,6 +1308,15 @@ fn is_pypy_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
     path == "lib_pypy" || path.starts_with(&pypy_version_pat)
 }
 
+fn is_graalpy_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
+    let graalpy_version_pat = if let Some(v) = v {
+        format!("graalpy{}", v)
+    } else {
+        "graalpy2".into()
+    };
+    path == "lib_graalpython" || path.starts_with(&graalpy_version_pat)
+}
+
 fn is_cpython_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
     let cpython_version_pat = if let Some(v) = v {
         format!("python{}", v)
@@ -1297,6 +1350,7 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
                     search_lib_dir(f.path(), cross)
                 } else if is_cpython_lib_dir(&file_name, &cross.version)
                     || is_pypy_lib_dir(&file_name, &cross.version)
+                    || is_graalpy_lib_dir(&file_name, &cross.version)
                 {
                     search_lib_dir(f.path(), cross)
                 } else {
@@ -1418,7 +1472,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
 ///
 /// Must be called from a PyO3 crate build script.
 fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConfig {
-    // FIXME: PyPy does not support the Stable ABI yet.
+    // FIXME: PyPy & GraalPy do not support the Stable ABI.
     let implementation = PythonImplementation::CPython;
     let abi3 = true;
 
@@ -1524,7 +1578,7 @@ fn default_lib_name_windows(
         // CPython bug: linking against python3_d.dll raises error
         // https://github.com/python/cpython/issues/101614
         format!("python{}{}_d", version.major, version.minor)
-    } else if abi3 && !implementation.is_pypy() {
+    } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) {
         WINDOWS_ABI3_LIB_NAME.to_owned()
     } else if mingw {
         // https://packages.msys2.org/base/mingw-w64-python
@@ -1562,6 +1616,7 @@ fn default_lib_name_unix(
                 format!("pypy{}-c", version.major)
             }
         }
+        PythonImplementation::GraalPy => "python-native".to_string(),
     }
 }
 
@@ -1662,7 +1717,9 @@ pub fn find_interpreter() -> Result<PathBuf> {
             .find(|bin| {
                 if let Ok(out) = Command::new(bin).arg("--version").output() {
                     // begin with `Python 3.X.X :: additional info`
-                    out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3")
+                    out.stdout.starts_with(b"Python 3")
+                        || out.stderr.starts_with(b"Python 3")
+                        || out.stdout.starts_with(b"GraalPy 3")
                 } else {
                     false
                 }

pyo3-build-config/src/import_lib.rs

@@ -7,7 +7,7 @@ use python3_dll_a::ImportLibraryGenerator;
 use target_lexicon::{Architecture, OperatingSystem, Triple};
 
 use super::{PythonImplementation, PythonVersion};
-use crate::errors::{Context, Result};
+use crate::errors::{Context, Error, Result};
 
 /// Generates the `python3.dll` or `pythonXY.dll` import library for Windows targets.
 ///
@@ -42,6 +42,9 @@ pub(super) fn generate_import_lib(
     let implementation = match py_impl {
         PythonImplementation::CPython => python3_dll_a::PythonImplementation::CPython,
         PythonImplementation::PyPy => python3_dll_a::PythonImplementation::PyPy,
+        PythonImplementation::GraalPy => {
+            return Err(Error::from("No support for GraalPy on Windows"))
+        }
     };
 
     ImportLibraryGenerator::new(&arch, &env)

pyo3-build-config/src/lib.rs

@@ -38,6 +38,7 @@ use target_lexicon::OperatingSystem;
 /// | `#[cfg(Py_3_7)]`, `#[cfg(Py_3_8)]`, `#[cfg(Py_3_9)]`, `#[cfg(Py_3_10)]` | These attributes mark code only for a given Python version and up. For example, `#[cfg(Py_3_7)]` marks code which can run on Python 3.7 **and newer**. |
 /// | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. |
 /// | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. |
+/// | `#[cfg(GraalPy)]` | This marks code which is run when compiling for GraalPy. |
 ///
 /// For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building-and-distribution/multiple_python_versions.html).
 #[cfg(feature = "resolve-config")]

pyo3-ffi/build.rs

@@ -29,6 +29,17 @@ const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions {
     },
 };
 
+const SUPPORTED_VERSIONS_GRAALPY: SupportedVersions = SupportedVersions {
+    min: PythonVersion {
+        major: 3,
+        minor: 10,
+    },
+    max: PythonVersion {
+        major: 3,
+        minor: 11,
+    },
+};
+
 fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
     // This is an undocumented env var which is only really intended to be used in CI / for testing
     // and development.
@@ -73,6 +84,24 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
                 std::env::var("CARGO_PKG_VERSION").unwrap()
             );
         }
+        PythonImplementation::GraalPy => {
+            let versions = SUPPORTED_VERSIONS_GRAALPY;
+            ensure!(
+                interpreter_config.version >= versions.min,
+                "the configured GraalPy interpreter version ({}) is lower than PyO3's minimum supported version ({})",
+                interpreter_config.version,
+                versions.min,
+            );
+            // GraalPy does not support abi3, so we cannot offer forward compatibility
+            ensure!(
+                interpreter_config.version <= versions.max,
+                "the configured GraalPy interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\
+                 = help: please check if an updated version of PyO3 is available. Current version: {}",
+                interpreter_config.version,
+                versions.max,
+                std::env::var("CARGO_PKG_VERSION").unwrap()
+            );
+        }
     }
 
     Ok(())

pyo3-ffi/src/abstract_.rs

@@ -23,6 +23,7 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_
 extern "C" {
     #[cfg(all(
         not(PyPy),
+        not(GraalPy),
         any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10
     ))]
     #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")]