Skip to content

WIP: Generalize the rpath API for relocatable Python installations #4890

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions build.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ use std::env;

use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result};
use pyo3_build_config::{
add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig,
add_python_link_args, bail, print_feature_cfgs, InterpreterConfig,
};

fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
@@ -44,8 +44,8 @@ fn configure_pyo3() -> Result<()> {
// Emit cfgs like `invalid_from_utf8_lint`
print_feature_cfgs();

// Make `cargo test` etc work on macOS with Xcode bundled Python
add_python_framework_link_args();
// Make `cargo test` etc work on non-global Python installations
add_python_link_args();

Ok(())
}
14 changes: 14 additions & 0 deletions guide/src/getting-started.md
Original file line number Diff line number Diff line change
@@ -120,6 +120,9 @@ crate-type = ["cdylib"]

[dependencies]
pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["extension-module"] }

[build-dependencies]
pyo3-build-config = { {{#PYO3_CRATE_VERSION} }
```

## pyproject.toml
@@ -141,6 +144,17 @@ classifiers = [
]
```

## build.rs

Finally, in order to make `cargo test` work correctly, you should create
a `build.rs` file with the following contents:

```
fn main() {
pyo3_build_config::add_python_link_args();
}
```

## Running code

After this you can setup Rust code to be available in Python as below; for example, you can place this code in `src/lib.rs`:
81 changes: 80 additions & 1 deletion pyo3-build-config/src/impl_.rs
Original file line number Diff line number Diff line change
@@ -167,8 +167,12 @@ pub struct InterpreterConfig {
///
/// Serialized to multiple `extra_build_script_line` values.
pub extra_build_script_lines: Vec<String>,

/// macOS Python3.framework requires special rpath handling
pub python_framework_prefix: Option<String>,

/// Relocatable Python installations require special rpath handling
pub relocatable: bool,
}

impl InterpreterConfig {
@@ -265,6 +269,7 @@ print("calcsize_pointer", struct.calcsize("P"))
print("mingw", get_platform().startswith("mingw"))
print("ext_suffix", get_config_var("EXT_SUFFIX"))
print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
print_if_set("python_build_standalone", get_config_var("PYTHON_BUILD_STANDALONE"))
"#;
let output = run_python_script(interpreter.as_ref(), SCRIPT)?;
let map: HashMap<String, String> = parse_script_output(&output);
@@ -315,6 +320,13 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
_ => panic!("Unknown Py_GIL_DISABLED value"),
};

let relocatable = match map.get("python_build_standalone").map(String::as_str) {
None => false,
Some("0") => false,
Some("1") => true,
_ => panic!("Unknown PYTHON_BUILD_STANDALONE value"),
};

let lib_name = if cfg!(windows) {
default_lib_name_windows(
version,
@@ -365,6 +377,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix,
relocatable,
})
}

@@ -410,6 +423,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
Some(value) => value == "1",
None => false,
};
let relocatable = match sysconfigdata.get_value("python_build_standalone") {
Some(value) => value == "1",
None => false,
};
let lib_name = Some(default_lib_name_unix(
version,
implementation,
@@ -434,6 +451,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix,
relocatable,
})
}

@@ -511,6 +529,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
let mut suppress_build_script_link_lines = None;
let mut extra_build_script_lines = vec![];
let mut python_framework_prefix = None;
let mut relocatable = None;

for (i, line) in lines.enumerate() {
let line = line.context("failed to read line from config")?;
@@ -540,6 +559,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
extra_build_script_lines.push(value.to_string());
}
"python_framework_prefix" => parse_value!(python_framework_prefix, value),
"relocatable" => parse_value!(relocatable, value),
unknown => warn!("unknown config key `{}`", unknown),
}
}
@@ -571,6 +591,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false),
extra_build_script_lines,
python_framework_prefix,
relocatable: relocatable.unwrap_or(false),
})
}

@@ -663,12 +684,13 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
write_option_line!(executable)?;
write_option_line!(pointer_width)?;
write_line!(build_flags)?;
write_option_line!(python_framework_prefix)?;
write_line!(suppress_build_script_link_lines)?;
for line in &self.extra_build_script_lines {
writeln!(writer, "extra_build_script_line={}", line)
.context("failed to write extra_build_script_line")?;
}
write_option_line!(python_framework_prefix)?;
write_line!(relocatable)?;
Ok(())
}

@@ -1601,7 +1623,10 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
build_flags: BuildFlags::default(),
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
// Because this is only used for extensions, we know we do not
// need any rpath handling.
python_framework_prefix: None,
relocatable: false,
})
}

@@ -1644,7 +1669,10 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result<Interpre
build_flags: BuildFlags::default(),
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
// Because this is only used for extensions, we know we do not
// need any rpath handling.
python_framework_prefix: None,
relocatable: false,
})
}

@@ -2028,6 +2056,7 @@ mod tests {
suppress_build_script_link_lines: true,
extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()],
python_framework_prefix: None,
relocatable: false,
};
let mut buf: Vec<u8> = Vec::new();
config.to_writer(&mut buf).unwrap();
@@ -2057,6 +2086,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: true,
};
let mut buf: Vec<u8> = Vec::new();
config.to_writer(&mut buf).unwrap();
@@ -2079,6 +2109,7 @@ mod tests {
suppress_build_script_link_lines: true,
extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()],
python_framework_prefix: None,
relocatable: false,
};
let mut buf: Vec<u8> = Vec::new();
config.to_writer(&mut buf).unwrap();
@@ -2106,6 +2137,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
)
}
@@ -2129,6 +2161,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
)
}
@@ -2232,6 +2265,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}
@@ -2262,6 +2296,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);

@@ -2289,6 +2324,37 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}

#[test]
fn config_from_sysconfigdata_relocatable() {
let mut sysconfigdata = Sysconfigdata::new();
sysconfigdata.insert("SOABI", "cpython-37m-x86_64-linux-gnu");
sysconfigdata.insert("VERSION", "3.7");
sysconfigdata.insert("Py_ENABLE_SHARED", "1");
sysconfigdata.insert("LIBDIR", "/home/xyz/Downloads/python/lib");
sysconfigdata.insert("LDVERSION", "3.7m");
sysconfigdata.insert("SIZEOF_VOID_P", "8");
sysconfigdata.insert("PYTHON_BUILD_STANDALONE", "1");
assert_eq!(
InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap(),
InterpreterConfig {
abi3: false,
build_flags: BuildFlags::from_sysconfigdata(&sysconfigdata),
pointer_width: Some(64),
executable: None,
implementation: PythonImplementation::CPython,
lib_dir: Some("/home/xyz/Downloads/python/lib".into()),
lib_name: Some("python3.7m".into()),
shared: true,
version: PythonVersion::PY37,
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: true,
}
);
}
@@ -2313,6 +2379,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}
@@ -2337,6 +2404,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}
@@ -2372,6 +2440,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relcoatable: false,
}
);
}
@@ -2407,6 +2476,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}
@@ -2442,6 +2512,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable; false,
}
);
}
@@ -2479,6 +2550,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
);
}
@@ -2827,6 +2899,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

config
@@ -2850,6 +2923,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

assert!(config
@@ -2915,6 +2989,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
}
)
}
@@ -3040,6 +3115,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};
assert_eq!(
interpreter_config.build_script_outputs(),
@@ -3080,6 +3156,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

assert_eq!(
@@ -3128,6 +3205,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

assert_eq!(
@@ -3162,6 +3240,7 @@ mod tests {
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

assert_eq!(
92 changes: 77 additions & 15 deletions pyo3-build-config/src/lib.rs
Original file line number Diff line number Diff line change
@@ -74,18 +74,47 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr
}
}

/// Adds linker arguments suitable for linking against the Python framework on macOS.
#[doc(hidden)]
#[deprecated = "Use add_python_link_args instead"]
pub fn add_python_framework_link_args() {
add_python_link_args();
}

/// Adds any linker arguments needed to locate libpython.
///
/// This should be called from a build script.
/// This should be called from a build script. While this is only needed
/// on certain platforms, it is recommended to unconditionally call this
/// from your build.rs to ensure that your crate correctly builds on
/// those platforms.
///
/// This is necessary for any code that dynamically links libpython,
/// including binary crates as well as Rust test cases that call into
/// the Python interpreter. It is not necessary for an extension module,
/// provided that your test cases do not call the Python interpreter.
///
/// The following link flags are added:
/// - macOS: `-Wl,-rpath,<framework_prefix>`
/// - macOS framework builds: `-Wl,-rpath,<framework_prefix>`
/// - UNIX relocatable Python builds: `-Wl,rpath,<lib_dir>`
///
/// All other platforms currently are no-ops.
///
/// Note that this function works by including a reference to the location of
/// the libpython dynamic library into your executable. If your only use of this
/// library is in `cargo test`, or if the Python installation will be in the same
/// place on your build system and production system (e.g., you are building on
/// the same machine where you run, or you are generating a container or system
/// image, or you are building against a Python installation packaged by your
/// operating system), then this is probably the behavior you want. If you are
/// distributing an application to run on other systems, you will likely want to
/// figure out a plan to distribute libpython along with your application and
/// manually set the correct rpath, either by having your build.rs emit the right
/// flags instead of calling this function, or by using a tool like patchelf
/// after the fact, or alternatively you will want to build against a
/// static libpython.
#[cfg(feature = "resolve-config")]
pub fn add_python_framework_link_args() {
pub fn add_python_link_args() {
let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap();
_add_python_framework_link_args(
_add_python_link_args(
&interpreter_config,
&impl_::target_triple_from_env(),
impl_::is_linking_libpython(),
@@ -94,7 +123,7 @@ pub fn add_python_framework_link_args() {
}

#[cfg(feature = "resolve-config")]
fn _add_python_framework_link_args(
fn _add_python_link_args(
interpreter_config: &InterpreterConfig,
triple: &Triple,
link_libpython: bool,
@@ -109,6 +138,15 @@ fn _add_python_framework_link_args(
)
.unwrap();
}
} else if interpreter_config.relocatable && link_libpython && triple.operating_system != OperatingSystem::Windows {
if let Some(lib_dir) = interpreter_config.lib_dir.as_ref() {
writeln!(
writer,
"cargo:rustc-link-arg=-Wl,-rpath,{}",
lib_dir
)
.unwrap();
}
}
}

@@ -347,10 +385,10 @@ mod tests {

#[cfg(feature = "resolve-config")]
#[test]
fn python_framework_link_args() {
fn python_link_args() {
let mut buf = Vec::new();

let interpreter_config = InterpreterConfig {
let base = InterpreterConfig {
implementation: PythonImplementation::CPython,
version: PythonVersion {
major: 3,
@@ -365,28 +403,52 @@ mod tests {
build_flags: BuildFlags::default(),
suppress_build_script_link_lines: false,
extra_build_script_lines: vec![],
python_framework_prefix: None,
relocatable: false,
};

let mac_framework_config = InterpreterConfig {
python_framework_prefix: Some(
"/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(),
),
..base
};
let relocatable_config = InterpreterConfig {
lib_dir: "/home/xyz/Downloads/python/lib",
relocatable: true,
..base
};
// Does nothing on non-mac
_add_python_framework_link_args(
&interpreter_config,
&Triple::from_str("x86_64-pc-windows-msvc").unwrap(),
for interpreter_config in &[mac_framework_config, relocatable_config] {
_add_python_link_args(
&interpreter_config,
&Triple::from_str("x86_64-pc-windows-msvc").unwrap(),
true,
&mut buf,
);
assert_eq!(buf, Vec::new());
}

_add_python_link_args(
&mac_framework_config,
&Triple::from_str("x86_64-apple-darwin").unwrap(),
true,
&mut buf,
);
assert_eq!(buf, Vec::new());
assert_eq!(
std::str::from_utf8(&buf).unwrap(),
"cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n"
);

_add_python_framework_link_args(
&interpreter_config,
_add_python_link_args(
&relocatable_config,
&Triple::from_str("x86_64-apple-darwin").unwrap(),
true,
&mut buf,
);
assert_eq!(
std::str::from_utf8(&buf).unwrap(),
"cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n"
"cargo:rustc-link-arg=-Wl,-rpath,/home/xyz/Downloads/python/lib\n"
);
}
}