Skip to content

Commit 1825d34

Browse files
committed
Make PyJulia usable in virtual environments
1 parent 54d1116 commit 1825d34

File tree

3 files changed

+235
-27
lines changed

3 files changed

+235
-27
lines changed

julia/core.py

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
# this is python 3.3 specific
3333
from types import ModuleType, FunctionType
3434

35+
from .find_libpython import find_libpython, normalize_path
36+
3537
#-----------------------------------------------------------------------------
3638
# Classes and funtions
3739
#-----------------------------------------------------------------------------
@@ -260,7 +262,11 @@ def determine_if_statically_linked():
260262

261263
JuliaInfo = namedtuple(
262264
'JuliaInfo',
263-
['JULIA_HOME', 'libjulia_path', 'image_file', 'pyprogramname'])
265+
['JULIA_HOME', 'libjulia_path', 'image_file',
266+
# Variables in PyCall/deps/deps.jl:
267+
'pyprogramname', 'libpython'],
268+
# PyCall/deps/deps.jl may not exist; The variables are then set to None:
269+
defaults=[None, None])
264270

265271

266272
def juliainfo(runtime='julia'):
@@ -283,28 +289,39 @@ def juliainfo(runtime='julia'):
283289
if PyCall_depsfile !== nothing && isfile(PyCall_depsfile)
284290
include(PyCall_depsfile)
285291
println(pyprogramname)
292+
println(libpython)
286293
end
287294
"""],
288295
# Use the original environment variables to avoid a cryptic
289296
# error "fake-julia/../lib/julia/sys.so: cannot open shared
290297
# object file: No such file or directory":
291298
env=_enviorn)
292299
args = output.decode("utf-8").rstrip().split("\n")
293-
if len(args) == 3:
294-
args.append(None) # no pyprogramname set
295300
return JuliaInfo(*args)
296301

297302

298-
def is_same_path(a, b):
299-
a = os.path.normpath(os.path.normcase(a))
300-
b = os.path.normpath(os.path.normcase(b))
301-
return a == b
303+
def is_compatible_exe(jlinfo):
304+
"""
305+
Determine if Python used by PyCall.jl is compatible with this Python.
306+
307+
Current Python executable is considered compatible if it is dynamically
308+
linked to libpython (usually the case in macOS and Windows) and
309+
both of them are using identical libpython. If this function returns
310+
`True`, PyJulia use the same precompilation cache of PyCall.jl used by
311+
Julia itself.
312+
313+
Parameters
314+
----------
315+
jlinfo : JuliaInfo
316+
A `JuliaInfo` object returned by `juliainfo` function.
317+
"""
318+
if jlinfo.libpython is None:
319+
return False
302320

321+
if determine_if_statically_linked():
322+
return False
303323

304-
def is_different_exe(pyprogramname, sys_executable):
305-
if pyprogramname is None:
306-
return True
307-
return not is_same_path(pyprogramname, sys_executable)
324+
return find_libpython() == normalize_path(jlinfo.libpython)
308325

309326

310327
_julia_runtime = [False]
@@ -359,11 +376,10 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
359376
runtime = jl_runtime_path
360377
else:
361378
runtime = 'julia'
362-
JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo(runtime)
379+
jlinfo = juliainfo(runtime)
380+
JULIA_HOME, libjulia_path, image_file, depsjlexe = jlinfo[:4]
363381
self._debug("pyprogramname =", depsjlexe)
364382
self._debug("sys.executable =", sys.executable)
365-
exe_differs = is_different_exe(depsjlexe, sys.executable)
366-
self._debug("exe_differs =", exe_differs)
367383
self._debug("JULIA_HOME = %s, libjulia_path = %s" % (JULIA_HOME, libjulia_path))
368384
if not os.path.exists(libjulia_path):
369385
raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path))
@@ -381,7 +397,8 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
381397
else:
382398
jl_init_path = JULIA_HOME.encode("utf-8") # initialize with JULIA_HOME
383399

384-
use_separate_cache = exe_differs or determine_if_statically_linked()
400+
use_separate_cache = not is_compatible_exe(jlinfo)
401+
self._debug("use_separate_cache =", use_separate_cache)
385402
if use_separate_cache:
386403
PYCALL_JULIA_HOME = os.path.join(
387404
os.path.dirname(os.path.realpath(__file__)),"fake-julia").replace("\\","\\\\")

julia/find_libpython.py

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
Locate libpython associated with this Python executable.
5+
"""
6+
7+
from __future__ import print_function
8+
9+
from logging import getLogger
10+
import ctypes.util
11+
import os
12+
import platform
13+
import sys
14+
import sysconfig
15+
16+
logger = getLogger("find_libpython")
17+
18+
SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so"
19+
20+
21+
def library_name(name, suffix=SHLIB_SUFFIX,
22+
is_windows=platform.system() == "Windows"):
23+
"""
24+
Convert a file basename `name` to a library name (no "lib" and ".so" etc.)
25+
26+
>>> library_name("libpython3.7m.so") # doctest: +SKIP
27+
'python3.7m'
28+
>>> library_name("libpython3.7m.so", suffix=".so", is_windows=False)
29+
'python3.7m'
30+
>>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False)
31+
'python3.7m'
32+
>>> library_name("python37.dll", suffix=".dll", is_windows=True)
33+
'python37'
34+
"""
35+
if not is_windows:
36+
name = name[len("lib"):]
37+
if suffix and name.endswith(suffix):
38+
name = name[:-len(suffix)]
39+
return name
40+
41+
42+
def append_truthy(list, item):
43+
if item:
44+
list.append(item)
45+
46+
47+
def libpython_candidates(suffix=SHLIB_SUFFIX):
48+
"""
49+
Iterate over candidate paths of libpython.
50+
51+
Yields
52+
------
53+
path : str or None
54+
Candidate path to libpython. The path may not be a fullpath
55+
and may not exist.
56+
"""
57+
is_windows = platform.system() == "Windows"
58+
59+
# List candidates for libpython basenames
60+
lib_basenames = []
61+
append_truthy(lib_basenames, sysconfig.get_config_var("LDLIBRARY"))
62+
63+
LIBRARY = sysconfig.get_config_var("LIBRARY")
64+
if LIBRARY:
65+
lib_basenames.append(os.path.splitext(LIBRARY)[0] + suffix)
66+
67+
dlprefix = "" if is_windows else "lib"
68+
sysdata = dict(
69+
v=sys.version_info,
70+
abiflags=(sysconfig.get_config_var("ABIFLAGS") or
71+
sysconfig.get_config_var("abiflags") or ""),
72+
)
73+
lib_basenames.extend(dlprefix + p + suffix for p in [
74+
"python{v.major}.{v.minor}{abiflags}".format(**sysdata),
75+
"python{v.major}.{v.minor}".format(**sysdata),
76+
"python{v.major}".format(**sysdata),
77+
"python",
78+
])
79+
80+
# List candidates for directories in which libpython may exist
81+
lib_dirs = []
82+
append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR"))
83+
84+
if is_windows:
85+
lib_dirs.append(os.path.join(os.path.dirname(sys.executable)))
86+
else:
87+
lib_dirs.append(os.path.join(
88+
os.path.dirname(os.path.dirname(sys.executable)),
89+
"lib"))
90+
91+
# For macOS:
92+
append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX"))
93+
94+
lib_dirs.append(sys.exec_prefix)
95+
lib_dirs.append(os.path.join(sys.exec_prefix, "lib"))
96+
97+
for directory in lib_dirs:
98+
for basename in lib_basenames:
99+
yield os.path.join(directory, basename)
100+
101+
# In macOS and Windows, ctypes.util.find_library returns a full path:
102+
for basename in lib_basenames:
103+
yield ctypes.util.find_library(library_name(basename))
104+
105+
106+
def normalize_path(path, suffix=SHLIB_SUFFIX):
107+
"""
108+
Normalize shared library `path` to a real path.
109+
110+
If `path` is not a full path, `None` is returned. If `path` does
111+
not exists, append `SHLIB_SUFFIX` and check if it exists.
112+
Finally, the path is canonicalized by following the symlinks.
113+
114+
Parameters
115+
----------
116+
path : str ot None
117+
A candidate path to a shared library.
118+
"""
119+
if not path:
120+
return None
121+
if not os.path.isabs(path):
122+
return None
123+
if os.path.exists(path):
124+
return os.path.realpath(path)
125+
if os.path.exists(path + suffix):
126+
return os.path.realpath(path + suffix)
127+
return None
128+
129+
130+
def finding_libpython():
131+
"""
132+
Iterate over existing libpython paths.
133+
134+
The first item is likely to be the best one. It may yield
135+
duplicated paths.
136+
137+
Yields
138+
------
139+
path : str
140+
Existing path to a libpython.
141+
"""
142+
for path in libpython_candidates():
143+
logger.debug("Candidate: %s", path)
144+
normalized = normalize_path(path)
145+
logger.debug("Normalized: %s", normalized)
146+
if normalized:
147+
logger.debug("Found: %s", normalized)
148+
yield normalized
149+
150+
151+
def find_libpython():
152+
"""
153+
Return a path (`str`) to libpython or `None` if not found.
154+
155+
Parameters
156+
----------
157+
path : str or None
158+
Existing path to the (supposedly) correct libpython.
159+
"""
160+
for path in finding_libpython():
161+
return os.path.realpath(path)
162+
163+
164+
def cli_find_libpython(verbose, list_all):
165+
import logging
166+
# Importing `logging` module here so that using `logging.debug`
167+
# instead of `logger.debug` outside of this function becomes an
168+
# error.
169+
170+
if verbose:
171+
logging.basicConfig(level=logging.DEBUG)
172+
173+
if list_all:
174+
for path in finding_libpython():
175+
print(path)
176+
return
177+
178+
path = find_libpython()
179+
if path is None:
180+
return 1
181+
print(path, end="")
182+
183+
184+
def main(args=None):
185+
import argparse
186+
parser = argparse.ArgumentParser(
187+
description=__doc__)
188+
parser.add_argument(
189+
"--verbose", "-v", action="store_true",
190+
help="Print debugging information.")
191+
parser.add_argument(
192+
"--list-all", action="store_true",
193+
help="Print list of all paths found.")
194+
ns = parser.parse_args(args)
195+
parser.exit(cli_find_libpython(**vars(ns)))
196+
197+
198+
if __name__ == "__main__":
199+
main()

test/test_utils.py

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,9 @@
22
Unit tests which can be done without loading `libjulia`.
33
"""
44

5-
import sys
5+
from julia.find_libpython import finding_libpython
66

7-
import pytest
87

9-
from julia.core import is_different_exe
10-
11-
12-
@pytest.mark.parametrize('pyprogramname, sys_executable, exe_differs', [
13-
(sys.executable, sys.executable, False),
14-
(None, sys.executable, True),
15-
('/dev/null', sys.executable, True),
16-
])
17-
def test_is_different_exe(pyprogramname, sys_executable, exe_differs):
18-
assert is_different_exe(pyprogramname, sys_executable) == exe_differs
8+
def test_smoke_finding_libpython():
9+
paths = list(finding_libpython())
10+
assert set(map(type, paths)) == {str}

0 commit comments

Comments
 (0)