Solve in-process symbol problem

Create a python script that emulates being julia and simply hands off control.
This is necessary because the precompilation and runtime environment needs to be
the same in order for PyCall to be able to be used without major modification.

Try a different approach

Only use hack if python is statically linked

New version yet again

Author: Keno

fake-julia/README

@@ -0,0 +1,2 @@
+This directory contains a python script that pretends to be the julia executable
+and is used as such to allow julia precompilation to happen in the same environment.

fake-julia/julia

@@ -0,0 +1,4 @@
+#!/bin/sh
+export PYCALL_JULIA_FLAVOR=julia
+SCRIPTDIR=`cd "$(dirname "$0")" && pwd`
+exec ${PYCALL_PYTHON_EXE:-python} "$SCRIPTDIR/julia.py" -- "$@"

fake-julia/julia-debug

@@ -0,0 +1,4 @@
+#!/bin/sh
+export PYCALL_JULIA_FLAVOR=julia-debug
+SCRIPTDIR=`cd "$(dirname "$0")" && pwd`
+exec ${PYCALL_PYTHON_EXE:-python} "$SCRIPTDIR/julia.py" -- "$@"

fake-julia/julia.py

@@ -0,0 +1,69 @@
+# Minimal repl.c to support precompilation with python symbols already loaded
+import ctypes
+import sys
+import os
+from ctypes import *
+if sys.platform.startswith('darwin'):
+    sh_ext = ".dylib"
+elif sys.platform.startswith('win32'):
+    sh_ext = ".dll"
+else:
+    sh_ext = ".so"
+libjulia = ctypes.CDLL(os.environ["PYCALL_LIBJULIA_PATH"] + "/lib" +
+    os.environ["PYCALL_JULIA_FLAVOR"] + sh_ext, ctypes.RTLD_GLOBAL)
+os.environ["JULIA_HOME"] = os.environ["PYCALL_JULIA_HOME"]
+
+# Set up the calls from libjulia we'll use
+libjulia.jl_parse_opts.argtypes = [POINTER(c_int), POINTER(POINTER(c_char_p))]
+libjulia.jl_parse_opts.restype = None
+libjulia.jl_init.argtypes = [c_void_p]
+libjulia.jl_init.restype = None
+libjulia.jl_get_global.argtypes = [c_void_p,c_void_p]
+libjulia.jl_get_global.restype = c_void_p
+libjulia.jl_symbol.argtypes = [c_char_p]
+libjulia.jl_symbol.restype = c_void_p
+libjulia.jl_apply_generic.argtypes = [POINTER(c_void_p), c_int]
+libjulia.jl_apply_generic.restype = c_void_p
+libjulia.jl_set_ARGS.argtypes = [c_int, POINTER(c_char_p)]
+libjulia.jl_set_ARGS.restype = None
+libjulia.jl_atexit_hook.argtypes = [c_int]
+libjulia.jl_atexit_hook.restype = None
+libjulia.jl_eval_string.argtypes = [c_char_p]
+libjulia.jl_eval_string.restype = None
+
+# Ok, go
+argc = c_int(len(sys.argv)-1)
+argv = (c_char_p * (len(sys.argv)-1))()
+if sys.version_info[0] < 3:
+    argv_strings = sys.argv
+else:
+    argv_strings = [str.encode('utf-8') for str in sys.argv]
+argv[1:] = argv_strings[2:]
+argv[0] = argv_strings[0]
+argv2 = (POINTER(c_char_p) * 1)()
+argv2[0] = ctypes.cast(ctypes.addressof(argv),POINTER(c_char_p))
+libjulia.jl_parse_opts(byref(argc),argv2)
+libjulia.jl_init(0)
+libjulia.jl_set_ARGS(argc,argv2[0])
+jl_base_module = c_void_p.in_dll(libjulia, "jl_base_module")
+_start = libjulia.jl_get_global(jl_base_module, libjulia.jl_symbol(b"_start"))
+args = (c_void_p * 1)()
+args[0] = _start
+libjulia.jl_apply_generic(args, 1)
+libjulia.jl_atexit_hook(0)
+
+# As an optimization, share precompiled packages with the main cache directory
+libjulia.jl_eval_string(b"""
+    outputji = Base.JLOptions().outputji
+    if outputji != C_NULL && !isdefined(Main, :PyCall)
+        outputfile = unsafe_string(outputji)
+        target = Base.LOAD_CACHE_PATH[2]
+        targetpath = joinpath(target, basename(outputfile))
+        if is_windows()
+            cp(outputfile, targetpath)
+        else
+            mv(outputfile, targetpath; remove_destination = true)
+            symlink(targetpath, outputfile)
+        end
+    end
+""")

julia/core.py

@@ -48,6 +48,7 @@ class JuliaModule(ModuleType):
     pass
 
 
+
 # add custom import behavior for the julia "module"
 class JuliaImporter(object):
     def __init__(self, julia):
@@ -192,6 +193,15 @@ def module_functions(julia, module):
             pass
     return bases
 
+def determine_if_statically_linked():
+    """Determines if this python executable is statically linked"""
+    # Windows and OS X are generally always dynamically linked
+    if not sys.platform.startswith('linux'):
+        return False
+    lddoutput = subprocess.check_output(["ldd",sys.executable])
+    return not ("libpython" in lddoutput)
+
+
 _julia_runtime = [False]
 
 class Julia(object):
@@ -246,9 +256,20 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
                 [runtime, "-e",
                  """
                  println(JULIA_HOME)
-                 println(Libdl.dlpath("libjulia"))
+                 println(Libdl.dlpath(string("lib",Base.julia_exename())))
+                 PyCall_depsfile = Pkg.dir("PyCall","deps","deps.jl")
+                 if isfile(PyCall_depsfile)
+                    eval(Module(:__anon__),
+                        Expr(:toplevel,
+                         :(using Compat),
+                         :(Main.Base.include($PyCall_depsfile)),
+                         :(println(python))))
+                 else
+                    println("nowhere")
+                 end
                  """])
-            JULIA_HOME, libjulia_path = juliainfo.decode("utf-8").rstrip().split("\n")
+            JULIA_HOME, libjulia_path, depsjlexe = juliainfo.decode("utf-8").rstrip().split("\n")
+            exe_differs = not depsjlexe == sys.executable
             self._debug("JULIA_HOME = %s,  libjulia_path = %s" % (JULIA_HOME, libjulia_path))
             if not os.path.exists(libjulia_path):
                 raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path))
@@ -297,14 +318,37 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None,
         self.api.jl_exception_clear()
 
         if init_julia:
-            # Replace the cache directory with a private one. PyCall needs a different
-            # configuration and so do any packages that depend on it. Ideally, we could
-            # detect packages that depend on PyCall and only use LOAD_CACHE_PATH for them
-            # but that would be significantly more complicated and brittle, and may not
-            # be worth it.
-            self._call(u"empty!(Base.LOAD_CACHE_PATH)")
-            self._call(u"push!(Base.LOAD_CACHE_PATH, abspath(Pkg.Dir._pkgroot()," +
-                "\"lib\", \"pyjulia-v$(VERSION.major).$(VERSION.minor)\"))")
+            use_separate_cache = exe_differs or determine_if_statically_linked()
+            if use_separate_cache:
+                # First check that this is supported
+                self._call("""
+                    if VERSION < v"0.5-"
+                        error(\"""Using pyjulia with a statically-compiled version
+                                  of python or with a version of python that
+                                  differs from that used by PyCall.jl is not
+                                  supported on julia 0.4""\")
+                    end
+                """)
+                # Intercept precompilation
+                os.environ["PYCALL_PYTHON_EXE"] = sys.executable
+                PYCALL_JULIA_HOME = os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)),"..","fake-julia").replace("\\","\\\\")
+                os.environ["PYCALL_JULIA_HOME"] = PYCALL_JULIA_HOME
+                os.environ["PYCALL_LIBJULIA_PATH"] = os.path.dirname(libjulia_path)
+                self._call(u"eval(Base,:(JULIA_HOME=\""+PYCALL_JULIA_HOME+"\"))")
+                # Add a private cache directory. PyCall needs a different
+                # configuration and so do any packages that depend on it.
+                self._call(u"unshift!(Base.LOAD_CACHE_PATH, abspath(Pkg.Dir._pkgroot()," +
+                    "\"lib\", \"pyjulia%s-v$(VERSION.major).$(VERSION.minor)\"))" % sys.version_info[0])
+                # If PyCall.ji does not exist, create an empty file to force
+                # recompilation
+                self._call(u"""
+                    isdir(Base.LOAD_CACHE_PATH[1]) ||
+                        mkpath(Base.LOAD_CACHE_PATH[1])
+                    depsfile = joinpath(Base.LOAD_CACHE_PATH[1],"PyCall.ji")
+                    isfile(depsfile) || touch(depsfile)
+                """)
+
             self._call(u"using PyCall")
         # Whether we initialized Julia or not, we MUST create at least one
         # instance of PyObject and the convert function. Since these will be