Refactor (#360)

* factor out _CPython module

* factor out _Py module

* factor out _pyconvert, _pymacro and _pywrap modules

* move pyconst macro

* _Py module now loads Python juliacall module

* pyconvert now forward-declared in _Py

* standard module layout

* don't depend on top-level of PythonCall

* factor out _jlwrap module

* adds conversion rules for ctypes, numpy and pandas

* compat

* case hack

* case hack pt 2

* test coverage

* add non-exported conversion API

* tests

---------

Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: cjdoris

pysrc/juliacall/ipython.py

@@ -69,12 +69,12 @@ def load_ipython_extension(ip):
     # redirect stdout/stderr
     if ip.__class__.__name__ == 'TerminalInteractiveShell':
         # no redirection in the terminal
-        PythonCall.seval("""begin
+        PythonCall.seval("""module _ipython
             function _flush_stdio()
             end
         end""")
     else:
-        PythonCall.seval("""begin
+        PythonCall.seval("""module _ipython
             const _redirected_stdout = redirect_stdout()
             const _redirected_stderr = redirect_stderr()
             const _py_stdout = PyIO(pyimport("sys" => "stdout"); line_buffering=true)
@@ -92,10 +92,10 @@ def load_ipython_extension(ip):
             end
             nothing
         end""")
-    ip.events.register('post_execute', PythonCall._flush_stdio)
+    ip.events.register('post_execute', PythonCall._ipython._flush_stdio)
     # push displays
     PythonCall.seval("""begin
-        pushdisplay(PythonDisplay())
-        pushdisplay(IPythonDisplay())
+        pushdisplay(_compat.PythonDisplay())
+        pushdisplay(_compat.IPythonDisplay())
         nothing
     end""")

src/CPython/_.jl

@@ -0,0 +1,30 @@
+"""
+    module _CPython
+
+This module provides a direct interface to the Python C API.
+"""
+module _CPython
+
+using Base: @kwdef
+using UnsafePointers: UnsafePtr
+using CondaPkg: CondaPkg
+using Pkg: Pkg
+using Requires: @require
+using Libdl: dlpath, dlopen, dlopen_e, dlclose, dlsym, dlsym_e, RTLD_LAZY, RTLD_DEEPBIND, RTLD_GLOBAL
+
+# import Base: @kwdef
+# import CondaPkg
+# import Pkg
+# using Libdl, Requires, UnsafePointers, Serialization, ..Utils
+
+include("consts.jl")
+include("pointers.jl")
+include("extras.jl")
+include("context.jl")
+include("gil.jl")
+
+function __init__()
+    init_context()
+end
+
+end

src/cpython/consts.jl renamed to src/CPython/consts.jl

@@ -324,12 +324,6 @@ const PyTypePtr = Ptr{PyTypeObject}
     value::T
 end
 
-@kwdef struct PyJuliaValueObject
-    ob_base::PyObject = PyObject()
-    value::Int = 0
-    weaklist::PyPtr = C_NULL
-end
-
 @kwdef struct PyArrayInterface
     two::Cint = 0
     nd::Cint = 0

src/cpython/context.jl renamed to src/CPython/context.jl

@@ -137,11 +137,14 @@ function init_context()
         # Get function pointers from the library
         init_pointers()
 
+        # Compare libpath with PyCall
+        @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
+
         # Initialize the interpreter
         with_gil() do
             CTX.is_preinitialized = Py_IsInitialized() != 0
             if CTX.is_preinitialized
-                @assert CTX.which == :PyCall
+                @assert CTX.which == :PyCall || CTX.matches_pycall isa Bool
             else
                 @assert CTX.which != :PyCall
                 # Find ProgramName and PythonHome
@@ -211,9 +214,6 @@ function init_context()
         ENV["JULIA_PYTHONCALL_EXE"] = CTX.exe_path::String
     end
 
-    # Compare libpath with PyCall
-    @require PyCall="438e738f-606a-5dbb-bf0a-cddfbfd45ab0" init_pycall(PyCall)
-
     with_gil() do
 
         # Get the python version

src/cpython/extras.jl renamed to src/CPython/extras.jl

@@ -274,8 +274,6 @@ const CAPI_OBJECTS = Set([
     $([:($name :: PyPtr = C_NULL) for name in CAPI_EXCEPTIONS]...)
     $([:($name :: PyPtr = C_NULL) for name in CAPI_OBJECTS]...)
     PyOS_InputHookPtr :: Ptr{Ptr{Cvoid}} = C_NULL
-    PyJuliaBase_Type :: PyPtr = C_NULL
-    PyExc_JuliaError :: PyPtr = C_NULL
 end
 
 const POINTERS = CAPIPointers()
@@ -289,7 +287,7 @@ const POINTERS = CAPIPointers()
         end
         for name in CAPI_FUNCS
     ]...)
-    $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))))) for name in CAPI_EXCEPTIONS]...)
+    $([:(p.$name = Base.unsafe_load(Ptr{PyPtr}(dlsym(lib, $(QuoteNode(name)))::Ptr))) for name in CAPI_EXCEPTIONS]...)
     $([:(p.$name = dlsym(lib, $(QuoteNode(name)))) for name in CAPI_OBJECTS]...)
     p.PyOS_InputHookPtr = dlsym(CTX.lib_ptr, :PyOS_InputHook)
 end

src/cpython/find_libpython.py renamed to src/CPython/find_libpython.py

@@ -47,6 +47,7 @@ py_finalizer(x::Py) = GC.enqueue(getptr(x))
 
 ispy(::Py) = true
 getptr(x::Py) = getfield(x, :ptr)
+pyconvert(::Type{Py}, x::Py) = x
 
 setptr!(x::Py, ptr::C.PyPtr) = (setfield!(x, :ptr, ptr); x)
 
@@ -148,7 +149,6 @@ Py(x::AbstractRange{<:Union{Int8,Int16,Int32,Int64,Int128,UInt8,UInt16,UInt32,UI
 Py(x::Date) = pydate(x)
 Py(x::Time) = pytime(x)
 Py(x::DateTime) = pydatetime(x)
-Py(x) = ispy(x) ? throw(MethodError(Py, (x,))) : pyjl(x)
 
 Base.string(x::Py) = pyisnull(x) ? "<py NULL>" : pystr(String, x)
 Base.print(io::IO, x::Py) = print(io, string(x))
@@ -251,12 +251,8 @@ function Base.show(io::IO, ::MIME"text/plain", o::Py)
     end
 end
 
-Base.show(io::IO, mime::MIME, o::Py) = pyshow(io, mime, o)
-Base.show(io::IO, mime::MIME"text/csv", o::Py) = pyshow(io, mime, o)
-Base.show(io::IO, mime::MIME"text/tab-separated-values", o::Py) = pyshow(io, mime, o)
 
 Base.showable(::MIME"text/plain", ::Py) = true
-Base.showable(mime::MIME, o::Py) = pyshowable(mime, o)
 
 Base.getproperty(x::Py, k::Symbol) = pygetattr(x, string(k))
 Base.getproperty(x::Py, k::String) = pygetattr(x, k)

src/cpython/gil.jl renamed to src/CPython/gil.jl

@@ -0,0 +1,38 @@
+"""
+    module _Py
+
+Defines the `Py` type and directly related functions.
+"""
+module _Py
+
+const VERSION = v"0.9.15"
+const ROOT_DIR = dirname(dirname(@__DIR__))
+
+using .._CPython: _CPython as C
+using .._Utils: _Utils as Utils
+using Base: @propagate_inbounds, @kwdef
+using Dates: Date, Time, DateTime, year, month, day, hour, minute, second, millisecond, microsecond, nanosecond
+using MacroTools: @capture
+using Markdown: Markdown
+# using MacroTools, Dates, Tables, Markdown, Serialization, Requires, Pkg, REPL
+
+include("gc.jl")
+include("Py.jl")
+include("err.jl")
+include("config.jl")
+include("consts.jl")
+include("builtins.jl")
+include("stdlib.jl")
+include("juliacall.jl")
+include("pyconst_macro.jl")
+
+function __init__()
+    C.with_gil() do
+        init_consts()
+        init_datetime()
+        init_stdlib()
+        init_juliacall()
+    end
+end
+
+end