From 9471f7f6e58b89b2c9a957ffd3bb46ada6ba2039 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 26 Feb 2019 14:57:38 -0500 Subject: [PATCH 1/3] fix array2py and friends for adjoints etc. --- src/conversions.jl | 38 ++++++++++++++++---------------------- src/numpy.jl | 10 ++++++++++ src/pybuffer.jl | 2 +- test/runtests.jl | 3 +++ 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/conversions.jl b/src/conversions.jl index ba4db31e..f6f73b42 100644 --- a/src/conversions.jl +++ b/src/conversions.jl @@ -292,37 +292,31 @@ append!(a::PyVector{T}, items) where {T} = PyVector{T}(append!(a.o, items)) ######################################################################### # Lists and 1d arrays. +if VERSION < v"1.1.0-DEV.392" # #29440 + cirange(I,J) = CartesianIndices(map((i,j) -> i:j, Tuple(I), Tuple(J))) +else + cirange(I,J) = I:J +end + # recursive conversion of A to a list of list of lists... starting -# with dimension dim and index i in A. -function array2py(A::AbstractArray{T, N}, dim::Integer, i::Integer) where {T, N} - if dim > N +# with dimension dim and Cartesian index i in A. +function array2py(A::AbstractArray{<:Any, N}, dim::Integer, i::CartesianIndex{N}) where {N} + if dim > N # base case return PyObject(A[i]) - elseif dim == N # special case last dim to coarsen recursion leaves - len = size(A, dim) - s = N == 1 ? 1 : stride(A, dim) - o = PyObject(@pycheckn ccall((@pysym :PyList_New), PyPtr, (Int,), len)) - for j = 0:len-1 - oi = PyObject(A[i+j*s]) - @pycheckz ccall((@pysym :PyList_SetItem), Cint, (PyPtr,Int,PyPtr), - o, j, oi) - pyincref(oi) # PyList_SetItem steals the reference - end - return o - else # dim < N: store multidimensional array as list of lists - len = size(A, dim) - s = stride(A, dim) - o = PyObject(@pycheckn ccall((@pysym :PyList_New), PyPtr, (Int,), len)) - for j = 0:len-1 - oi = array2py(A, dim+1, i+j*s) + else # recursively store multidimensional array as list of lists + ilast = CartesianIndex(ntuple(j -> j == dim ? lastindex(A, dim) : i[j], Val{N}())) + o = PyObject(@pycheckn ccall((@pysym :PyList_New), PyPtr, (Int,), size(A, dim))) + for icur in cirange(i,ilast) + oi = array2py(A, dim+1, icur) @pycheckz ccall((@pysym :PyList_SetItem), Cint, (PyPtr,Int,PyPtr), - o, j, oi) + o, icur[dim]-i[dim], oi) pyincref(oi) # PyList_SetItem steals the reference end return o end end -array2py(A::AbstractArray) = array2py(A, 1, 1) +array2py(A::AbstractArray) = array2py(A, 1, first(CartesianIndices(A))) PyObject(A::AbstractArray) = ndims(A) <= 1 || hasmethod(stride, Tuple{typeof(A),Int}) ? array2py(A) : diff --git a/src/numpy.jl b/src/numpy.jl index c332a249..0b619cbe 100644 --- a/src/numpy.jl +++ b/src/numpy.jl @@ -197,6 +197,10 @@ end PyReverseDims(a::StridedArray{T}) where {T<:PYARR_TYPES} = NpyArray(a, true) PyReverseDims(a::BitArray) = PyReverseDims(Array(a)) +# fallback to physically transposing the array +PyReverseDims(a::AbstractArray{<:Any,N}) where {N} = PyObject(permutedims(a, N:-1:1)) +PyReverseDims(a::AbstractMatrix) = PyObject(permutedims(a)) + """ PyReverseDims(array) @@ -209,3 +213,9 @@ libraries that expect row-major data. PyReverseDims(a::AbstractArray) ######################################################################### + +# transposed arrays can be passed to NumPy without copying +PyObject(a::Union{LinearAlgebra.Adjoint{<:Real},LinearAlgebra.Transpose}) = + PyReverseDims(a.parent) + +PyObject(a::LinearAlgebra.Adjoint) = PyObject(Matrix(a)) # non-real arrays require a copy diff --git a/src/pybuffer.jl b/src/pybuffer.jl index 5e082352..30bc8b9a 100644 --- a/src/pybuffer.jl +++ b/src/pybuffer.jl @@ -251,7 +251,7 @@ function array_format(pybuf::PyBuffer) use_native_sizes = false elseif fmt_str[1] == '=' use_native_sizes = false - elseif fmt_str[1] == "Z" + elseif fmt_str[1] == 'Z' type_start_idx = 1 else error("Unsupported format string: \"$fmt_str\"") diff --git a/test/runtests.jl b/test/runtests.jl index 771528ff..193020d0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -60,6 +60,8 @@ const PyInt = pyversion < v"3" ? Int : Clonglong @test roundtripeq(C_NULL) && roundtripeq(convert(Ptr{Cvoid}, 12345)) @test roundtripeq([1,3,4,5]) && roundtripeq([1,3.2,"hello",true]) @test roundtripeq([1 2 3;4 5 6]) && roundtripeq([1. 2 3;4 5 6]) + @test roundtripeq([1. 2 3;4 5 6]') + @test roundtripeq([1.0+2im 2+3im 3;4 5 6]') @test roundtripeq((1,(3.2,"hello"),true)) && roundtripeq(()) @test roundtripeq(Int32) @test roundtripeq(Dict(1 => "hello", 2 => "goodbye")) && roundtripeq(Dict()) @@ -119,6 +121,7 @@ const PyInt = pyversion < v"3" ? Int : Clonglong array2py2arrayeq(x) = PyCall.py2array(Float64,PyCall.array2py(x)) == x @test array2py2arrayeq(rand(3)) @test array2py2arrayeq(rand(3,4)) + @test array2py2arrayeq(rand(3,4)') @test array2py2arrayeq(rand(3,4,5)) @test roundtripeq(2:10) && roundtripeq(10:-1:2) From 9bcff32774f01f6890049cb62d0992fdf8aa6ba6 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 26 Feb 2019 17:00:16 -0500 Subject: [PATCH 2/3] fallback for missing numpy --- src/numpy.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/numpy.jl b/src/numpy.jl index 0b619cbe..a73c9d02 100644 --- a/src/numpy.jl +++ b/src/numpy.jl @@ -190,11 +190,17 @@ function PyObject(a::StridedArray{T}) where T<:PYARR_TYPES try return NpyArray(a, false) catch - array2py(a) # fallback to non-NumPy version + return array2py(a) # fallback to non-NumPy version end end -PyReverseDims(a::StridedArray{T}) where {T<:PYARR_TYPES} = NpyArray(a, true) +function PyReverseDims(a::StridedArray{T,N}) where {T<:PYARR_TYPES,N} + try + return NpyArray(a, true) + catch + return array2py(a, permutedims(a, N:-1:1)) # fallback to non-NumPy version + end +end PyReverseDims(a::BitArray) = PyReverseDims(Array(a)) # fallback to physically transposing the array From 75dd100927ae6cc06d2a64c346ec71635846f93b Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Tue, 26 Feb 2019 18:47:25 -0500 Subject: [PATCH 3/3] whoops --- src/numpy.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/numpy.jl b/src/numpy.jl index a73c9d02..f3d523aa 100644 --- a/src/numpy.jl +++ b/src/numpy.jl @@ -198,7 +198,7 @@ function PyReverseDims(a::StridedArray{T,N}) where {T<:PYARR_TYPES,N} try return NpyArray(a, true) catch - return array2py(a, permutedims(a, N:-1:1)) # fallback to non-NumPy version + return array2py(permutedims(a, N:-1:1)) # fallback to non-NumPy version end end PyReverseDims(a::BitArray) = PyReverseDims(Array(a))