From 15e7007457842c436f0d083624c2fc6f8278307e Mon Sep 17 00:00:00 2001 From: Jason Newton Date: Sun, 4 Sep 2016 03:10:33 -0400 Subject: [PATCH 1/4] make dtype enum class of int --- include/pybind11/numpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 51c68ad96d..2dffe9a0bd 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -208,7 +208,7 @@ class array : public buffer { public: PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) - enum { + enum : int{ c_style = detail::npy_api::NPY_C_CONTIGUOUS_, f_style = detail::npy_api::NPY_F_CONTIGUOUS_, forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_ From d693834279b29c646f97a9a6ae54e30f396008f9 Mon Sep 17 00:00:00 2001 From: Jason Newton Date: Sun, 4 Sep 2016 04:59:50 -0400 Subject: [PATCH 2/4] bring in and build on the numpy c-api for a more first-class ndarray experience --- CMakeLists.txt | 4 + include/pybind11/numpy.h | 211 ++++++++++++++++++++++++++++++++++++++- tests/pybind11_tests.cpp | 5 + tools/FindNumpy.cmake | 64 ++++++++++++ 4 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 tools/FindNumpy.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 281de2480c..455f10d6bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,10 @@ set(PYBIND11_PYTHON_VERSION "" CACHE STRING "Python version to use for compiling list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/tools") set(Python_ADDITIONAL_VERSIONS 3.4 3.5 3.6 3.7) find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} REQUIRED) +find_package(Numpy) +if(PYTHON_NUMPY_FOUND) + list(APPEND PYTHON_INCLUDE_DIRS ${PYTHON_NUMPY_INCLUDE_DIR}) +endif() include(CheckCXXCompilerFlag) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2dffe9a0bd..f2a4c7b7ef 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -20,6 +20,14 @@ #include #include +#ifndef PYBIND_DONT_INCLUDE_NUMPY +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include +#include +#undef NPY_NO_DEPRECATED_API +#endif + #if defined(_MSC_VER) #pragma warning(push) #pragma warning(disable: 4127) // warning C4127: Conditional expression is constant @@ -153,14 +161,82 @@ class dtype : public object { return attr("itemsize").cast(); } - bool has_fields() const { - return attr("fields").cast().ptr() != Py_None; - } - std::string kind() const { return (std::string) attr("kind").cast(); } + bool check_scalar() const{ + return PyArray_CheckScalar(m_ptr); + } + + bool is_any_scalar() const{ + return PyArray_CheckScalar(m_ptr); + } + + bool is_unsigned() const{ + return PyDataType_ISUNSIGNED(m_ptr); + } + + bool is_signed() const{ + return PyDataType_ISSIGNED(m_ptr); + } + + bool is_integer() const{ + return PyDataType_ISINTEGER(m_ptr); + } + + bool is_float() const{ + return PyDataType_ISFLOAT(m_ptr); + } + + bool is_complex() const{ + return PyDataType_ISCOMPLEX(m_ptr); + } + + bool is_number() const{ + return PyDataType_ISNUMBER(m_ptr); + } + + bool is_string() const{ + return PyDataType_ISSTRING(m_ptr); + } + + bool is_python() const{ + return PyDataType_ISPYTHON(m_ptr); + } + + bool is_flexible() const{ + return PyDataType_ISFLEXIBLE(m_ptr); + } + + bool is_userdef() const{ + return PyDataType_ISUSERDEF(m_ptr); + } + + bool is_extended() const{ + return PyDataType_ISEXTENDED(m_ptr); + } + + bool is_object() const{ + return PyDataType_ISOBJECT(m_ptr); + } + + bool is_bool() const{ + return PyTypeNum_ISBOOL(((PyArray_Descr*)(m_ptr))->type_num);//bool has bitrotted, see https://mail.scipy.org/pipermail/numpy-discussion/2013-August/067549.html + } + + bool has_fields() const{ + return PyDataType_HASFIELDS(m_ptr); + } + + bool is_swapped() const{ + return PyDataType_ISBYTESWAPPED(m_ptr); + } + + bool operator==(const dtype &rhs) const{ + return PyArray_EquivTypes((PyArray_Descr *) m_ptr, (PyArray_Descr *) rhs.m_ptr); + } + private: static object _dtype_from_pep3118() { static PyObject *obj = module::import("numpy.core._internal") @@ -251,7 +327,87 @@ class array : public buffer { : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } pybind11::dtype dtype() { - return attr("dtype").cast(); + return object((PyObject *) PyArray_DTYPE((PyArrayObject *) m_ptr), true).cast(); + } + int ndims() const{ + return PyArray_NDIM((PyArrayObject *) m_ptr); + } + npy_intp dim(int d) const{ + return PyArray_DIM((PyArrayObject *) m_ptr, d); + } + const npy_intp* shape() const{ + return PyArray_SHAPE((PyArrayObject *) m_ptr); + } + const npy_intp* strides() const{ + return PyArray_STRIDES((PyArrayObject *) m_ptr); + } + npy_intp stride(int d) const{ + return PyArray_STRIDE((PyArrayObject *) m_ptr, d); + } + uint8_t *data() { + return reinterpret_cast(PyArray_DATA((PyArrayObject *) m_ptr)); + } + uint8_t *data() const{ + return reinterpret_cast(PyArray_DATA((PyArrayObject *) m_ptr)); + } + object base() const{ + return object(PyArray_BASE((PyArrayObject *) m_ptr), true); + } + int type() const{ + return PyArray_TYPE((PyArrayObject *) m_ptr); + } + int flags() const{ + return PyArray_FLAGS((PyArrayObject *) m_ptr); + } + size_t itemsize() const{ + return PyArray_ITEMSIZE((PyArrayObject *) m_ptr); + } + size_t size() const{ + return PyArray_SIZE((PyArrayObject *) m_ptr); + } + size_t nbytes() const{ + return PyArray_NBYTES((PyArrayObject *) m_ptr); + } + + template + uint8_t* data_at(Args&&... indices){ + const int nterms = sizeof...(indices); + const uint32_t _indices[] = { uint32_t(indices)... }; + auto *strides = this->strides(); + assert(nterms < this->ndims()); + uint8_t *p = this->data(); + for(int i = 0; i < nterms; ++i){ + p += _indices[i] * strides[i]; + } + return p; + } + template + const uint8_t* data_at(Args&&... indices) const{ + return const_cast(this)->data_at(indices...); + } + + template + static array empty(const std::array &shape, const pybind11::dtype &type, int order = 'C'){ + npy_intp _shape[N]; + for(int i = 0; i < N; ++i){ + _shape[i] = shape[i]; + } + object result(PyArray_Empty(N, _shape, type, order == 'f' || order == 'F' ), false); + if (!result) + pybind11_fail("NumPy: unable to create array!"); + return result; + } + + template + static array zeros(const std::array &shape, const pybind11::dtype &type, int order = 'C'){ + npy_intp _shape[N]; + for(int i = 0; i < N; ++i){ + _shape[i] = shape[i]; + } + object result(PyArray_Zeros(N, _shape, type, order == 'f' || order == 'F' ), false); + if (!result) + pybind11_fail("NumPy: unable to create array!"); + return result; } protected: @@ -287,6 +443,51 @@ template class array_t : public array_t(size_t size, T* ptr = nullptr) : array(size, ptr) { } + T *data() { + return reinterpret_cast(PyArray_DATA((PyArrayObject *) m_ptr)); + } + const T *data() const{ + return reinterpret_cast(PyArray_DATA((PyArrayObject *) m_ptr)); + } + + template + T* data_at(Args&&... indices){ + const int nterms = sizeof...(indices); + const uint32_t _indices[] = { uint32_t(indices)... }; + auto *strides = this->strides(); + assert(nterms < this->ndims()); + uint8_t *p = this->data(); + for(int i = 0; i < nterms; ++i){ + p += _indices[i] * strides[i]; + } + return p; + } + template + T* data_at(Args&&... indices) const{ + return const_cast(this)->data_at(indices...); + } + + template + T* at(Args&&... indices){ + assert(sizeof...(indices) == this->ndims()); + return *this->data_at(indices...); + } + template + const T* at(Args&&... indices) const{ + assert(sizeof...(indices) == this->ndims()); + return const_cast *>(this)->at(indices...); + } + + template + static array_t empty(const std::array &shape, int order = 'C'){ + return empty(shape, dtype::of(), order); + } + + template + static array_t zeros(const std::array &shape, int order = 'C'){ + return zeros(shape, dtype::of(), order); + } + static bool is_non_null(PyObject *ptr) { return ptr != nullptr; } static PyObject *ensure(PyObject *ptr) { diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 525f379ebf..372e37d655 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -10,6 +10,10 @@ #include "pybind11_tests.h" #include "constructor_stats.h" +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include + void init_ex_methods_and_attributes(py::module &); void init_ex_python_types(py::module &); void init_ex_operator_overloading(py::module &); @@ -50,6 +54,7 @@ void bind_ConstructorStats(py::module &m) { } PYBIND11_PLUGIN(pybind11_tests) { + import_array(); //import numpy py::module m("pybind11_tests", "pybind example plugin"); bind_ConstructorStats(m); diff --git a/tools/FindNumpy.cmake b/tools/FindNumpy.cmake new file mode 100644 index 0000000000..d92db4c464 --- /dev/null +++ b/tools/FindNumpy.cmake @@ -0,0 +1,64 @@ +#Unless otherwise noted in the file, all files in this repository are +#licensed under the BSD license, reproduced below. +# +#Redistribution and use in source and binary forms, with or without +#modification, are permitted provided that the following conditions are met: +# +#- Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +#- Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +#- Neither the name of Eyescale Software GmbH nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +#AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +#IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +#ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +#LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +#CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +#SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +#INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +#CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +#ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +#POSSIBILITY OF SUCH DAMAGE. + +cmake_minimum_required(VERSION 2.6) + +if(NOT PYTHON_EXECUTABLE) + if(NumPy_FIND_QUIETLY) + find_package(PythonInterp QUIET) + else() + find_package(PythonInterp) + set(__numpy_out 1) + endif() +endif() + +if (PYTHON_EXECUTABLE) + # Find out the include path + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c + "from __future__ import print_function\ntry: import numpy; print(numpy.get_include(), end='')\nexcept:pass\n" + OUTPUT_VARIABLE __numpy_path) + # And the version + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c + "from __future__ import print_function\ntry: import numpy; print(numpy.__version__, end='')\nexcept:pass\n" + OUTPUT_VARIABLE __numpy_version) +elseif(__numpy_out) + message(STATUS "Python executable not found.") +endif(PYTHON_EXECUTABLE) + +find_path(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h + HINTS "${__numpy_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) + +if(PYTHON_NUMPY_INCLUDE_DIR) + set(PYTHON_NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found") +endif(PYTHON_NUMPY_INCLUDE_DIR) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NumPy REQUIRED_VARS PYTHON_NUMPY_INCLUDE_DIR + VERSION_VAR __numpy_version) + From 4680a4c3808130af2cd05d2f365b7a97d64784a2 Mon Sep 17 00:00:00 2001 From: Jason Newton Date: Sun, 4 Sep 2016 05:00:26 -0400 Subject: [PATCH 3/4] add dtype builder for dynamic / complicated data types --- include/pybind11/numpy.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index f2a4c7b7ef..d76b19ab1b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -280,6 +280,26 @@ class dtype : public object { } }; +class dtype_record_builder { + list names; + list offsets; + list types; + size_t itemsize; + + dtype_record_builder(size_t itemsize):itemsize(itemsize){} + + dtype_record_builder& add(const std::string &name, size_t offset, dtype &type){ + names.append(str(name)); + offsets.append(int_(offset)); + types.append(type); + return *this; + } + + dtype build(){ + return dtype(names, types, offsets, itemsize); + } +}; + class array : public buffer { public: PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_) From 1059ed4258c1e829e2d680d8f8510557e6ddcbd3 Mon Sep 17 00:00:00 2001 From: Jason Newton Date: Sun, 4 Sep 2016 05:03:17 -0400 Subject: [PATCH 4/4] add copy and flags parameters to array constructors --- include/pybind11/numpy.h | 58 +++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index d76b19ab1b..a19e373658 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -311,7 +311,7 @@ class array : public buffer { }; array(const pybind11::dtype& dt, const std::vector& shape, - const std::vector& strides, void *ptr = nullptr) { + const std::vector& strides, void *ptr = nullptr, bool copy = true, int flags = 0) { auto& api = detail::npy_api::get(); auto ndim = shape.size(); if (shape.size() != strides.size()) @@ -319,29 +319,29 @@ class array : public buffer { auto descr = dt; object tmp(api.PyArray_NewFromDescr_( api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(), - (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false); + (Py_intptr_t *) strides.data(), ptr, flags, nullptr), false); if (!tmp) pybind11_fail("NumPy: unable to create array!"); - if (ptr) + if (ptr && copy) tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false); m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr) - : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { } + array(const pybind11::dtype& dt, const std::vector& shape, void *ptr = nullptr, bool copy = true, int flags = 0) + : array(dt, shape, default_strides(shape, dt.itemsize(), flags), ptr, copy, flags) { } - array(const pybind11::dtype& dt, size_t count, void *ptr = nullptr) - : array(dt, std::vector { count }, ptr) { } + array(const pybind11::dtype& dt, size_t count, void *ptr = nullptr, bool copy = true, int flags = 0) + : array(dt, std::vector { count }, ptr, copy, flags) { } template array(const std::vector& shape, - const std::vector& strides, T* ptr) - : array(pybind11::dtype::of(), shape, strides, (void *) ptr) { } + const std::vector& strides, T* ptr, bool copy = true, int flags = 0) + : array(pybind11::dtype::of(), shape, strides, (void *) ptr, copy, flags) { } - template array(const std::vector& shape, T* ptr) - : array(shape, default_strides(shape, sizeof(T)), ptr) { } + template array(const std::vector& shape, T* ptr, bool copy = true, int flags = 0) + : array(shape, default_strides(shape, sizeof(T), flags), ptr, copy, flags) { } - template array(size_t size, T* ptr) - : array(std::vector { size }, ptr) { } + template array(size_t size, T* ptr, bool copy = true, int flags = 0) + : array(std::vector { size }, ptr, copy, flags) { } array(const buffer_info &info) : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { } @@ -433,14 +433,22 @@ class array : public buffer { protected: template friend struct detail::npy_format_descriptor; - static std::vector default_strides(const std::vector& shape, size_t itemsize) { - auto ndim = shape.size(); + static std::vector default_strides(const std::vector& shape, size_t itemsize, int flags) { + const int ndim = (int) shape.size(); std::vector strides(ndim); if (ndim) { - std::fill(strides.begin(), strides.end(), itemsize); - for (size_t i = 0; i < ndim - 1; i++) - for (size_t j = 0; j < ndim - 1 - i; j++) - strides[j] *= shape[ndim - 1 - i]; + size_t cumprod = itemsize; + if(flags & c_style){ + for(int i = ndim - 1; i >= 0; --i){ + strides[i] = cumprod; + cumprod *= shape[i]; + } + }else{ + for(int i = 0; i < ndim; ++i){ + strides[i] = cumprod; + cumprod *= shape[i]; + } + } } return strides; } @@ -454,14 +462,14 @@ template class array_t : public array_t(const buffer_info& info) : array(info) { } - array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr) - : array(shape, strides, ptr) { } + array_t(const std::vector& shape, const std::vector& strides, T* ptr = nullptr, bool copy = true, int flags = 0) + : array(shape, strides, ptr, copy, flags) { } - array_t(const std::vector& shape, T* ptr = nullptr) - : array(shape, ptr) { } + array_t(const std::vector& shape, T* ptr = nullptr, bool copy = true, int flags = 0) + : array(shape, ptr, copy, flags) { } - array_t(size_t size, T* ptr = nullptr) - : array(size, ptr) { } + array_t(size_t size, T* ptr = nullptr, bool copy = true, int flags = 0) + : array(size, ptr, copy, flags) { } T *data() { return reinterpret_cast(PyArray_DATA((PyArrayObject *) m_ptr));