From 4aa3de572ad20ca2f683de842ed144cc36693be3 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Fri, 22 May 2020 14:32:39 +0900 Subject: [PATCH 01/18] Fix undefined memoryview format --- include/pybind11/pytypes.h | 7 ++++++- tests/test_pytypes.cpp | 4 ++++ tests/test_pytypes.py | 8 ++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 63cbf2e561..c65b2c0927 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1336,9 +1336,14 @@ class memoryview : public object { // Py_buffer uses signed sizes, strides and shape!.. static std::vector py_strides { }; static std::vector py_shape { }; + static std::vector formats = { + "?", "b", "B", "h", "H", "i", "I", "q", "Q", "f", "d", "g" + }; buf.buf = info.ptr; buf.itemsize = info.itemsize; - buf.format = const_cast(info.format.c_str()); + auto it = std::find(formats.begin(), formats.end(), info.format); + buf.format = (it == formats.end()) ? + nullptr : const_cast(it->c_str()); buf.ndim = (int) info.ndim; buf.len = info.size; py_strides.clear(); diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 244e1db0d2..bd39e573df 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -307,4 +307,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("test_list_slicing", [](py::list a) { return a[py::slice(0, -1, 2)]; }); + + m.def("test_memoryview", [](py::buffer b) { + return py::memoryview(b.request()); + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 0e8d6c33a7..76b161d872 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -261,3 +261,11 @@ def test_number_protocol(): def test_list_slicing(): li = list(range(100)) assert li[::2] == m.test_list_slicing(li) + + +def test_memoryview(): + import array + view = m.test_memoryview(b'abc') + _ = m.test_memoryview(array.array('I', [1, 1])) + assert view.format == 'B' + assert view[0] == ord(b'a') From de4baa008ae7004c47d434fe3a76c6c29060dddc Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Fri, 22 May 2020 16:23:55 +0900 Subject: [PATCH 02/18] Add missing header --- include/pybind11/pytypes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index c65b2c0927..3f8702ccba 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -13,6 +13,7 @@ #include "buffer_info.h" #include #include +#include NAMESPACE_BEGIN(PYBIND11_NAMESPACE) From 2c53550853351fe1a727f7d9bb035ba9cc3d5c2e Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Fri, 22 May 2020 17:07:29 +0900 Subject: [PATCH 03/18] Add workaround for py27 array compatibility --- tests/test_pytypes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 76b161d872..532c929f2f 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -266,6 +266,8 @@ def test_list_slicing(): def test_memoryview(): import array view = m.test_memoryview(b'abc') - _ = m.test_memoryview(array.array('I', [1, 1])) + if sys.version_info[0] == 3: + # Python 2.7 array does not implement the new buffer protocol + m.test_memoryview(array.array('I', [1, 1])) assert view.format == 'B' assert view[0] == ord(b'a') From 9ee151438c53027e8065096631e15a53284bdd7c Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Fri, 22 May 2020 17:25:10 +0900 Subject: [PATCH 04/18] Workaround py27 memoryview behavior --- tests/test_pytypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 532c929f2f..76a0f8a5b9 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -270,4 +270,4 @@ def test_memoryview(): # Python 2.7 array does not implement the new buffer protocol m.test_memoryview(array.array('I', [1, 1])) assert view.format == 'B' - assert view[0] == ord(b'a') + assert view[0] == ord(b'a') if sys.version_info[0] == 3 else 'a' From a239716ed0b2239fba4fa8f645f771b917eab779 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Thu, 2 Jul 2020 15:22:53 +0900 Subject: [PATCH 05/18] Fix memoryview constructor from buffer_info --- include/pybind11/buffer_info.h | 10 ++-- include/pybind11/pytypes.h | 84 +++++++++++++++++++++------------- tests/test_pytypes.cpp | 15 +++++- tests/test_pytypes.py | 25 +++++++--- 4 files changed, 90 insertions(+), 44 deletions(-) diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index 1f4115a1fa..4c65412e1a 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -54,7 +54,7 @@ struct buffer_info { explicit buffer_info(Py_buffer *view, bool ownview = true) : buffer_info(view->buf, view->itemsize, view->format, view->ndim, {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}, view->readonly) { - this->view = view; + this->m_view = view; this->ownview = ownview; } @@ -73,16 +73,18 @@ struct buffer_info { ndim = rhs.ndim; shape = std::move(rhs.shape); strides = std::move(rhs.strides); - std::swap(view, rhs.view); + std::swap(m_view, rhs.m_view); std::swap(ownview, rhs.ownview); readonly = rhs.readonly; return *this; } ~buffer_info() { - if (view && ownview) { PyBuffer_Release(view); delete view; } + if (m_view && ownview) { PyBuffer_Release(m_view); delete m_view; } } + Py_buffer *view() const { return m_view; } + Py_buffer *&view() { return m_view; } private: struct private_ctr_tag { }; @@ -90,7 +92,7 @@ struct buffer_info { detail::any_container &&shape_in, detail::any_container &&strides_in, bool readonly) : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) { } - Py_buffer *view = nullptr; + Py_buffer *m_view = nullptr; bool ownview = false; }; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 3f8702ccba..d1b56817dd 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1332,40 +1332,60 @@ class buffer : public object { class memoryview : public object { public: - explicit memoryview(const buffer_info& info) { - static Py_buffer buf { }; - // Py_buffer uses signed sizes, strides and shape!.. - static std::vector py_strides { }; - static std::vector py_shape { }; - static std::vector formats = { - "?", "b", "B", "h", "H", "i", "I", "q", "Q", "f", "d", "g" - }; - buf.buf = info.ptr; - buf.itemsize = info.itemsize; - auto it = std::find(formats.begin(), formats.end(), info.format); - buf.format = (it == formats.end()) ? - nullptr : const_cast(it->c_str()); - buf.ndim = (int) info.ndim; - buf.len = info.size; - py_strides.clear(); - py_shape.clear(); - for (size_t i = 0; i < (size_t) info.ndim; ++i) { - py_strides.push_back(info.strides[i]); - py_shape.push_back(info.shape[i]); - } - buf.strides = py_strides.data(); - buf.shape = py_shape.data(); - buf.suboffsets = nullptr; - buf.readonly = info.readonly; - buf.internal = nullptr; - - m_ptr = PyMemoryView_FromBuffer(&buf); - if (!m_ptr) - pybind11_fail("Unable to create memoryview from buffer descriptor"); - } - PYBIND11_OBJECT_CVT(memoryview, object, PyMemoryView_Check, PyMemoryView_FromObject) + explicit memoryview(char *mem, ssize_t size, bool writable = false) + : object(PyMemoryView_FromMemory(mem, size, (writable) ? PyBUF_WRITE : PyBUF_READ), stolen_t{}) { + if (!m_ptr) pybind11_fail("Could not allocate memoryview object!"); + } + explicit memoryview(const buffer_info& info); }; + +inline memoryview::memoryview(const buffer_info& info) { + // TODO: two-letter formats are not supported. + static const char* formats[] = { + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + pybind11::format_descriptor::value, + }; + if (info.view()) { + // Note: PyMemoryView_FromBuffer never increments obj reference. + m_ptr = (info.view()->obj) ? + PyMemoryView_FromObject(info.view()->obj) : + PyMemoryView_FromBuffer(info.view()); + } + else { + size_t length = sizeof(formats) / sizeof(char*); + auto format = std::find(formats, formats + length, info.format); + if (format == (formats + length)) + pybind11_fail("Invalid format string"); + std::vector shape(info.shape.begin(), info.shape.end()); + std::vector strides(info.strides.begin(), info.strides.end()); + Py_buffer view; + view.buf = info.ptr; + view.obj = nullptr; + view.len = info.size * info.itemsize; + view.readonly = info.readonly; + view.itemsize = info.itemsize; + view.format = const_cast(*format); + view.ndim = static_cast(info.ndim); + view.shape = shape.data(); + view.strides = strides.data(); + view.suboffsets = nullptr; + view.internal = nullptr; + m_ptr = PyMemoryView_FromBuffer(&view); + } + if (!m_ptr) + pybind11_fail("Unable to create memoryview from buffer descriptor"); +} /// @} pytypes /// \addtogroup python_builtins diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index bd39e573df..c086ecee10 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -308,7 +308,20 @@ TEST_SUBMODULE(pytypes, m) { return a[py::slice(0, -1, 2)]; }); - m.def("test_memoryview", [](py::buffer b) { + m.def("test_memoryview_fromobject", [](py::buffer b) { + return py::memoryview(b); + }); + + m.def("test_memoryview_frombuffer_reference", [](py::buffer b) { return py::memoryview(b.request()); }); + + m.def("test_memoryview_frombuffer_new", []() { + const char* buf = "abc"; + const char* buf2 = "\x00\x00\x00\x00"; + auto mv = py::memoryview(py::buffer_info(buf, 3, 1)); + // Call twice with a different buffer to check the view content. + py::memoryview(py::buffer_info(const_cast(buf2), 4, "i", 1)); + return mv; + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 76a0f8a5b9..8b832cc93f 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -263,11 +263,22 @@ def test_list_slicing(): assert li[::2] == m.test_list_slicing(li) -def test_memoryview(): - import array - view = m.test_memoryview(b'abc') - if sys.version_info[0] == 3: - # Python 2.7 array does not implement the new buffer protocol - m.test_memoryview(array.array('I', [1, 1])) - assert view.format == 'B' +@pytest.mark.parametrize('method, args, format', [ + (m.test_memoryview_fromobject, (b'abc', ), 'B'), + (m.test_memoryview_frombuffer_reference, (b'abc',), 'B'), + (m.test_memoryview_frombuffer_new, tuple(), 'b'), +]) +def test_memoryview(method, args, format): + view = method(*args) + assert isinstance(view, memoryview) + assert view.format == format assert view[0] == ord(b'a') if sys.version_info[0] == 3 else 'a' + assert len(view) == 3 + + +def test_memoryview_refcount(): + buf = b'\x00\x00\x00\x00' + ref_before = sys.getrefcount(buf) + view = m.test_memoryview_frombuffer_reference(buf) + ref_after = sys.getrefcount(buf) + assert ref_before < ref_after From 5a3842f267aee4a23e37f35d83bc76027f529028 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Thu, 2 Jul 2020 16:10:12 +0900 Subject: [PATCH 06/18] Workaround PyMemoryView_FromMemory availability in py27 --- include/pybind11/pytypes.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index d1b56817dd..2a23362960 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1333,10 +1333,12 @@ class buffer : public object { class memoryview : public object { public: PYBIND11_OBJECT_CVT(memoryview, object, PyMemoryView_Check, PyMemoryView_FromObject) +#if PY_MAJOR_VERSION >= 3 explicit memoryview(char *mem, ssize_t size, bool writable = false) : object(PyMemoryView_FromMemory(mem, size, (writable) ? PyBUF_WRITE : PyBUF_READ), stolen_t{}) { if (!m_ptr) pybind11_fail("Could not allocate memoryview object!"); } +#endif explicit memoryview(const buffer_info& info); }; From 7db007dd78d8c1cfff79c27b59c1b3473bfe8489 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Mon, 6 Jul 2020 10:24:50 +0900 Subject: [PATCH 07/18] Fix up memoryview tests --- tests/test_pytypes.cpp | 8 ++------ tests/test_pytypes.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index c086ecee10..d41e024503 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -317,11 +317,7 @@ TEST_SUBMODULE(pytypes, m) { }); m.def("test_memoryview_frombuffer_new", []() { - const char* buf = "abc"; - const char* buf2 = "\x00\x00\x00\x00"; - auto mv = py::memoryview(py::buffer_info(buf, 3, 1)); - // Call twice with a different buffer to check the view content. - py::memoryview(py::buffer_info(const_cast(buf2), 4, "i", 1)); - return mv; + const char* buf = "ghi"; + return py::memoryview(py::buffer_info(buf, 3, 1)); }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 8b832cc93f..573a22a535 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -263,22 +263,26 @@ def test_list_slicing(): assert li[::2] == m.test_list_slicing(li) -@pytest.mark.parametrize('method, args, format', [ - (m.test_memoryview_fromobject, (b'abc', ), 'B'), - (m.test_memoryview_frombuffer_reference, (b'abc',), 'B'), - (m.test_memoryview_frombuffer_new, tuple(), 'b'), +@pytest.mark.parametrize('method, args, format, content', [ + (m.test_memoryview_fromobject, (b'abc',), 'B', b'abc'), + (m.test_memoryview_frombuffer_reference, (b'def',), 'B', b'def'), + (m.test_memoryview_frombuffer_new, tuple(), 'b', b'ghi'), ]) -def test_memoryview(method, args, format): +def test_memoryview(method, args, format, content): view = method(*args) assert isinstance(view, memoryview) assert view.format == format - assert view[0] == ord(b'a') if sys.version_info[0] == 3 else 'a' - assert len(view) == 3 + assert view[:] == content + assert len(view) == len(content) -def test_memoryview_refcount(): +@pytest.mark.parametrize('method', [ + m.test_memoryview_frombuffer_reference, + m.test_memoryview_fromobject, +]) +def test_memoryview_refcount(method): buf = b'\x00\x00\x00\x00' ref_before = sys.getrefcount(buf) - view = m.test_memoryview_frombuffer_reference(buf) + view = method(buf) ref_after = sys.getrefcount(buf) assert ref_before < ref_after From 72e439414cbdc4f117bfded831dfac694f622ed8 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Mon, 6 Jul 2020 13:33:44 +0900 Subject: [PATCH 08/18] Update memoryview test from buffer to check signedness --- tests/test_pytypes.cpp | 9 ++++++--- tests/test_pytypes.py | 26 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index d41e024503..2f8fbf5f94 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -316,8 +316,11 @@ TEST_SUBMODULE(pytypes, m) { return py::memoryview(b.request()); }); - m.def("test_memoryview_frombuffer_new", []() { - const char* buf = "ghi"; - return py::memoryview(py::buffer_info(buf, 3, 1)); + m.def("test_memoryview_frombuffer_new", [](bool is_unsigned) { + static const int16_t si16[] = { 3, 1, 4, 1, 5 }; + static const uint16_t ui16[] = { 2, 7, 1, 8 }; + auto info = (is_unsigned) ? + py::buffer_info(ui16, 4, 1) : py::buffer_info(si16, 5, 1); + return py::memoryview(info); }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 573a22a535..2faca7626f 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -263,17 +263,22 @@ def test_list_slicing(): assert li[::2] == m.test_list_slicing(li) -@pytest.mark.parametrize('method, args, format, content', [ - (m.test_memoryview_fromobject, (b'abc',), 'B', b'abc'), - (m.test_memoryview_frombuffer_reference, (b'def',), 'B', b'def'), - (m.test_memoryview_frombuffer_new, tuple(), 'b', b'ghi'), +@pytest.mark.parametrize('method, arg, fmt, expected_view', [ + (m.test_memoryview_fromobject, b'red', 'B', b'red'), + (m.test_memoryview_frombuffer_reference, b'green', 'B', b'green'), + (m.test_memoryview_frombuffer_new, False, 'h', [3, 1, 4, 1, 5]), + (m.test_memoryview_frombuffer_new, True, 'H', [2, 7, 1, 8]), ]) -def test_memoryview(method, args, format, content): - view = method(*args) +def test_memoryview(method, arg, fmt, expected_view): + view = method(arg) assert isinstance(view, memoryview) - assert view.format == format - assert view[:] == content - assert len(view) == len(content) + assert view.format == fmt + if isinstance(expected_view, bytes) or sys.version_info[0] >= 3: + view_as_list = list(view) + else: + # Using max to pick non-zero byte (big-endian vs little-endian). + view_as_list = [max([ord(c) for c in s]) for s in view] + assert view_as_list == list(expected_view) @pytest.mark.parametrize('method', [ @@ -281,8 +286,9 @@ def test_memoryview(method, args, format, content): m.test_memoryview_fromobject, ]) def test_memoryview_refcount(method): - buf = b'\x00\x00\x00\x00' + buf = b'\x0a\x0b\x0c\x0d' ref_before = sys.getrefcount(buf) view = method(buf) ref_after = sys.getrefcount(buf) assert ref_before < ref_after + assert list(view) == list(buf) From a585ed3835974faee7ebd86e6cb863f85583f6f1 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Fri, 10 Jul 2020 23:38:09 +0900 Subject: [PATCH 09/18] Use static factory method to create memoryview --- include/pybind11/pytypes.h | 143 ++++++++++++++++++++++++------------- tests/test_pytypes.cpp | 32 +++++++-- tests/test_pytypes.py | 21 ++++-- 3 files changed, 136 insertions(+), 60 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2a23362960..1177f490ff 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1333,60 +1333,107 @@ class buffer : public object { class memoryview : public object { public: PYBIND11_OBJECT_CVT(memoryview, object, PyMemoryView_Check, PyMemoryView_FromObject) -#if PY_MAJOR_VERSION >= 3 - explicit memoryview(char *mem, ssize_t size, bool writable = false) - : object(PyMemoryView_FromMemory(mem, size, (writable) ? PyBUF_WRITE : PyBUF_READ), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate memoryview object!"); - } -#endif - explicit memoryview(const buffer_info& info); -}; -inline memoryview::memoryview(const buffer_info& info) { - // TODO: two-letter formats are not supported. - static const char* formats[] = { - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - pybind11::format_descriptor::value, - }; - if (info.view()) { + /** \rst + Creates ``memoryview`` from ``buffer_info``. + + ``buffer_info`` must be created from ``buffer::request()``. Otherwise + throws an exception. + + For creating a ``memoryview`` from objects that support buffer protocol, + use ``memoryview(const object& obj)`` instead of this constructor. + \endrst */ + explicit memoryview(const buffer_info& info) { + if (!info.view()) + pybind11_fail("Prohibited to create memoryview without Py_buffer"); // Note: PyMemoryView_FromBuffer never increments obj reference. m_ptr = (info.view()->obj) ? PyMemoryView_FromObject(info.view()->obj) : PyMemoryView_FromBuffer(info.view()); + if (!m_ptr) + pybind11_fail("Unable to create memoryview from buffer descriptor"); } - else { - size_t length = sizeof(formats) / sizeof(char*); - auto format = std::find(formats, formats + length, info.format); - if (format == (formats + length)) - pybind11_fail("Invalid format string"); - std::vector shape(info.shape.begin(), info.shape.end()); - std::vector strides(info.strides.begin(), info.strides.end()); - Py_buffer view; - view.buf = info.ptr; - view.obj = nullptr; - view.len = info.size * info.itemsize; - view.readonly = info.readonly; - view.itemsize = info.itemsize; - view.format = const_cast(*format); - view.ndim = static_cast(info.ndim); - view.shape = shape.data(); - view.strides = strides.data(); - view.suboffsets = nullptr; - view.internal = nullptr; - m_ptr = PyMemoryView_FromBuffer(&view); - } - if (!m_ptr) - pybind11_fail("Unable to create memoryview from buffer descriptor"); + + /** \rst + Creates ``memoryview`` from static buffer. + + The caller is responsible for managing the lifetime of ``ptr`` and + ``format``. This method is meant for providing a ``memoryview`` for + C/C++ buffer not managed by Python. + + :param ptr: Pointer to the buffer. + :param itemsize: Byte size of an element. + :param format: Pointer to the null-terminated format string. For + homogeneous Buffers, this should be set to + format_descriptor::value. + :param ndim: Number of dimensions. + :param shape_in: Shape of the tensor (1 entry per dimension). + :param strides_in: Number of bytes between adjacent entries (for each + per dimension). + :param readonly: Flag to indicate if the underlying storage may be + written to. + \endrst */ + static memoryview frombuffer( + void *ptr, ssize_t itemsize, const char* format, ssize_t ndim, + detail::any_container shape_in, + detail::any_container strides_in, bool readonly=false); + + template + static memoryview frombuffer( + T *ptr, detail::any_container shape_in, + detail::any_container strides_in, bool readonly=false) { + return memoryview::frombuffer( + reinterpret_cast(ptr), sizeof(T), + format_descriptor::value, static_cast(shape_in->size()), + shape_in, strides_in, readonly); + } + +#if PY_MAJOR_VERSION >= 3 + /** \rst + Creates ``memoryview`` from static memory. + + The caller is responsible for managing the lifetime of ``mem``. This + constructor is meant for providing a ``memoryview`` for C/C++ buffer not + managed by Python. + \endrst */ + static memoryview frommemory(char *mem, ssize_t size, bool writable = false) { + PyObject* ptr = PyMemoryView_FromMemory( + mem, size, (writable) ? PyBUF_WRITE : PyBUF_READ); + if (!ptr) + pybind11_fail("Could not allocate memoryview object!"); + return memoryview(object(ptr, stolen_t{})); + } +#endif +}; + +inline memoryview memoryview::frombuffer( + void *ptr, ssize_t itemsize, const char* format, ssize_t ndim, + detail::any_container shape_in, + detail::any_container strides_in, bool readonly) { + std::vector shape(std::move(shape_in)); + std::vector strides(std::move(strides_in)); + if (ndim != static_cast(shape.size()) || + ndim != static_cast(strides.size())) + pybind11_fail("memoryview: ndim doesn't match shape and/or strides length"); + ssize_t size = 1; + for (size_t i = 0; i < static_cast(ndim); ++i) + size *= shape[i]; + Py_buffer view; + view.buf = ptr; + view.obj = nullptr; + view.len = size * itemsize; + view.readonly = static_cast(readonly); + view.itemsize = itemsize; + view.format = const_cast(format); + view.ndim = static_cast(ndim); + view.shape = shape.data(); + view.strides = strides.data(); + view.suboffsets = nullptr; + view.internal = nullptr; + PyObject* obj = PyMemoryView_FromBuffer(&view); + if (!obj) + pybind11_fail("Unable to create memoryview from buffer structure"); + return memoryview(object(obj, stolen_t{})); } /// @} pytypes diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 2f8fbf5f94..e6f99601b5 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -308,19 +308,39 @@ TEST_SUBMODULE(pytypes, m) { return a[py::slice(0, -1, 2)]; }); - m.def("test_memoryview_fromobject", [](py::buffer b) { + m.def("test_memoryview_object", [](py::buffer b) { return py::memoryview(b); }); - m.def("test_memoryview_frombuffer_reference", [](py::buffer b) { + m.def("test_memoryview_buffer_info", [](py::buffer b) { return py::memoryview(b.request()); }); - m.def("test_memoryview_frombuffer_new", [](bool is_unsigned) { + m.def("test_memoryview_frombuffer", [](bool is_unsigned) { static const int16_t si16[] = { 3, 1, 4, 1, 5 }; static const uint16_t ui16[] = { 2, 7, 1, 8 }; - auto info = (is_unsigned) ? - py::buffer_info(ui16, 4, 1) : py::buffer_info(si16, 5, 1); - return py::memoryview(info); + if (is_unsigned) + return py::memoryview::frombuffer( + const_cast(ui16), { 4 }, { sizeof(uint16_t) }, true); + else + return py::memoryview::frombuffer( + const_cast(si16), { 5 }, { sizeof(int16_t) }, true); }); + + m.def("test_memoryview_frombuffer_nativeformat", [](py::none unused) { + static const char* format = "@i"; + static const int32_t arr[] = { 4, 7, 5 }; + unused.is_none(); // Only to suppress unused compiler warn. + return py::memoryview::frombuffer( + const_cast(arr), sizeof(int32_t), format, 1, { 3 }, + { sizeof(int32_t) }, true); + }); + +#if PY_MAJOR_VERSION >= 3 + m.def("test_memoryview_frommemory", []() { + const char* buf = "\xff\xe1\xab\x37"; + return py::memoryview::frommemory( + const_cast(buf), strlen(buf), true); + }); +#endif } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 2faca7626f..5e5ff32042 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -264,10 +264,11 @@ def test_list_slicing(): @pytest.mark.parametrize('method, arg, fmt, expected_view', [ - (m.test_memoryview_fromobject, b'red', 'B', b'red'), - (m.test_memoryview_frombuffer_reference, b'green', 'B', b'green'), - (m.test_memoryview_frombuffer_new, False, 'h', [3, 1, 4, 1, 5]), - (m.test_memoryview_frombuffer_new, True, 'H', [2, 7, 1, 8]), + (m.test_memoryview_object, b'red', 'B', b'red'), + (m.test_memoryview_buffer_info, b'green', 'B', b'green'), + (m.test_memoryview_frombuffer, False, 'h', [3, 1, 4, 1, 5]), + (m.test_memoryview_frombuffer, True, 'H', [2, 7, 1, 8]), + (m.test_memoryview_frombuffer_nativeformat, None, '@i', [4, 7, 5]), ]) def test_memoryview(method, arg, fmt, expected_view): view = method(arg) @@ -282,8 +283,8 @@ def test_memoryview(method, arg, fmt, expected_view): @pytest.mark.parametrize('method', [ - m.test_memoryview_frombuffer_reference, - m.test_memoryview_fromobject, + m.test_memoryview_buffer_info, + m.test_memoryview_object, ]) def test_memoryview_refcount(method): buf = b'\x0a\x0b\x0c\x0d' @@ -292,3 +293,11 @@ def test_memoryview_refcount(method): ref_after = sys.getrefcount(buf) assert ref_before < ref_after assert list(view) == list(buf) + + +@pytest.mark.skipif(sys.version_info.major < 3, reason='API not available') +def test_memoryview_frommemory(): + view = m.test_memoryview_frommemory() + assert isinstance(view, memoryview) + assert view.format == 'B' + assert bytes(view) == b'\xff\xe1\xab\x37' From 1c08215e43c8f5bb8e6b9ddffc92d1043cde0e8c Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Mon, 13 Jul 2020 13:53:59 +0900 Subject: [PATCH 10/18] Remove ndim arg from memoryview::frombuffer and add tests --- include/pybind11/pytypes.h | 58 +++++++++++++++++++++----------------- tests/test_pytypes.cpp | 16 +++++++++-- tests/test_pytypes.py | 14 ++++++++- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 1177f490ff..92455253c1 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -13,7 +13,6 @@ #include "buffer_info.h" #include #include -#include NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -1357,16 +1356,20 @@ class memoryview : public object { /** \rst Creates ``memoryview`` from static buffer. - The caller is responsible for managing the lifetime of ``ptr`` and - ``format``. This method is meant for providing a ``memoryview`` for - C/C++ buffer not managed by Python. + This method is meant for providing a ``memoryview`` for C/C++ buffer not + managed by Python. The caller is responsible for managing the lifetime + of ``ptr`` and ``format``, which MUST outlive the memoryview constructed + here. + + See also: Python C API documentation for `PyMemoryView_FromBuffer`_. + + .. _PyMemoryView_FromBuffer: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromBuffer :param ptr: Pointer to the buffer. :param itemsize: Byte size of an element. :param format: Pointer to the null-terminated format string. For homogeneous Buffers, this should be set to - format_descriptor::value. - :param ndim: Number of dimensions. + ``format_descriptor::value``. :param shape_in: Shape of the tensor (1 entry per dimension). :param strides_in: Number of bytes between adjacent entries (for each per dimension). @@ -1374,31 +1377,34 @@ class memoryview : public object { written to. \endrst */ static memoryview frombuffer( - void *ptr, ssize_t itemsize, const char* format, ssize_t ndim, + void *ptr, ssize_t itemsize, const char* format, detail::any_container shape_in, - detail::any_container strides_in, bool readonly=false); + detail::any_container strides_in, bool readonly = false); template static memoryview frombuffer( T *ptr, detail::any_container shape_in, - detail::any_container strides_in, bool readonly=false) { + detail::any_container strides_in, bool readonly = false) { return memoryview::frombuffer( reinterpret_cast(ptr), sizeof(T), - format_descriptor::value, static_cast(shape_in->size()), - shape_in, strides_in, readonly); + format_descriptor::value, shape_in, strides_in, readonly); } #if PY_MAJOR_VERSION >= 3 /** \rst Creates ``memoryview`` from static memory. - The caller is responsible for managing the lifetime of ``mem``. This - constructor is meant for providing a ``memoryview`` for C/C++ buffer not - managed by Python. + This method is meant for providing a ``memoryview`` for C/C++ buffer not + managed by Python. The caller is responsible for managing the lifetime + of ``mem``, which MUST outlive the memoryview constructed here. + + See also: Python C API documentation for `PyMemoryView_FromBuffer`_. + + .. _PyMemoryView_FromMemory: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory \endrst */ - static memoryview frommemory(char *mem, ssize_t size, bool writable = false) { + static memoryview frommemory(char *mem, ssize_t size, bool readonly = false) { PyObject* ptr = PyMemoryView_FromMemory( - mem, size, (writable) ? PyBUF_WRITE : PyBUF_READ); + mem, size, (readonly) ? PyBUF_READ : PyBUF_WRITE); if (!ptr) pybind11_fail("Could not allocate memoryview object!"); return memoryview(object(ptr, stolen_t{})); @@ -1407,17 +1413,17 @@ class memoryview : public object { }; inline memoryview memoryview::frombuffer( - void *ptr, ssize_t itemsize, const char* format, ssize_t ndim, + void *ptr, ssize_t itemsize, const char* format, detail::any_container shape_in, detail::any_container strides_in, bool readonly) { - std::vector shape(std::move(shape_in)); - std::vector strides(std::move(strides_in)); - if (ndim != static_cast(shape.size()) || - ndim != static_cast(strides.size())) - pybind11_fail("memoryview: ndim doesn't match shape and/or strides length"); + size_t ndim = shape_in->size(); + if (ndim < 1) + pybind11_fail("memoryview: shape length (ndim) must be greater than zero"); + if (ndim != strides_in->size()) + pybind11_fail("memoryview: shape length doesn't match strides length"); ssize_t size = 1; - for (size_t i = 0; i < static_cast(ndim); ++i) - size *= shape[i]; + for (size_t i = 0; i < ndim; ++i) + size *= (*shape_in)[i]; Py_buffer view; view.buf = ptr; view.obj = nullptr; @@ -1426,8 +1432,8 @@ inline memoryview memoryview::frombuffer( view.itemsize = itemsize; view.format = const_cast(format); view.ndim = static_cast(ndim); - view.shape = shape.data(); - view.strides = strides.data(); + view.shape = shape_in->data(); + view.strides = strides_in->data(); view.suboffsets = nullptr; view.internal = nullptr; PyObject* obj = PyMemoryView_FromBuffer(&view); diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index e6f99601b5..ac828f310a 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -332,15 +332,27 @@ TEST_SUBMODULE(pytypes, m) { static const int32_t arr[] = { 4, 7, 5 }; unused.is_none(); // Only to suppress unused compiler warn. return py::memoryview::frombuffer( - const_cast(arr), sizeof(int32_t), format, 1, { 3 }, + const_cast(arr), sizeof(int32_t), format, { 3 }, { sizeof(int32_t) }, true); }); + m.def("test_memoryview_frombuffer_empty_shape", []() { + static const char* buf = "\x00\x01"; + return py::memoryview::frombuffer( + const_cast(buf), 1, "B", { }, { }); + }); + + m.def("test_memoryview_frombuffer_invalid_strides", []() { + static const char* buf = "\x02\x03\x04"; + return py::memoryview::frombuffer( + const_cast(buf), 1, "B", { 3 }, { }); + }); + #if PY_MAJOR_VERSION >= 3 m.def("test_memoryview_frommemory", []() { const char* buf = "\xff\xe1\xab\x37"; return py::memoryview::frommemory( - const_cast(buf), strlen(buf), true); + const_cast(buf), static_cast(strlen(buf)), true); }); #endif } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 5e5ff32042..0a5384b21c 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -282,9 +282,12 @@ def test_memoryview(method, arg, fmt, expected_view): assert view_as_list == list(expected_view) +@pytest.mark.skipif( + not hasattr(sys, 'getrefcount'), + reason='getrefcount is not available') @pytest.mark.parametrize('method', [ - m.test_memoryview_buffer_info, m.test_memoryview_object, + m.test_memoryview_buffer_info, ]) def test_memoryview_refcount(method): buf = b'\x0a\x0b\x0c\x0d' @@ -295,6 +298,15 @@ def test_memoryview_refcount(method): assert list(view) == list(buf) +@pytest.mark.parametrize('method', [ + m.test_memoryview_frombuffer_empty_shape, + m.test_memoryview_frombuffer_invalid_strides, +]) +def test_memoryview_failures(method): + with pytest.raises(RuntimeError): + method() + + @pytest.mark.skipif(sys.version_info.major < 3, reason='API not available') def test_memoryview_frommemory(): view = m.test_memoryview_frommemory() From db1a2f450676cc4b0dad50d1810500dc940f8577 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Tue, 14 Jul 2020 11:14:43 +0900 Subject: [PATCH 11/18] Allow ndim=0 memoryview and documentation fixup --- docs/Doxyfile | 2 ++ include/pybind11/pytypes.h | 47 ++++++++++++++++++++++---------------- tests/test_pytypes.cpp | 5 ++-- tests/test_pytypes.py | 35 ++++++++++++++++------------ 4 files changed, 52 insertions(+), 37 deletions(-) diff --git a/docs/Doxyfile b/docs/Doxyfile index 1b9d1297c5..24ece0d8db 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -18,3 +18,5 @@ ALIASES += "endrst=\endverbatim" QUIET = YES WARNINGS = YES WARN_IF_UNDOCUMENTED = NO +PREDEFINED = DOXYGEN_SHOULD_SKIP_THIS \ + PY_MAJOR_VERSION=3 diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 92455253c1..d9d5ab4039 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1370,25 +1370,21 @@ class memoryview : public object { :param format: Pointer to the null-terminated format string. For homogeneous Buffers, this should be set to ``format_descriptor::value``. - :param shape_in: Shape of the tensor (1 entry per dimension). - :param strides_in: Number of bytes between adjacent entries (for each + :param shape: Shape of the tensor (1 entry per dimension). + :param strides: Number of bytes between adjacent entries (for each per dimension). :param readonly: Flag to indicate if the underlying storage may be written to. \endrst */ static memoryview frombuffer( void *ptr, ssize_t itemsize, const char* format, - detail::any_container shape_in, - detail::any_container strides_in, bool readonly = false); + detail::any_container shape, + detail::any_container strides, bool readonly = false); template static memoryview frombuffer( - T *ptr, detail::any_container shape_in, - detail::any_container strides_in, bool readonly = false) { - return memoryview::frombuffer( - reinterpret_cast(ptr), sizeof(T), - format_descriptor::value, shape_in, strides_in, readonly); - } + T *ptr, detail::any_container shape, + detail::any_container strides, bool readonly = false); #if PY_MAJOR_VERSION >= 3 /** \rst @@ -1398,6 +1394,8 @@ class memoryview : public object { managed by Python. The caller is responsible for managing the lifetime of ``mem``, which MUST outlive the memoryview constructed here. + This method is not available in Python 2. + See also: Python C API documentation for `PyMemoryView_FromBuffer`_. .. _PyMemoryView_FromMemory: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory @@ -1412,18 +1410,17 @@ class memoryview : public object { #endif }; +#ifndef DOXYGEN_SHOULD_SKIP_THIS inline memoryview memoryview::frombuffer( void *ptr, ssize_t itemsize, const char* format, - detail::any_container shape_in, - detail::any_container strides_in, bool readonly) { - size_t ndim = shape_in->size(); - if (ndim < 1) - pybind11_fail("memoryview: shape length (ndim) must be greater than zero"); - if (ndim != strides_in->size()) + detail::any_container shape, + detail::any_container strides, bool readonly) { + size_t ndim = shape->size(); + if (ndim != strides->size()) pybind11_fail("memoryview: shape length doesn't match strides length"); - ssize_t size = 1; + ssize_t size = ndim ? 1 : 0; for (size_t i = 0; i < ndim; ++i) - size *= (*shape_in)[i]; + size *= (*shape)[i]; Py_buffer view; view.buf = ptr; view.obj = nullptr; @@ -1432,8 +1429,8 @@ inline memoryview memoryview::frombuffer( view.itemsize = itemsize; view.format = const_cast(format); view.ndim = static_cast(ndim); - view.shape = shape_in->data(); - view.strides = strides_in->data(); + view.shape = shape->data(); + view.strides = strides->data(); view.suboffsets = nullptr; view.internal = nullptr; PyObject* obj = PyMemoryView_FromBuffer(&view); @@ -1441,6 +1438,16 @@ inline memoryview memoryview::frombuffer( pybind11_fail("Unable to create memoryview from buffer structure"); return memoryview(object(obj, stolen_t{})); } +#endif // DOXYGEN_SHOULD_SKIP_THIS + +template +inline memoryview memoryview::frombuffer( + T *ptr, detail::any_container shape, + detail::any_container strides, bool readonly) { + return memoryview::frombuffer( + reinterpret_cast(ptr), sizeof(T), + format_descriptor::value, shape, strides, readonly); +} /// @} pytypes /// \addtogroup python_builtins diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index ac828f310a..164c45908b 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -327,17 +327,16 @@ TEST_SUBMODULE(pytypes, m) { const_cast(si16), { 5 }, { sizeof(int16_t) }, true); }); - m.def("test_memoryview_frombuffer_nativeformat", [](py::none unused) { + m.def("test_memoryview_frombuffer_nativeformat", []() { static const char* format = "@i"; static const int32_t arr[] = { 4, 7, 5 }; - unused.is_none(); // Only to suppress unused compiler warn. return py::memoryview::frombuffer( const_cast(arr), sizeof(int32_t), format, { 3 }, { sizeof(int32_t) }, true); }); m.def("test_memoryview_frombuffer_empty_shape", []() { - static const char* buf = "\x00\x01"; + static const char* buf = ""; return py::memoryview::frombuffer( const_cast(buf), 1, "B", { }, { }); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 0a5384b21c..c52aabedf9 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -263,15 +263,15 @@ def test_list_slicing(): assert li[::2] == m.test_list_slicing(li) -@pytest.mark.parametrize('method, arg, fmt, expected_view', [ - (m.test_memoryview_object, b'red', 'B', b'red'), - (m.test_memoryview_buffer_info, b'green', 'B', b'green'), - (m.test_memoryview_frombuffer, False, 'h', [3, 1, 4, 1, 5]), - (m.test_memoryview_frombuffer, True, 'H', [2, 7, 1, 8]), - (m.test_memoryview_frombuffer_nativeformat, None, '@i', [4, 7, 5]), +@pytest.mark.parametrize('method, args, fmt, expected_view', [ + (m.test_memoryview_object, (b'red',), 'B', b'red'), + (m.test_memoryview_buffer_info, (b'green',), 'B', b'green'), + (m.test_memoryview_frombuffer, (False,), 'h', [3, 1, 4, 1, 5]), + (m.test_memoryview_frombuffer, (True,), 'H', [2, 7, 1, 8]), + (m.test_memoryview_frombuffer_nativeformat, (), '@i', [4, 7, 5]), ]) -def test_memoryview(method, arg, fmt, expected_view): - view = method(arg) +def test_memoryview(method, args, fmt, expected_view): + view = method(*args) assert isinstance(view, memoryview) assert view.format == fmt if isinstance(expected_view, bytes) or sys.version_info[0] >= 3: @@ -298,13 +298,20 @@ def test_memoryview_refcount(method): assert list(view) == list(buf) -@pytest.mark.parametrize('method', [ - m.test_memoryview_frombuffer_empty_shape, - m.test_memoryview_frombuffer_invalid_strides, -]) -def test_memoryview_failures(method): +def test_memoryview_frombuffer_empty_shape(): + view = m.test_memoryview_frombuffer_empty_shape() + assert isinstance(view, memoryview) + assert view.format == 'B' + if sys.version_info.major < 3: + # Python 2 behavior is weird, but Python 3 (the future) is fine. + assert bytes(view).startswith(b' Date: Tue, 14 Jul 2020 11:51:44 +0900 Subject: [PATCH 12/18] Use void* to align to frombuffer method signature --- include/pybind11/pytypes.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index d9d5ab4039..00a9c68ec1 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1400,9 +1400,10 @@ class memoryview : public object { .. _PyMemoryView_FromMemory: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory \endrst */ - static memoryview frommemory(char *mem, ssize_t size, bool readonly = false) { + static memoryview frommemory(void *mem, ssize_t size, bool readonly = false) { PyObject* ptr = PyMemoryView_FromMemory( - mem, size, (readonly) ? PyBUF_READ : PyBUF_WRITE); + reinterpret_cast(mem), size, + (readonly) ? PyBUF_READ : PyBUF_WRITE); if (!ptr) pybind11_fail("Could not allocate memoryview object!"); return memoryview(object(ptr, stolen_t{})); From 640b45cd82674ba1696b6e228f2c83c0fdd902ef Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Tue, 14 Jul 2020 12:18:50 +0900 Subject: [PATCH 13/18] Add const variants of frombuffer and frommemory --- include/pybind11/pytypes.h | 37 ++++++++++++++++++++++++++----------- tests/test_pytypes.cpp | 15 ++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 00a9c68ec1..70fa4973bb 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1377,14 +1377,34 @@ class memoryview : public object { written to. \endrst */ static memoryview frombuffer( - void *ptr, ssize_t itemsize, const char* format, + void *ptr, ssize_t itemsize, const char *format, detail::any_container shape, detail::any_container strides, bool readonly = false); + static memoryview frombuffer( + const void *ptr, ssize_t itemsize, const char *format, + detail::any_container shape, + detail::any_container strides) { + return memoryview::frombuffer( + const_cast(ptr), itemsize, format, shape, strides, true); + } + template static memoryview frombuffer( T *ptr, detail::any_container shape, - detail::any_container strides, bool readonly = false); + detail::any_container strides, bool readonly = false) { + return memoryview::frombuffer( + reinterpret_cast(ptr), sizeof(T), + format_descriptor::value, shape, strides, readonly); + } + + template + static memoryview frombuffer( + const T *ptr, detail::any_container shape, + detail::any_container strides) { + return memoryview::frombuffer( + const_cast(ptr), shape, strides, true); + } #if PY_MAJOR_VERSION >= 3 /** \rst @@ -1408,6 +1428,10 @@ class memoryview : public object { pybind11_fail("Could not allocate memoryview object!"); return memoryview(object(ptr, stolen_t{})); } + + static memoryview frommemory(const void *mem, ssize_t size) { + return memoryview::frommemory(const_cast(mem), size, true); + } #endif }; @@ -1440,15 +1464,6 @@ inline memoryview memoryview::frombuffer( return memoryview(object(obj, stolen_t{})); } #endif // DOXYGEN_SHOULD_SKIP_THIS - -template -inline memoryview memoryview::frombuffer( - T *ptr, detail::any_container shape, - detail::any_container strides, bool readonly) { - return memoryview::frombuffer( - reinterpret_cast(ptr), sizeof(T), - format_descriptor::value, shape, strides, readonly); -} /// @} pytypes /// \addtogroup python_builtins diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 164c45908b..6258deb9d2 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -321,37 +321,34 @@ TEST_SUBMODULE(pytypes, m) { static const uint16_t ui16[] = { 2, 7, 1, 8 }; if (is_unsigned) return py::memoryview::frombuffer( - const_cast(ui16), { 4 }, { sizeof(uint16_t) }, true); + ui16, { 4 }, { sizeof(uint16_t) }); else return py::memoryview::frombuffer( - const_cast(si16), { 5 }, { sizeof(int16_t) }, true); + si16, { 5 }, { sizeof(int16_t) }); }); m.def("test_memoryview_frombuffer_nativeformat", []() { static const char* format = "@i"; static const int32_t arr[] = { 4, 7, 5 }; return py::memoryview::frombuffer( - const_cast(arr), sizeof(int32_t), format, { 3 }, - { sizeof(int32_t) }, true); + arr, sizeof(int32_t), format, { 3 }, { sizeof(int32_t) }); }); m.def("test_memoryview_frombuffer_empty_shape", []() { static const char* buf = ""; - return py::memoryview::frombuffer( - const_cast(buf), 1, "B", { }, { }); + return py::memoryview::frombuffer(buf, 1, "B", { }, { }); }); m.def("test_memoryview_frombuffer_invalid_strides", []() { static const char* buf = "\x02\x03\x04"; - return py::memoryview::frombuffer( - const_cast(buf), 1, "B", { 3 }, { }); + return py::memoryview::frombuffer(buf, 1, "B", { 3 }, { }); }); #if PY_MAJOR_VERSION >= 3 m.def("test_memoryview_frommemory", []() { const char* buf = "\xff\xe1\xab\x37"; return py::memoryview::frommemory( - const_cast(buf), static_cast(strlen(buf)), true); + buf, static_cast(strlen(buf))); }); #endif } From 8d508eec002578b63310269b00c8e3f9124c1f47 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Tue, 14 Jul 2020 12:19:13 +0900 Subject: [PATCH 14/18] Add memory view section in doc --- docs/advanced/pycpp/numpy.rst | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index 458f99e978..f548c90aab 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -384,3 +384,45 @@ operation on the C++ side: py::array a = /* A NumPy array */; py::array b = a[py::make_tuple(0, py::ellipsis(), 0)]; + +Memory view +=========== + +For a case when we simply want to provide a direct accessor to C/C++ buffer +without a concrete class object, we can return a ``memoryview`` object. Suppose +we wish to expose a ``memoryview`` for 2x4 uint8_t array, we can do the +following: + +.. code-block:: cpp + + const uint8_t buffer[] = { + 0, 1, 2, 3, + 4, 5, 6, 7 + }; + m.def("get_memoryview2d", []() { + return py::memoryview::frombuffer( + buffer, // buffer pointer + { 2, 4 }, // shape (rows, cols) + { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes + ); + }) + +This approach is meant for providing a ``memoryview`` for C/C++ buffer not +managed by Python. The user is responsible for managing the lifetime of the +buffer. Using ``memoryview`` created from this approach after deleting the +buffer in C++ side results in undefined behavior. + +We can also use ``memoryview::frommemory`` for a simple 1D contiguous buffer: + +.. code-block:: cpp + + m.def("get_memoryview1d", []() { + return py::memoryview::frommemory( + buffer, // buffer pointer + sizeof(uint8_t) * 8 // buffer size + ); + }) + +.. note:: + + ``memoryview::frommemory`` is not available in Python 2. From 841ba2524cda3f7ccc8a17883745158ab6544eb6 Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Wed, 15 Jul 2020 10:06:38 +0900 Subject: [PATCH 15/18] Fix docs --- docs/advanced/pycpp/numpy.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index f548c90aab..d6b1288616 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -407,10 +407,10 @@ following: ); }) -This approach is meant for providing a ``memoryview`` for C/C++ buffer not +This approach is meant for providing a ``memoryview`` for a C/C++ buffer not managed by Python. The user is responsible for managing the lifetime of the -buffer. Using ``memoryview`` created from this approach after deleting the -buffer in C++ side results in undefined behavior. +buffer. Using a ``memoryview`` created in this way after deleting the buffer in +C++ side results in undefined behavior. We can also use ``memoryview::frommemory`` for a simple 1D contiguous buffer: From 41cd25ae63f156e119f5a62b251825f20ba5496f Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Wed, 15 Jul 2020 10:15:24 +0900 Subject: [PATCH 16/18] Add test for null buffer --- include/pybind11/pytypes.h | 2 +- tests/test_pytypes.cpp | 5 +++++ tests/test_pytypes.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 70fa4973bb..75d1c881d8 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1460,7 +1460,7 @@ inline memoryview memoryview::frombuffer( view.internal = nullptr; PyObject* obj = PyMemoryView_FromBuffer(&view); if (!obj) - pybind11_fail("Unable to create memoryview from buffer structure"); + throw error_already_set(); return memoryview(object(obj, stolen_t{})); } #endif // DOXYGEN_SHOULD_SKIP_THIS diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 6258deb9d2..0ee78c0fc1 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -344,6 +344,11 @@ TEST_SUBMODULE(pytypes, m) { return py::memoryview::frombuffer(buf, 1, "B", { 3 }, { }); }); + m.def("test_memoryview_frombuffer_nullptr", []() { + return py::memoryview::frombuffer( + static_cast(nullptr), 1, "B", { }, { }); + }); + #if PY_MAJOR_VERSION >= 3 m.def("test_memoryview_frommemory", []() { const char* buf = "\xff\xe1\xab\x37"; diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index c52aabedf9..8427efcd05 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -314,6 +314,11 @@ def test_test_memoryview_frombuffer_invalid_strides(): m.test_memoryview_frombuffer_invalid_strides() +def test_test_memoryview_frombuffer_nullptr(): + with pytest.raises(ValueError): + m.test_memoryview_frombuffer_nullptr() + + @pytest.mark.skipif(sys.version_info.major < 3, reason='API not available') def test_memoryview_frommemory(): view = m.test_memoryview_frommemory() From 2719c94610daa7c02220aa59ce1e5c493cedc58f Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Wed, 15 Jul 2020 11:28:52 +0900 Subject: [PATCH 17/18] Workaround py27 nullptr behavior in test --- tests/test_pytypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 8427efcd05..550f7795d0 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -315,8 +315,11 @@ def test_test_memoryview_frombuffer_invalid_strides(): def test_test_memoryview_frombuffer_nullptr(): - with pytest.raises(ValueError): + if sys.version_info.major < 3: m.test_memoryview_frombuffer_nullptr() + else: + with pytest.raises(ValueError): + m.test_memoryview_frombuffer_nullptr() @pytest.mark.skipif(sys.version_info.major < 3, reason='API not available') From 7f64f8f7453dd038d4192c767e4b76ede2570a7e Mon Sep 17 00:00:00 2001 From: Kota Yamaguchi Date: Wed, 15 Jul 2020 18:13:16 +0900 Subject: [PATCH 18/18] Rename frombuffer to from_buffer --- docs/advanced/pycpp/numpy.rst | 8 ++++---- include/pybind11/pytypes.h | 22 +++++++++++----------- tests/test_pytypes.cpp | 26 +++++++++++++------------- tests/test_pytypes.py | 24 ++++++++++++------------ 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index d6b1288616..0ae592ed56 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -400,7 +400,7 @@ following: 4, 5, 6, 7 }; m.def("get_memoryview2d", []() { - return py::memoryview::frombuffer( + return py::memoryview::from_buffer( buffer, // buffer pointer { 2, 4 }, // shape (rows, cols) { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes @@ -412,12 +412,12 @@ managed by Python. The user is responsible for managing the lifetime of the buffer. Using a ``memoryview`` created in this way after deleting the buffer in C++ side results in undefined behavior. -We can also use ``memoryview::frommemory`` for a simple 1D contiguous buffer: +We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: .. code-block:: cpp m.def("get_memoryview1d", []() { - return py::memoryview::frommemory( + return py::memoryview::from_memory( buffer, // buffer pointer sizeof(uint8_t) * 8 // buffer size ); @@ -425,4 +425,4 @@ We can also use ``memoryview::frommemory`` for a simple 1D contiguous buffer: .. note:: - ``memoryview::frommemory`` is not available in Python 2. + ``memoryview::from_memory`` is not available in Python 2. diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 75d1c881d8..6afea9c911 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1376,33 +1376,33 @@ class memoryview : public object { :param readonly: Flag to indicate if the underlying storage may be written to. \endrst */ - static memoryview frombuffer( + static memoryview from_buffer( void *ptr, ssize_t itemsize, const char *format, detail::any_container shape, detail::any_container strides, bool readonly = false); - static memoryview frombuffer( + static memoryview from_buffer( const void *ptr, ssize_t itemsize, const char *format, detail::any_container shape, detail::any_container strides) { - return memoryview::frombuffer( + return memoryview::from_buffer( const_cast(ptr), itemsize, format, shape, strides, true); } template - static memoryview frombuffer( + static memoryview from_buffer( T *ptr, detail::any_container shape, detail::any_container strides, bool readonly = false) { - return memoryview::frombuffer( + return memoryview::from_buffer( reinterpret_cast(ptr), sizeof(T), format_descriptor::value, shape, strides, readonly); } template - static memoryview frombuffer( + static memoryview from_buffer( const T *ptr, detail::any_container shape, detail::any_container strides) { - return memoryview::frombuffer( + return memoryview::from_buffer( const_cast(ptr), shape, strides, true); } @@ -1420,7 +1420,7 @@ class memoryview : public object { .. _PyMemoryView_FromMemory: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory \endrst */ - static memoryview frommemory(void *mem, ssize_t size, bool readonly = false) { + static memoryview from_memory(void *mem, ssize_t size, bool readonly = false) { PyObject* ptr = PyMemoryView_FromMemory( reinterpret_cast(mem), size, (readonly) ? PyBUF_READ : PyBUF_WRITE); @@ -1429,14 +1429,14 @@ class memoryview : public object { return memoryview(object(ptr, stolen_t{})); } - static memoryview frommemory(const void *mem, ssize_t size) { - return memoryview::frommemory(const_cast(mem), size, true); + static memoryview from_memory(const void *mem, ssize_t size) { + return memoryview::from_memory(const_cast(mem), size, true); } #endif }; #ifndef DOXYGEN_SHOULD_SKIP_THIS -inline memoryview memoryview::frombuffer( +inline memoryview memoryview::from_buffer( void *ptr, ssize_t itemsize, const char* format, detail::any_container shape, detail::any_container strides, bool readonly) { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 0ee78c0fc1..eb5b934229 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -316,43 +316,43 @@ TEST_SUBMODULE(pytypes, m) { return py::memoryview(b.request()); }); - m.def("test_memoryview_frombuffer", [](bool is_unsigned) { + m.def("test_memoryview_from_buffer", [](bool is_unsigned) { static const int16_t si16[] = { 3, 1, 4, 1, 5 }; static const uint16_t ui16[] = { 2, 7, 1, 8 }; if (is_unsigned) - return py::memoryview::frombuffer( + return py::memoryview::from_buffer( ui16, { 4 }, { sizeof(uint16_t) }); else - return py::memoryview::frombuffer( + return py::memoryview::from_buffer( si16, { 5 }, { sizeof(int16_t) }); }); - m.def("test_memoryview_frombuffer_nativeformat", []() { + m.def("test_memoryview_from_buffer_nativeformat", []() { static const char* format = "@i"; static const int32_t arr[] = { 4, 7, 5 }; - return py::memoryview::frombuffer( + return py::memoryview::from_buffer( arr, sizeof(int32_t), format, { 3 }, { sizeof(int32_t) }); }); - m.def("test_memoryview_frombuffer_empty_shape", []() { + m.def("test_memoryview_from_buffer_empty_shape", []() { static const char* buf = ""; - return py::memoryview::frombuffer(buf, 1, "B", { }, { }); + return py::memoryview::from_buffer(buf, 1, "B", { }, { }); }); - m.def("test_memoryview_frombuffer_invalid_strides", []() { + m.def("test_memoryview_from_buffer_invalid_strides", []() { static const char* buf = "\x02\x03\x04"; - return py::memoryview::frombuffer(buf, 1, "B", { 3 }, { }); + return py::memoryview::from_buffer(buf, 1, "B", { 3 }, { }); }); - m.def("test_memoryview_frombuffer_nullptr", []() { - return py::memoryview::frombuffer( + m.def("test_memoryview_from_buffer_nullptr", []() { + return py::memoryview::from_buffer( static_cast(nullptr), 1, "B", { }, { }); }); #if PY_MAJOR_VERSION >= 3 - m.def("test_memoryview_frommemory", []() { + m.def("test_memoryview_from_memory", []() { const char* buf = "\xff\xe1\xab\x37"; - return py::memoryview::frommemory( + return py::memoryview::from_memory( buf, static_cast(strlen(buf))); }); #endif diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 550f7795d0..287f84229f 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -266,9 +266,9 @@ def test_list_slicing(): @pytest.mark.parametrize('method, args, fmt, expected_view', [ (m.test_memoryview_object, (b'red',), 'B', b'red'), (m.test_memoryview_buffer_info, (b'green',), 'B', b'green'), - (m.test_memoryview_frombuffer, (False,), 'h', [3, 1, 4, 1, 5]), - (m.test_memoryview_frombuffer, (True,), 'H', [2, 7, 1, 8]), - (m.test_memoryview_frombuffer_nativeformat, (), '@i', [4, 7, 5]), + (m.test_memoryview_from_buffer, (False,), 'h', [3, 1, 4, 1, 5]), + (m.test_memoryview_from_buffer, (True,), 'H', [2, 7, 1, 8]), + (m.test_memoryview_from_buffer_nativeformat, (), '@i', [4, 7, 5]), ]) def test_memoryview(method, args, fmt, expected_view): view = method(*args) @@ -298,8 +298,8 @@ def test_memoryview_refcount(method): assert list(view) == list(buf) -def test_memoryview_frombuffer_empty_shape(): - view = m.test_memoryview_frombuffer_empty_shape() +def test_memoryview_from_buffer_empty_shape(): + view = m.test_memoryview_from_buffer_empty_shape() assert isinstance(view, memoryview) assert view.format == 'B' if sys.version_info.major < 3: @@ -309,22 +309,22 @@ def test_memoryview_frombuffer_empty_shape(): assert bytes(view) == b'' -def test_test_memoryview_frombuffer_invalid_strides(): +def test_test_memoryview_from_buffer_invalid_strides(): with pytest.raises(RuntimeError): - m.test_memoryview_frombuffer_invalid_strides() + m.test_memoryview_from_buffer_invalid_strides() -def test_test_memoryview_frombuffer_nullptr(): +def test_test_memoryview_from_buffer_nullptr(): if sys.version_info.major < 3: - m.test_memoryview_frombuffer_nullptr() + m.test_memoryview_from_buffer_nullptr() else: with pytest.raises(ValueError): - m.test_memoryview_frombuffer_nullptr() + m.test_memoryview_from_buffer_nullptr() @pytest.mark.skipif(sys.version_info.major < 3, reason='API not available') -def test_memoryview_frommemory(): - view = m.test_memoryview_frommemory() +def test_memoryview_from_memory(): + view = m.test_memoryview_from_memory() assert isinstance(view, memoryview) assert view.format == 'B' assert bytes(view) == b'\xff\xe1\xab\x37'