Make py::isinstance<py::str> less permissive

docs/changelog.rst

@@ -6,7 +6,7 @@ Changelog
 Starting with version 1.8.0, pybind11 releases use a `semantic versioning
 <http://semver.org>`_ policy.
 
-v2.3.0 (Not yet released)
+v2.3.0 (June 11, 2019)
 -----------------------------------------------------
 
 * Significantly reduced module binary size (10-20%) when compiled in C++11 mode
@@ -19,9 +19,6 @@ v2.3.0 (Not yet released)
   provide a method to returns the desired type of an instance.
   `#1326 <https://github.com/pybind/pybind11/pull/1326>`_.
 
-* Added support for write only properties.
-  `#1144 <https://github.com/pybind/pybind11/pull/1144>`_.
-
 * Python type wrappers (``py::handle``, ``py::object``, etc.)
   now support map Python's number protocol onto C++ arithmetic
   operators such as ``operator+``, ``operator/=``, etc.
@@ -41,15 +38,57 @@ v2.3.0 (Not yet released)
    3. check for already existing enum value and throw an error if present.
       `#1453 <https://github.com/pybind/pybind11/pull/1453>`_.
 
-* added ``py::ellipsis()`` method for slicing of multidimensional NumPy arrays
+* Support for over-aligned type allocation via C++17's aligned ``new``
+  statement. `#1582 <https://github.com/pybind/pybind11/pull/1582>`_.
+
+* Added ``py::ellipsis()`` method for slicing of multidimensional NumPy arrays
   `#1502 <https://github.com/pybind/pybind11/pull/1502>`_.
 
+* Numerous Improvements to the ``mkdoc.py`` script for extracting documentation
+  from C++ header files.
+  `#1788 <https://github.com/pybind/pybind11/pull/1788>`_.
+
 * ``pybind11_add_module()``: allow including Python as a ``SYSTEM`` include path.
   `#1416 <https://github.com/pybind/pybind11/pull/1416>`_.
 
 * ``pybind11/stl.h`` does not convert strings to ``vector<string>`` anymore.
   `#1258 <https://github.com/pybind/pybind11/issues/1258>`_.
 
+* Mark static methods as such to fix auto-generated Sphinx documentation.
+  `#1732 <https://github.com/pybind/pybind11/pull/1732>`_.
+
+* Re-throw forced unwind exceptions (e.g. during pthread termination).
+  `#1208 <https://github.com/pybind/pybind11/pull/1208>`_.
+
+* Added ``__contains__`` method to the bindings of maps (``std::map``,
+  ``std::unordered_map``).
+  `#1767 <https://github.com/pybind/pybind11/pull/1767>`_.
+
+* Improvements to ``gil_scoped_acquire``.
+  `#1211 <https://github.com/pybind/pybind11/pull/1211>`_.
+
+* Type caster support for ``std::deque<T>``.
+  `#1609 <https://github.com/pybind/pybind11/pull/1609>`_.
+
+* Support for ``std::unique_ptr`` holders, whose deleters differ between a base and derived
+  class. `#1353 <https://github.com/pybind/pybind11/pull/1353>`_.
+
+* Fixes regarding return value policy propagation in STL type casters.
+  `#1603 <https://github.com/pybind/pybind11/pull/1603>`_.
+
+* Fixed scoped enum comparisons.
+  `#1571 <https://github.com/pybind/pybind11/pull/1571>`_.
+
+* CMake build system improvements for projects that include non-C++
+  files (e.g. plain C, CUDA) in ``pybind11_add_module`` et al.
+  `#1678 <https://github.com/pybind/pybind11/pull/1678>`_.
+
+* A number of CI-related fixes.
+  `#1757 <https://github.com/pybind/pybind11/pull/1757>`_,
+  `#1744 <https://github.com/pybind/pybind11/pull/1744>`_,
+  `#1670 <https://github.com/pybind/pybind11/pull/1670>`_.
+
+
 v2.2.4 (September 11, 2018)
 -----------------------------------------------------
 
@@ -94,6 +133,11 @@ v2.2.4 (September 11, 2018)
 * A few minor typo fixes and improvements to the test suite, and
   patches that silence compiler warnings.
 
+* Vectors now support construction from generators, as well as ``extend()`` from a
+  list or generator.
+  `#1496 <https://github.com/pybind/pybind11/pull/1496>`_.
+
+
 v2.2.3 (April 29, 2018)
 -----------------------------------------------------
 
@@ -403,6 +447,9 @@ v2.2.0 (August 31, 2017)
 * Fixed overriding static properties in derived classes.
   `#784 <https://github.com/pybind/pybind11/pull/784>`_.
 
+* Added support for write only properties.
+  `#1144 <https://github.com/pybind/pybind11/pull/1144>`_.
+
 * Improved deduction of member functions of a derived class when its bases
   aren't registered with pybind11.
   `#855 <https://github.com/pybind/pybind11/pull/855>`_.

include/pybind11/cast.h

@@ -204,10 +204,10 @@ PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool t
 }
 
 struct value_and_holder {
-    instance *inst;
-    size_t index;
-    const detail::type_info *type;
-    void **vh;
+    instance *inst = nullptr;
+    size_t index = 0u;
+    const detail::type_info *type = nullptr;
+    void **vh = nullptr;
 
     // Main constructor for a found value/holder:
     value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) :
@@ -216,7 +216,7 @@ struct value_and_holder {
     {}
 
     // Default constructor (used to signal a value-and-holder not found by get_value_and_holder())
-    value_and_holder() : inst{nullptr} {}
+    value_and_holder() {}
 
     // Used for past-the-end iterator
     value_and_holder(size_t index) : index{index} {}
@@ -270,8 +270,8 @@ struct values_and_holders {
 
     struct iterator {
     private:
-        instance *inst;
-        const type_vec *types;
+        instance *inst = nullptr;
+        const type_vec *types = nullptr;
         value_and_holder curr;
         friend struct values_and_holders;
         iterator(instance *inst, const type_vec *tinfo)

include/pybind11/functional.h

@@ -54,9 +54,20 @@ struct type_caster<std::function<Return(Args...)>> {
             }
         }
 
-        value = [func](Args... args) -> Return {
+        // ensure GIL is held during functor destruction
+        struct func_handle {
+            function f;
+            func_handle(function&& f_) : f(std::move(f_)) {}
+            func_handle(const func_handle&) = default;
+            ~func_handle() {
+                gil_scoped_acquire acq;
+                function kill_f(std::move(f));
+            }
+        };
+
+        value = [hfunc = func_handle(std::move(func))](Args... args) -> Return {
             gil_scoped_acquire acq;
-            object retval(func(std::forward<Args>(args)...));
+            object retval(hfunc.f(std::forward<Args>(args)...));
             /* Visual studio 2015 parser issue: need parentheses around this expression */
             return (retval.template cast<Return>());
         };

include/pybind11/iostream.h

@@ -25,7 +25,8 @@ class pythonbuf : public std::streambuf {
 private:
     using traits_type = std::streambuf::traits_type;
 
-    char d_buffer[1024];
+    const size_t buf_size;
+    std::unique_ptr<char[]> d_buffer;
     object pywrite;
     object pyflush;
 
@@ -42,19 +43,25 @@ class pythonbuf : public std::streambuf {
             // This subtraction cannot be negative, so dropping the sign
             str line(pbase(), static_cast<size_t>(pptr() - pbase()));
 
-            pywrite(line);
-            pyflush();
+            {
+                gil_scoped_acquire tmp;
+                pywrite(line);
+                pyflush();
+            }
 
             setp(pbase(), epptr());
         }
         return 0;
     }
 
 public:
-    pythonbuf(object pyostream)
-        : pywrite(pyostream.attr("write")),
+
+    pythonbuf(object pyostream, size_t buffer_size = 1024)
+        : buf_size(buffer_size),
+          d_buffer(new char[buf_size]),
+          pywrite(pyostream.attr("write")),
           pyflush(pyostream.attr("flush")) {
-        setp(d_buffer, d_buffer + sizeof(d_buffer) - 1);
+        setp(d_buffer.get(), d_buffer.get() + buf_size - 1);
     }
 
     /// Sync before destroy

include/pybind11/pybind11.h

@@ -1185,7 +1185,7 @@ class class_ : public detail::generic_type {
 
     template <typename C, typename D, typename... Extra>
     class_ &def_readwrite(const char *name, D C::*pm, const Extra&... extra) {
-        static_assert(std::is_base_of<C, type>::value, "def_readwrite() requires a class member (or base class member)");
+        static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, "def_readwrite() requires a class member (or base class member)");
         cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this)),
                      fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this));
         def_property(name, fget, fset, return_value_policy::reference_internal, extra...);
@@ -1194,7 +1194,7 @@ class class_ : public detail::generic_type {
 
     template <typename C, typename D, typename... Extra>
     class_ &def_readonly(const char *name, const D C::*pm, const Extra& ...extra) {
-        static_assert(std::is_base_of<C, type>::value, "def_readonly() requires a class member (or base class member)");
+        static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, "def_readonly() requires a class member (or base class member)");
         cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this));
         def_property_readonly(name, fget, return_value_policy::reference_internal, extra...);
         return *this;

include/pybind11/pytypes.h

@@ -943,6 +943,9 @@ class str : public object {
 };
 /// @} pytypes
 
+// str::check_ is too permissive for `isinstance` (in particular it allows a `bytes` object)
+template <> inline bool isinstance<str>(handle obj) { return PyUnicode_Check(obj.ptr()); }
+
 inline namespace literals {
 /** \rst
     String literal version of `str`
@@ -1346,6 +1349,21 @@ inline size_t len(handle h) {
     return (size_t) result;
 }
 
+inline size_t len_hint(handle h) {
+#if PY_VERSION_HEX >= 0x03040000
+    ssize_t result = PyObject_LengthHint(h.ptr(), 0);
+#else
+    ssize_t result = PyObject_Length(h.ptr());
+#endif
+    if (result < 0) {
+        // Sometimes a length can't be determined at all (eg generators)
+        // In which case simply return 0
+        PyErr_Clear();
+        return 0;
+    }
+    return (size_t) result;
+}
+
 inline str repr(handle h) {
     PyObject *str_value = PyObject_Repr(h.ptr());
     if (!str_value) throw error_already_set();

include/pybind11/stl_bind.h

@@ -122,7 +122,7 @@ void vector_modifiers(enable_if_t<is_copy_constructible<typename Vector::value_t
 
     cl.def(init([](iterable it) {
         auto v = std::unique_ptr<Vector>(new Vector());
-        v->reserve(len(it));
+        v->reserve(len_hint(it));
         for (handle h : it)
            v->push_back(h.cast<T>());
         return v.release();
@@ -136,6 +136,28 @@ void vector_modifiers(enable_if_t<is_copy_constructible<typename Vector::value_t
        "Extend the list by appending all the items in the given list"
     );
 
+    cl.def("extend",
+       [](Vector &v, iterable it) {
+           const size_t old_size = v.size();
+           v.reserve(old_size + len_hint(it));
+           try {
+               for (handle h : it) {
+                   v.push_back(h.cast<T>());
+               }
+           } catch (const cast_error &) {
+               v.erase(v.begin() + static_cast<typename Vector::difference_type>(old_size), v.end());
+               try {
+                   v.shrink_to_fit();
+               } catch (const std::exception &) {
+                   // Do nothing
+               }
+               throw;
+           }
+       },
+       arg("L"),
+       "Extend the list by appending all the items in the given list"
+    );
+
     cl.def("insert",
         [](Vector &v, SizeType i, const T &x) {
             if (i > v.size())

tests/CMakeLists.txt

@@ -59,6 +59,7 @@ set(PYBIND11_TEST_FILES
   test_stl.cpp
   test_stl_binders.cpp
   test_tagbased_polymorphic.cpp
+  test_union.cpp
   test_virtual_functions.cpp
 )
 

tests/test_callbacks.cpp

@@ -10,6 +10,7 @@
 #include "pybind11_tests.h"
 #include "constructor_stats.h"
 #include <pybind11/functional.h>
+#include <thread>
 
 
 int dummy_function(int i) { return i + 1; }
@@ -146,4 +147,22 @@ TEST_SUBMODULE(callbacks, m) {
     py::class_<CppBoundMethodTest>(m, "CppBoundMethodTest")
         .def(py::init<>())
         .def("triple", [](CppBoundMethodTest &, int val) { return 3 * val; });
+
+    // test async Python callbacks
+    using callback_f = std::function<void(int)>;
+    m.def("test_async_callback", [](callback_f f, py::list work) {
+        // make detached thread that calls `f` with piece of work after a little delay
+        auto start_f = [f](int j) {
+            auto invoke_f = [f, j] {
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                f(j);
+            };
+            auto t = std::thread(std::move(invoke_f));
+            t.detach();
+        };
+
+        // spawn worker threads
+        for (auto i : work)
+            start_f(py::cast<int>(i));
+    });
 }

tests/test_callbacks.py

@@ -1,5 +1,6 @@
 import pytest
 from pybind11_tests import callbacks as m
+from threading import Thread
 
 
 def test_callbacks():
@@ -105,3 +106,31 @@ def test_function_signatures(doc):
 
 def test_movable_object():
     assert m.callback_with_movable(lambda _: None) is True
+
+
+def test_async_callbacks():
+    # serves as state for async callback
+    class Item:
+        def __init__(self, value):
+            self.value = value
+
+    res = []
+
+    # generate stateful lambda that will store result in `res`
+    def gen_f():
+        s = Item(3)
+        return lambda j: res.append(s.value + j)
+
+    # do some work async
+    work = [1, 2, 3, 4]
+    m.test_async_callback(gen_f(), work)
+    # wait until work is done
+    from time import sleep
+    sleep(0.5)
+    assert sum(res) == sum([x + 3 for x in work])
+
+
+def test_async_async_callbacks():
+    t = Thread(target=test_async_callbacks)
+    t.start()
+    t.join()

tests/test_pytypes.cpp

@@ -270,6 +270,10 @@ TEST_SUBMODULE(pytypes, m) {
 
     m.def("hash_function", [](py::object obj) { return py::hash(obj); });
 
+
+    // test_str_isinstance
+    m.def("is_str_instance", [](py::object o) { return py::isinstance<py::str>(o); });
+
     m.def("test_number_protocol", [](py::object a, py::object b) {
         py::list l;
         l.append(a.equal(b));

tests/test_pytypes.py

@@ -241,6 +241,11 @@ class Unhashable(object):
         m.hash_function(Unhashable())
 
 
+def test_str_isinstance():
+    assert m.is_str_instance(u"abc")
+    assert not m.is_str_instance(b"abc")
+
+
 def test_number_protocol():
     for a, b in [(1, 1), (3, 5)]:
         li = [a == b, a != b, a < b, a <= b, a > b, a >= b, a + b,