Reimplement static properties by extending PyProperty_Type

Instead of creating a new unique metaclass for each type, the builtin
`property` type is subclassed to support static properties. The new
setter/getters always pass types instead of instances in their `self`
argument. A metaclass is still required to support this behavior, but
it doesn't store any data anymore, so a new one doesn't need to be
created for each class. There is now only one common metaclass which
is shared by all pybind11 types.

Author: dean0x7d

CMakeLists.txt

@@ -56,6 +56,7 @@ set(PYBIND11_HEADERS
   include/pybind11/attr.h
   include/pybind11/cast.h
   include/pybind11/chrono.h
+  include/pybind11/class_support.h
   include/pybind11/common.h
   include/pybind11/complex.h
   include/pybind11/descr.h

include/pybind11/cast.h

@@ -18,6 +18,8 @@
 
 NAMESPACE_BEGIN(pybind11)
 NAMESPACE_BEGIN(detail)
+inline PyTypeObject *make_static_property_type();
+inline PyTypeObject *make_default_metaclass();
 
 /// Additional type information which does not fit into the PyTypeObject
 struct type_info {
@@ -73,6 +75,8 @@ PYBIND11_NOINLINE inline internals &get_internals() {
                 }
             }
         );
+        internals_ptr->static_property_type = make_static_property_type();
+        internals_ptr->default_metaclass = make_default_metaclass();
     }
     return *internals_ptr;
 }

include/pybind11/class_support.h

@@ -0,0 +1,147 @@
+/*
+    pybind11/class_support.h: Python C API implementation details for py::class_
+
+    Copyright (c) 2017 Wenzel Jakob <[email protected]>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "attr.h"
+
+NAMESPACE_BEGIN(pybind11)
+NAMESPACE_BEGIN(detail)
+
+#if !defined(PYPY_VERSION)
+
+/// `pybind11_static_property.__get__()`: Always pass the class instead of the instance.
+extern "C" inline PyObject *pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls) {
+    return PyProperty_Type.tp_descr_get(self, cls, cls);
+}
+
+/// `pybind11_static_property.__set__()`: Just like the above `__get__()`.
+extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) {
+    PyObject *cls = PyType_Check(obj) ? obj : (PyObject *) Py_TYPE(obj);
+    return PyProperty_Type.tp_descr_set(self, cls, value);
+}
+
+/** A `static_property` is the same as a `property` but the `__get__()` and `__set__()`
+    methods are modified to always use the object type instead of a concrete instance.
+    Return value: New reference. */
+inline PyTypeObject *make_static_property_type() {
+    constexpr auto *name = "pybind11_static_property";
+    auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name));
+
+    /* Danger zone: from now (and until PyType_Ready), make sure to
+       issue no Python C API calls which could potentially invoke the
+       garbage collector (the GC will call type_traverse(), which will in
+       turn find the newly constructed type in an invalid state) */
+    auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
+    if (!heap_type)
+        pybind11_fail("make_static_property_type(): error allocating type!");
+
+    heap_type->ht_name = name_obj.inc_ref().ptr();
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
+    heap_type->ht_qualname = name_obj.inc_ref().ptr();
+#endif
+
+    auto type = &heap_type->ht_type;
+    type->tp_name = name;
+    type->tp_base = &PyProperty_Type;
+    type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
+    type->tp_descr_get = pybind11_static_get;
+    type->tp_descr_set = pybind11_static_set;
+
+    if (PyType_Ready(type) < 0)
+        pybind11_fail("make_static_property_type(): failure in PyType_Ready()!");
+
+    return type;
+}
+
+#else // PYPY
+
+/** PyPy has some issues with the above C API, so we evaluate Python code instead.
+    This function will only be called once so performance isn't really a concern.
+    Return value: New reference. */
+inline PyTypeObject *make_static_property_type() {
+    auto d = dict();
+    PyObject *result = PyRun_String(R"(\
+        class pybind11_static_property(property):
+            def __get__(self, obj, cls):
+                return property.__get__(self, cls, cls)
+
+            def __set__(self, obj, value):
+                cls = obj if isinstance(obj, type) else type(obj)
+                property.__set__(self, cls, value)
+        )", Py_file_input, d.ptr(), d.ptr()
+    );
+    if (result == nullptr)
+        throw error_already_set();
+    Py_DECREF(result);
+    return (PyTypeObject *) d["pybind11_static_property"].cast<object>().release().ptr();
+}
+
+#endif // PYPY
+
+/** Types with static properties need to handle `Type.static_prop = x` in a specific way.
+    By default, Python replaces the `static_property` itself, but for wrapped C++ types
+    we need to call `static_property.__set__()` in order to propagate the new value to
+    the underlying C++ data structure. */
+extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyObject* value) {
+    // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw
+    // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`).
+    PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name);
+
+    // Call `static_property.__set__()` instead of replacing the `static_property`.
+    if (descr && PyObject_IsInstance(descr, (PyObject *) get_internals().static_property_type)) {
+#if !defined(PYPY_VERSION)
+        return Py_TYPE(descr)->tp_descr_set(descr, obj, value);
+#else
+        if (PyObject *result = PyObject_CallMethod(descr, "__set__", "OO", obj, value)) {
+            Py_DECREF(result);
+            return 0;
+        } else {
+            return -1;
+        }
+#endif
+    } else {
+        return PyType_Type.tp_setattro(obj, name, value);
+    }
+}
+
+/** This metaclass is assigned by default to all pybind11 types and is required in order
+    for static properties to function correctly. Users may override this using `py::metaclass`.
+    Return value: New reference. */
+inline PyTypeObject* make_default_metaclass() {
+    constexpr auto *name = "pybind11_type";
+    auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name));
+
+    /* Danger zone: from now (and until PyType_Ready), make sure to
+       issue no Python C API calls which could potentially invoke the
+       garbage collector (the GC will call type_traverse(), which will in
+       turn find the newly constructed type in an invalid state) */
+    auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
+    if (!heap_type)
+        pybind11_fail("make_default_metaclass(): error allocating metaclass!");
+
+    heap_type->ht_name = name_obj.inc_ref().ptr();
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
+    heap_type->ht_qualname = name_obj.inc_ref().ptr();
+#endif
+
+    auto type = &heap_type->ht_type;
+    type->tp_name = name;
+    type->tp_base = &PyType_Type;
+    type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
+    type->tp_setattro = pybind11_meta_setattro;
+
+    if (PyType_Ready(type) < 0)
+        pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!");
+
+    return type;
+}
+
+NAMESPACE_END(detail)
+NAMESPACE_END(pybind11)

include/pybind11/common.h

@@ -352,7 +352,7 @@ struct overload_hash {
     }
 };
 
-/// Internal data struture used to track registered instances and types
+/// Internal data structure used to track registered instances and types
 struct internals {
     std::unordered_map<std::type_index, void*> registered_types_cpp;   // std::type_index -> type_info
     std::unordered_map<const void *, void*> registered_types_py;       // PyTypeObject* -> type_info
@@ -361,6 +361,8 @@ struct internals {
     std::unordered_map<std::type_index, std::vector<bool (*)(PyObject *, void *&)>> direct_conversions;
     std::forward_list<void (*) (std::exception_ptr)> registered_exception_translators;
     std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across extensions
+    PyTypeObject *static_property_type;
+    PyTypeObject *default_metaclass;
 #if defined(WITH_THREAD)
     decltype(PyThread_create_key()) tstate = 0; // Usually an int but a long on Cygwin64 with Python 3.x
     PyInterpreterState *istate = nullptr;

include/pybind11/pybind11.h

@@ -35,6 +35,7 @@
 
 #include "attr.h"
 #include "options.h"
+#include "class_support.h"
 
 NAMESPACE_BEGIN(pybind11)
 
@@ -818,15 +819,12 @@ class generic_type : public object {
         object scope_qualname;
         if (rec->scope && hasattr(rec->scope, "__qualname__"))
             scope_qualname = rec->scope.attr("__qualname__");
-        object ht_qualname, ht_qualname_meta;
+        object ht_qualname;
         if (scope_qualname)
             ht_qualname = reinterpret_steal<object>(PyUnicode_FromFormat(
                 "%U.%U", scope_qualname.ptr(), name.ptr()));
         else
             ht_qualname = name;
-        if (rec->metaclass)
-            ht_qualname_meta = reinterpret_steal<object>(
-                PyUnicode_FromFormat("%U__Meta", ht_qualname.ptr()));
 #endif
 
 #if !defined(PYPY_VERSION)
@@ -836,36 +834,6 @@ class generic_type : public object {
         std::string full_name = std::string(rec->name);
 #endif
 
-        /* Create a custom metaclass if requested (used for static properties) */
-        object metaclass;
-        if (rec->metaclass) {
-            std::string meta_name_ = full_name + "__Meta";
-            object meta_name = reinterpret_steal<object>(PYBIND11_FROM_STRING(meta_name_.c_str()));
-            metaclass = reinterpret_steal<object>(PyType_Type.tp_alloc(&PyType_Type, 0));
-            if (!metaclass || !name)
-                pybind11_fail("generic_type::generic_type(): unable to create metaclass!");
-
-            /* Danger zone: from now (and until PyType_Ready), make sure to
-               issue no Python C API calls which could potentially invoke the
-               garbage collector (the GC will call type_traverse(), which will in
-               turn find the newly constructed type in an invalid state) */
-
-            auto type = (PyHeapTypeObject*) metaclass.ptr();
-            type->ht_name = meta_name.release().ptr();
-
-#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
-            /* Qualified names for Python >= 3.3 */
-            type->ht_qualname = ht_qualname_meta.release().ptr();
-#endif
-            type->ht_type.tp_name = strdup(meta_name_.c_str());
-            type->ht_type.tp_base = &PyType_Type;
-            type->ht_type.tp_flags |= (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE) &
-                                      ~Py_TPFLAGS_HAVE_GC;
-
-            if (PyType_Ready(&type->ht_type) < 0)
-                pybind11_fail("generic_type::generic_type(): failure in PyType_Ready() for metaclass!");
-        }
-
         size_t num_bases = rec->bases.size();
         auto bases = tuple(rec->bases);
 
@@ -915,8 +883,9 @@ class generic_type : public object {
         type->ht_qualname = ht_qualname.release().ptr();
 #endif
 
-        /* Metaclass */
-        PYBIND11_OB_TYPE(type->ht_type) = (PyTypeObject *) metaclass.release().ptr();
+        /* Custom metaclass if requested (used for static properties) */
+        if (rec->metaclass)
+            PYBIND11_OB_TYPE(type->ht_type) = internals.default_metaclass;
 
         /* Supported protocols */
         type->ht_type.tp_as_number = &type->as_number;
@@ -1105,15 +1074,10 @@ class generic_type : public object {
     void def_property_static_impl(const char *name,
                                   handle fget, handle fset,
                                   detail::function_record *rec_fget) {
-        pybind11::str doc_obj = pybind11::str(
-            (rec_fget->doc && pybind11::options::show_user_defined_docstrings())
-                ? rec_fget->doc : "");
-        const auto property = reinterpret_steal<object>(
-            PyObject_CallFunctionObjArgs((PyObject *) &PyProperty_Type, fget.ptr() ? fget.ptr() : Py_None,
-                                         fset.ptr() ? fset.ptr() : Py_None, Py_None, doc_obj.ptr(), nullptr));
-        if (rec_fget->is_method && rec_fget->scope) {
-            attr(name) = property;
-        } else {
+        const auto is_static = !(rec_fget->is_method && rec_fget->scope);
+        const auto has_doc = rec_fget->doc && pybind11::options::show_user_defined_docstrings();
+
+        if (is_static) {
             auto mclass = handle((PyObject *) PYBIND11_OB_TYPE(*((PyTypeObject *) m_ptr)));
 
             if ((PyTypeObject *) mclass.ptr() == &PyType_Type)
@@ -1123,8 +1087,14 @@ class generic_type : public object {
                     "' requires the type to have a custom metaclass. Please "
                     "ensure that one is created by supplying the pybind11::metaclass() "
                     "annotation to the associated class_<>(..) invocation.");
-            mclass.attr(name) = property;
         }
+
+        auto property = handle((PyObject *) (is_static ? get_internals().static_property_type
+                                                       : &PyProperty_Type));
+        attr(name) = property(fget.ptr() ? fget : none(),
+                              fset.ptr() ? fset : none(),
+                              /*deleter*/none(),
+                              pybind11::str(has_doc ? rec_fget->doc : ""));
     }
 };
 

setup.py

@@ -15,6 +15,7 @@
         'include/pybind11/attr.h',
         'include/pybind11/cast.h',
         'include/pybind11/chrono.h',
+        'include/pybind11/class_support.h',
         'include/pybind11/common.h',
         'include/pybind11/complex.h',
         'include/pybind11/descr.h',

tests/test_methods_and_attributes.cpp

@@ -214,7 +214,10 @@ test_initializer methods_and_attributes([](py::module &m) {
                                       [](py::object) { return TestProperties::static_get(); })
         .def_property_static("def_property_static",
                              [](py::object) { return TestProperties::static_get(); },
-                             [](py::object, int v) { return TestProperties::static_set(v); });
+                             [](py::object, int v) { TestProperties::static_set(v); })
+        .def_property_static("static_cls",
+                             [](py::object cls) { return cls; },
+                             [](py::object cls, py::function f) { f(cls); });
 
     py::class_<SimpleValue>(m, "SimpleValue")
         .def_readwrite("value", &SimpleValue::value);

tests/test_methods_and_attributes.py

@@ -84,19 +84,47 @@ def test_static_properties():
     from pybind11_tests import TestProperties as Type
 
     assert Type.def_readonly_static == 1
-    with pytest.raises(AttributeError):
+    with pytest.raises(AttributeError) as excinfo:
         Type.def_readonly_static = 2
+    assert "can't set attribute" in str(excinfo)
 
     Type.def_readwrite_static = 2
     assert Type.def_readwrite_static == 2
 
     assert Type.def_property_readonly_static == 2
-    with pytest.raises(AttributeError):
+    with pytest.raises(AttributeError) as excinfo:
         Type.def_property_readonly_static = 3
+    assert "can't set attribute" in str(excinfo)
 
     Type.def_property_static = 3
     assert Type.def_property_static == 3
 
+    # Static property read and write via instance
+    instance = Type()
+
+    Type.def_readwrite_static = 0
+    assert Type.def_readwrite_static == 0
+    assert instance.def_readwrite_static == 0
+
+    instance.def_readwrite_static = 2
+    assert Type.def_readwrite_static == 2
+    assert instance.def_readwrite_static == 2
+
+
+def test_static_cls():
+    """Static property getter and setters expect the type object as the their only argument"""
+    from pybind11_tests import TestProperties as Type
+
+    instance = Type()
+    assert Type.static_cls is Type
+    assert instance.static_cls is Type
+
+    def check_self(self):
+        assert self is Type
+
+    Type.static_cls = check_self
+    instance.static_cls = check_self
+
 
 @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"])
 def test_property_return_value_policies(access):

tests/test_python_types.py

@@ -6,7 +6,7 @@
 
 def test_repr():
     # In Python 3.3+, repr() accesses __qualname__
-    assert "ExamplePythonTypes__Meta" in repr(type(ExamplePythonTypes))
+    assert "pybind11_type" in repr(type(ExamplePythonTypes))
     assert "ExamplePythonTypes" in repr(ExamplePythonTypes)