Add support for non-converting arguments

This adds support for controlling the `convert` flag of arguments
through the py::arg annotation.  This then allows arguments to be
flagged as non-converting, which the type_caster is able to use to
request different behaviour.

Currently, AFAICS `convert` is only used for type converters of regular
pybind11-registered types; all of the other core type_casters ignore it.
We can, however, repurpose it to control internal conversion of
converters like Eigen and `array`: most usefully to give callers a way
to disable the conversion that would otherwise occur when a
`Eigen::Ref<const Eigen::Matrix>` argument is passed a numpy array that
requires conversion (either because it has an incompatible stride or the
wrong dtype).

Specifying a noconvert looks like one of these:

    m.def("f1", &f, "a"_a.noconvert() = "default"); // Named, default, noconvert
    m.def("f2", &f, "a"_a.noconvert()); // Named, no default, no converting
    m.def("f3", &f, py::arg().noconvert()); // Unnamed, no default, no converting

(The last part--being able to declare a py::arg without a name--is new:
previous py::arg() only accepted named keyword arguments).

Such an non-convert argument is then passed `convert = false` by the
type caster when loading the argument.  Whether this has an effect is up
to the type caster itself, but as mentioned above, this would be
extremely helpful for the Eigen support to give a nicer way to specify
a "no-copy" mode than the custom wrapper in the current PR, and
moreover isn't an Eigen-specific hack.

Author: jagerman

include/pybind11/attr.h

@@ -69,17 +69,17 @@ struct undefined_t;
 template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_;
 template <typename... Args> struct init;
 template <typename... Args> struct init_alias;
-struct function_call;
 inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret);
 
 /// Internal data structure which holds metadata about a keyword argument
 struct argument_record {
     const char *name;  ///< Argument name
     const char *descr; ///< Human-readable version of the argument value
     handle value;      ///< Associated Python object
+    bool convert : 1;  ///< True if the argument is allowed to convert when loading
 
-    argument_record(const char *name, const char *descr, handle value)
-        : name(name), descr(descr), value(value) { }
+    argument_record(const char *name, const char *descr, handle value, bool convert)
+        : name(name), descr(descr), value(value), convert(convert) { }
 };
 
 /// Internal data structure which holds metadata about a bound function (signature, overloads, etc.)
@@ -131,7 +131,7 @@ struct function_record {
     bool is_method : 1;
 
     /// Number of arguments (including py::args and/or py::kwargs, if present)
-    uint16_t nargs;
+    std::uint16_t nargs;
 
     /// Python method object
     PyMethodDef *def = nullptr;
@@ -222,21 +222,11 @@ struct type_record {
     }
 };
 
-/// Internal data associated with a single function call
-struct function_call {
-    function_call(function_record &f, handle p) : func(f), parent(p) {
-        args.reserve(f.nargs);
-    }
-
-    /// The function data:
-    const function_record &func;
-
-    /// Arguments passed to the function:
-    std::vector<handle> args;
-
-    /// The parent, if any
-    handle parent;
-};
+inline function_call::function_call(function_record &f, handle p) :
+        func(f), parent(p) {
+    args.reserve(f.nargs);
+    args_convert.reserve(f.nargs);
+}
 
 /**
  * Partial template specializations to process custom attributes provided to
@@ -300,16 +290,16 @@ template <> struct process_attribute<is_operator> : process_attribute_default<is
 template <> struct process_attribute<arg> : process_attribute_default<arg> {
     static void init(const arg &a, function_record *r) {
         if (r->is_method && r->args.empty())
-            r->args.emplace_back("self", nullptr, handle());
-        r->args.emplace_back(a.name, nullptr, handle());
+            r->args.emplace_back("self", nullptr, handle(), true /*convert*/);
+        r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert);
     }
 };
 
 /// Process a keyword argument attribute (*with* a default value)
 template <> struct process_attribute<arg_v> : process_attribute_default<arg_v> {
     static void init(const arg_v &a, function_record *r) {
         if (r->is_method && r->args.empty())
-            r->args.emplace_back("self", nullptr, handle());
+            r->args.emplace_back("self", nullptr /*descr*/, handle() /*parent*/, true /*convert*/);
 
         if (!a.value) {
 #if !defined(NDEBUG)
@@ -330,7 +320,7 @@ template <> struct process_attribute<arg_v> : process_attribute_default<arg_v> {
                           "Compile in debug mode for more information.");
 #endif
         }
-        r->args.emplace_back(a.name, a.descr, a.value.inc_ref());
+        r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert);
     }
 };
 

include/pybind11/cast.h

@@ -1198,22 +1198,26 @@ template <return_value_policy policy = return_value_policy::automatic_reference,
 }
 
 /// \ingroup annotations
-/// Annotation for keyword arguments
+/// Annotation for arguments
 struct arg {
-    /// Set the name of the argument
-    constexpr explicit arg(const char *name) : name(name) { }
+    /// Constructs an argument with the name of the argument; if null or omitted, this is a positional argument.
+    constexpr explicit arg(const char *name = nullptr) : name(name), flag_noconvert(false) { }
     /// Assign a value to this argument
     template <typename T> arg_v operator=(T &&value) const;
+    /// Indicate that the type should not be converted in the type caster
+    arg &noconvert(bool flag = true) { flag_noconvert = flag; return *this; }
 
-    const char *name;
+    const char *name; ///< If non-null, this is a named kwargs argument
+    bool flag_noconvert : 1; ///< If set, do not allow conversion (requires a supporting type caster!)
 };
 
 /// \ingroup annotations
-/// Annotation for keyword arguments with values
+/// Annotation for arguments with values
 struct arg_v : arg {
+private:
     template <typename T>
-    arg_v(const char *name, T &&x, const char *descr = nullptr)
-        : arg(name),
+    arg_v(arg &&base, T &&x, const char *descr = nullptr)
+        : arg(base),
           value(reinterpret_steal<object>(
               detail::make_caster<T>::cast(x, return_value_policy::automatic, {})
           )),
@@ -1223,15 +1227,32 @@ struct arg_v : arg {
 #endif
     { }
 
+public:
+    /// Direct construction with name, default, and description
+    template <typename T>
+    arg_v(const char *name, T &&x, const char *descr = nullptr)
+        : arg_v(arg(name), std::forward<T>(x), descr) { }
+
+    /// Called internally when invoking `py::arg("a") = value`
+    template <typename T>
+    arg_v(const arg &base, T &&x, const char *descr = nullptr)
+        : arg_v(arg(base), std::forward<T>(x), descr) { }
+
+    /// Same as `arg::noconvert()`, but returns *this as arg_v&, not arg&
+    arg_v &noconvert(bool flag = true) { arg::noconvert(flag); return *this; }
+
+    /// The default value
     object value;
+    /// The (optional) description of the default value
     const char *descr;
 #if !defined(NDEBUG)
+    /// The C++ type name of the default value (only available when compiled in debug mode)
     std::string type;
 #endif
 };
 
 template <typename T>
-arg_v arg::operator=(T &&value) const { return {name, std::forward<T>(value)}; }
+arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward<T>(value)}; }
 
 /// Alias for backward compatibility -- to be removed in version 2.0
 template <typename /*unused*/> using arg_t = arg_v;
@@ -1248,11 +1269,28 @@ NAMESPACE_BEGIN(detail)
 // forward declaration
 struct function_record;
 
+/// Internal data associated with a single function call
+struct function_call {
+    function_call(function_record &f, handle p); // Implementation in attr.h
+
+    /// The function data:
+    const function_record &func;
+
+    /// Arguments passed to the function:
+    std::vector<handle> args;
+
+    /// The `convert` value the arguments should be loaded with
+    std::vector<bool> args_convert;
+
+    /// The parent, if any
+    handle parent;
+};
+
+
 /// Helper class which loads arguments for C++ functions called from Python
 template <typename... Args>
 class argument_loader {
     using indices = make_index_sequence<sizeof...(Args)>;
-    using function_arguments = const std::vector<handle> &;
 
     template <typename Arg> using argument_is_args   = std::is_same<intrinsic_t<Arg>, args>;
     template <typename Arg> using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>;
@@ -1270,8 +1308,8 @@ class argument_loader {
 
     static PYBIND11_DESCR arg_names() { return detail::concat(make_caster<Args>::name()...); }
 
-    bool load_args(function_arguments args) {
-        return load_impl_sequence(args, indices{});
+    bool load_args(function_call &call) {
+        return load_impl_sequence(call, indices{});
     }
 
     template <typename Return, typename Func>
@@ -1287,11 +1325,11 @@ class argument_loader {
 
 private:
 
-    static bool load_impl_sequence(function_arguments, index_sequence<>) { return true; }
+    static bool load_impl_sequence(function_call &, index_sequence<>) { return true; }
 
     template <size_t... Is>
-    bool load_impl_sequence(function_arguments args, index_sequence<Is...>) {
-        for (bool r : {std::get<Is>(value).load(args[Is], true)...})
+    bool load_impl_sequence(function_call &call, index_sequence<Is...>) {
+        for (bool r : {std::get<Is>(value).load(call.args[Is], call.args_convert[Is])...})
             if (!r)
                 return false;
         return true;
@@ -1380,6 +1418,13 @@ class unpacking_collector {
     }
 
     void process(list &/*args_list*/, arg_v a) {
+        if (!a.name)
+#if defined(NDEBUG)
+            nameless_argument_error();
+#else
+            nameless_argument_error(a.type);
+#endif
+
         if (m_kwargs.contains(a.name)) {
 #if defined(NDEBUG)
             multiple_values_error();
@@ -1412,6 +1457,15 @@ class unpacking_collector {
         }
     }
 
+    [[noreturn]] static void nameless_argument_error() {
+        throw type_error("Got kwargs without a name; only named arguments "
+                         "may be passed via py::arg() to a python function call. "
+                         "(compile in debug mode for details)");
+    }
+    [[noreturn]] static void nameless_argument_error(std::string type) {
+        throw type_error("Got kwargs without a name of type '" + type + "'; only named "
+                         "arguments may be passed via py::arg() to a python function call. ");
+    }
     [[noreturn]] static void multiple_values_error() {
         throw type_error("Got multiple values for keyword argument "
                          "(compile in debug mode for details)");

include/pybind11/pybind11.h

@@ -122,7 +122,7 @@ class cpp_function : public function {
             cast_in args_converter;
 
             /* Try to cast the function arguments into the C++ domain */
-            if (!args_converter.load_args(call.args))
+            if (!args_converter.load_args(call))
                 return PYBIND11_TRY_NEXT_OVERLOAD;
 
             /* Invoke call policy pre-call hook */
@@ -198,7 +198,7 @@ class cpp_function : public function {
             if (c == '{') {
                 // Write arg name for everything except *args, **kwargs and return type.
                 if (type_depth == 0 && text[char_index] != '*' && arg_index < args) {
-                    if (!rec->args.empty()) {
+                    if (!rec->args.empty() && rec->args[arg_index].name) {
                         signature += rec->args[arg_index].name;
                     } else if (arg_index == 0 && rec->is_method) {
                         signature += "self";
@@ -257,7 +257,7 @@ class cpp_function : public function {
         rec->signature = strdup(signature.c_str());
         rec->args.shrink_to_fit();
         rec->is_constructor = !strcmp(rec->name, "__init__") || !strcmp(rec->name, "__setstate__");
-        rec->nargs = (uint16_t) args;
+        rec->nargs = (std::uint16_t) args;
 
 #if PY_MAJOR_VERSION < 3
         if (rec->sibling && PyMethod_Check(rec->sibling.ptr()))
@@ -392,8 +392,10 @@ class cpp_function : public function {
 
         handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr,
                result = PYBIND11_TRY_NEXT_OVERLOAD;
+
         try {
             for (; it != nullptr; it = it->next) {
+
                 /* For each overload:
                    1. Copy all positional arguments we were given, also checking to make sure that
                       named positional arguments weren't *also* specified via kwarg.
@@ -435,14 +437,15 @@ class cpp_function : public function {
                     // raise a TypeError like Python does.  (We could also continue with the next
                     // overload, but this seems highly likely to be a caller mistake rather than a
                     // legitimate overload).
-                    if (kwargs_in && args_copied < it->args.size()) {
-                        handle value = PyDict_GetItemString(kwargs_in, it->args[args_copied].name);
+                    if (kwargs_in && args_copied < func.args.size() && func.args[args_copied].name) {
+                        handle value = PyDict_GetItemString(kwargs_in, func.args[args_copied].name);
                         if (value)
-                            throw type_error(std::string(it->name) + "(): got multiple values for argument '" +
-                                    std::string(it->args[args_copied].name) + "'");
+                            throw type_error(std::string(func.name) + "(): got multiple values for argument '" +
+                                    std::string(func.args[args_copied].name) + "'");
                     }
 
                     call.args.push_back(PyTuple_GET_ITEM(args_in, args_copied));
+                    call.args_convert.push_back(args_copied < func.args.size() ? func.args[args_copied].convert : true);
                 }
 
                 // We'll need to copy this if we steal some kwargs for defaults
@@ -453,10 +456,10 @@ class cpp_function : public function {
                     bool copied_kwargs = false;
 
                     for (; args_copied < pos_args; ++args_copied) {
-                        const auto &arg = it->args[args_copied];
+                        const auto &arg = func.args[args_copied];
 
                         handle value;
-                        if (kwargs_in)
+                        if (kwargs_in && arg.name)
                             value = PyDict_GetItemString(kwargs.ptr(), arg.name);
 
                         if (value) {
@@ -470,8 +473,10 @@ class cpp_function : public function {
                             value = arg.value;
                         }
 
-                        if (value)
+                        if (value) {
                             call.args.push_back(value);
+                            call.args_convert.push_back(arg.convert);
+                        }
                         else
                             break;
                     }
@@ -481,12 +486,12 @@ class cpp_function : public function {
                 }
 
                 // 3. Check everything was consumed (unless we have a kwargs arg)
-                if (kwargs && kwargs.size() > 0 && !it->has_kwargs)
+                if (kwargs && kwargs.size() > 0 && !func.has_kwargs)
                     continue; // Unconsumed kwargs, but no py::kwargs argument to accept them
 
                 // 4a. If we have a py::args argument, create a new tuple with leftovers
                 tuple extra_args;
-                if (it->has_args) {
+                if (func.has_args) {
                     if (args_to_copy == 0) {
                         // We didn't copy out any position arguments from the args_in tuple, so we
                         // can reuse it directly without copying:
@@ -502,31 +507,34 @@ class cpp_function : public function {
                         }
                     }
                     call.args.push_back(extra_args);
+                    call.args_convert.push_back(false);
                 }
 
                 // 4b. If we have a py::kwargs, pass on any remaining kwargs
-                if (it->has_kwargs) {
+                if (func.has_kwargs) {
                     if (!kwargs.ptr())
                         kwargs = dict(); // If we didn't get one, send an empty one
                     call.args.push_back(kwargs);
+                    call.args_convert.push_back(false);
                 }
 
                 // 5. Put everything in a vector.  Not technically step 5, we've been building it
                 // in `call.args` all along.
                 #if !defined(NDEBUG)
-                if (call.args.size() != call.func.nargs)
+                if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs)
                     pybind11_fail("Internal error: function call dispatcher inserted wrong number of arguments!");
                 #endif
 
                 // 6. Call the function.
                 try {
-                    result = it->impl(call);
+                    result = func.impl(call);
                 } catch (reference_cast_error &) {
                     result = PYBIND11_TRY_NEXT_OVERLOAD;
                 }
 
                 if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD)
                     break;
+
             }
         } catch (error_already_set &e) {
             e.restore();