Add support for u16/u32strings and literals

This adds support for wchar{16,32}_t character literals and the
associated std::u{16,32}string types.  It also folds the
character/string conversion into a single type_caster template, since
the type casters for string and wstring were mostly the same anyway.

Author: jagerman

docs/advanced/cast/overview.rst

@@ -94,14 +94,26 @@ as arguments and return values, refer to the section on binding :ref:`classes`.
 +------------------------------------+---------------------------+-------------------------------+
 | ``char``                           | Character literal         | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``char16_t``                       | UTF-16 character literal  | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``char32_t``                       | UTF-32 character literal  | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``wchar_t``                        | Wide character literal    | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``const char *``                   | UTF-8 string literal      | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``const char16_t *``               | UTF-16 string literal     | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``const char32_t *``               | UTF-32 string literal     | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``const wchar_t *``                | Wide string literal       | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``std::string``                    | STL dynamic UTF-8 string  | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
+| ``std::u16string``                 | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
+| ``std::u32string``                 | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h`   |
++------------------------------------+---------------------------+-------------------------------+
 | ``std::wstring``                   | STL dynamic wide string   | :file:`pybind11/pybind11.h`   |
 +------------------------------------+---------------------------+-------------------------------+
 | ``std::pair<T1, T2>``              | Pair of two custom types  | :file:`pybind11/pybind11.h`   |

include/pybind11/cast.h

@@ -471,8 +471,15 @@ template <typename type> class type_caster<std::reference_wrapper<type>> : publi
         template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>
 
 
+template <typename CharT> using is_std_char_type = any_of<
+    std::is_same<CharT, char>, /* std::string */
+    std::is_same<CharT, char16_t>, /* std::u16string */
+    std::is_same<CharT, char32_t>, /* std::u32string */
+    std::is_same<CharT, wchar_t> /* std::wstring */
+>;
+
 template <typename T>
-struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value>> {
+struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value && !is_std_char_type<T>::value>> {
     typedef typename std::conditional<sizeof(T) <= sizeof(long), long, long long>::type _py_type_0;
     typedef typename std::conditional<std::is_signed<T>::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>::type _py_type_1;
     typedef typename std::conditional<std::is_floating_point<T>::value, double, _py_type_1>::type py_type;
@@ -613,41 +620,24 @@ template <> class type_caster<bool> {
     PYBIND11_TYPE_CASTER(bool, _("bool"));
 };
 
-template <> class type_caster<std::string> {
-public:
-    bool load(handle src, bool) {
-        object temp;
-        handle load_src = src;
-        if (!src) {
-            return false;
-        } else if (PyUnicode_Check(load_src.ptr())) {
-            temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(load_src.ptr()));
-            if (!temp) { PyErr_Clear(); return false; }  // UnicodeEncodeError
-            load_src = temp;
-        }
-        char *buffer;
-        ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(load_src.ptr(), &buffer, &length);
-        if (err == -1) { PyErr_Clear(); return false; }  // TypeError
-        value = std::string(buffer, (size_t) length);
-        success = true;
-        return true;
-    }
+// Helper class for UTF-{8,16,32} strings:
+template <typename CharT, class Traits, class Allocator>
+struct type_caster<std::basic_string<CharT, Traits, Allocator>, enable_if_t<is_std_char_type<CharT>::value>> {
+    static constexpr unsigned int UTF_N =
+        std::is_same<CharT, char>::value     ? 8 :
+        std::is_same<CharT, char16_t>::value ? 16 :
+        std::is_same<CharT, char32_t>::value ? 32 :
+        (sizeof(CharT) == 2 ? 16 : 32); /* std::wstring is UTF-16 on Windows, UTF-32 everywhere else */
 
-    static handle cast(const std::string &src, return_value_policy /* policy */, handle /* parent */) {
-        handle s = PyUnicode_FromStringAndSize(src.c_str(), (ssize_t) src.length());
-        if (!s)
-            throw error_already_set();
-        return s;
-    }
+    static constexpr const char *encoding = UTF_N == 8 ? "utf8" : UTF_N == 16 ? "utf16" : "utf32";
 
-    PYBIND11_TYPE_CASTER(std::string, _(PYBIND11_STRING_NAME));
-protected:
-    bool success = false;
-};
+    // C++ only requires char/char16_t/char32_t to be at least 8/16/32 bits, but Python's encoding
+    // assumes exactly 1/2/4 bytes:
+    static_assert(sizeof(CharT) == UTF_N / 8,
+            "Internal error: string type_caster requires 1/2/4-sized character types");
+
+    using StringType = std::basic_string<CharT, Traits, Allocator>;
 
-template <> class type_caster<std::wstring> {
-public:
     bool load(handle src, bool) {
         object temp;
         handle load_src = src;
@@ -658,78 +648,60 @@ template <> class type_caster<std::wstring> {
             if (!temp) { PyErr_Clear(); return false; }
             load_src = temp;
         }
-        wchar_t *buffer = nullptr;
-        ssize_t length = -1;
-#if PY_MAJOR_VERSION >= 3
-        buffer = PyUnicode_AsWideCharString(load_src.ptr(), &length);
-#else
-        temp = reinterpret_steal<object>(PyUnicode_AsEncodedString(
-            load_src.ptr(), sizeof(wchar_t) == sizeof(short)
-            ? "utf16" : "utf32", nullptr));
-
-        if (temp) {
-            int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), (char **) &buffer, &length);
-            if (err == -1) { buffer = nullptr; }  // TypeError
-            length = length / (ssize_t) sizeof(wchar_t) - 1; ++buffer; // Skip BOM
-        }
-#endif
-        if (!buffer) { PyErr_Clear(); return false; }
-        value = std::wstring(buffer, (size_t) length);
+
+        object utfNbytes = reinterpret_steal<object>(PyUnicode_AsEncodedString(
+            load_src.ptr(), encoding, nullptr));
+        if (!utfNbytes) { PyErr_Clear(); return false; }
+
+        const CharT *buffer = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr()));
+        size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT);
+        if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32
+        value = StringType(buffer, length);
         success = true;
         return true;
     }
 
-    static handle cast(const std::wstring &src, return_value_policy /* policy */, handle /* parent */) {
-        return PyUnicode_FromWideChar(src.c_str(), (ssize_t) src.length());
+    static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) {
+        const char *buffer = reinterpret_cast<const char *>(src.c_str());
+        ssize_t nbytes = ssize_t(src.size() * sizeof(CharT));
+        handle s = PyUnicode_Decode(buffer, nbytes, encoding, nullptr);
+        if (!s) throw error_already_set();
+        return s;
     }
 
-    PYBIND11_TYPE_CASTER(std::wstring, _(PYBIND11_STRING_NAME));
+    PYBIND11_TYPE_CASTER(StringType, _(PYBIND11_STRING_NAME));
 protected:
     bool success = false;
 };
 
-template <> class type_caster<char> : public type_caster<std::string> {
+template <typename CharT> struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>>
+: type_caster<std::basic_string<CharT>> {
+    using StringType = std::basic_string<CharT>;
+    using StringCaster = type_caster<StringType>;
+    using StringCaster::success;
+    using StringCaster::value;
 public:
     bool load(handle src, bool convert) {
         if (src.is_none()) return true;
-        return type_caster<std::string>::load(src, convert);
+        return StringCaster::load(src, convert);
     }
 
-    static handle cast(const char *src, return_value_policy /* policy */, handle /* parent */) {
+    static handle cast(const CharT *src, return_value_policy policy, handle parent) {
         if (src == nullptr) return none().inc_ref();
-        return PyUnicode_FromString(src);
+        return StringCaster::cast(StringType(src), policy, parent);
     }
 
-    static handle cast(char src, return_value_policy /* policy */, handle /* parent */) {
-        char str[2] = { src, '\0' };
-        return PyUnicode_DecodeLatin1(str, 1, nullptr);
-    }
-
-    operator char*() { return success ? (char *) value.c_str() : nullptr; }
-    operator char&() { return value[0]; }
-
-    static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); }
-};
-
-template <> class type_caster<wchar_t> : public type_caster<std::wstring> {
-public:
-    bool load(handle src, bool convert) {
-        if (src.is_none()) return true;
-        return type_caster<std::wstring>::load(src, convert);
-    }
-
-    static handle cast(const wchar_t *src, return_value_policy /* policy */, handle /* parent */) {
-        if (src == nullptr) return none().inc_ref();
-        return PyUnicode_FromWideChar(src, (ssize_t) wcslen(src));
-    }
-
-    static handle cast(wchar_t src, return_value_policy /* policy */, handle /* parent */) {
-        wchar_t wstr[2] = { src, L'\0' };
-        return PyUnicode_FromWideChar(wstr, 1);
+    static handle cast(CharT src, return_value_policy policy, handle parent) {
+        if (std::is_same<char, CharT>::value) {
+            handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr);
+            if (!s) throw error_already_set();
+            return s;
+        }
+        return StringCaster::cast(StringType(1, src), policy, parent);
     }
 
-    operator wchar_t*() { return success ? (wchar_t *) value.c_str() : nullptr; }
-    operator wchar_t&() { return value[0]; }
+    operator CharT*() { return success ? (CharT *) value.c_str() : nullptr; }
+    operator CharT&() { return value[0]; }
 
     static PYBIND11_DESCR name() { return type_descr(_(PYBIND11_STRING_NAME)); }
 };

include/pybind11/common.h

@@ -111,6 +111,7 @@
 #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize
 #define PYBIND11_BYTES_AS_STRING_AND_SIZE PyBytes_AsStringAndSize
 #define PYBIND11_BYTES_AS_STRING PyBytes_AsString
+#define PYBIND11_BYTES_SIZE PyBytes_Size
 #define PYBIND11_LONG_CHECK(o) PyLong_Check(o)
 #define PYBIND11_LONG_AS_LONGLONG(o) PyLong_AsLongLong(o)
 #define PYBIND11_LONG_AS_UNSIGNED_LONGLONG(o) PyLong_AsUnsignedLongLong(o)
@@ -129,6 +130,7 @@
 #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyString_FromStringAndSize
 #define PYBIND11_BYTES_AS_STRING_AND_SIZE PyString_AsStringAndSize
 #define PYBIND11_BYTES_AS_STRING PyString_AsString
+#define PYBIND11_BYTES_SIZE PyString_Size
 #define PYBIND11_LONG_CHECK(o) (PyInt_Check(o) || PyLong_Check(o))
 #define PYBIND11_LONG_AS_LONGLONG(o) (PyInt_Check(o) ? (long long) PyLong_AsLong(o) : PyLong_AsLongLong(o))
 #define PYBIND11_LONG_AS_UNSIGNED_LONGLONG(o) (PyInt_Check(o) ? (unsigned long long) PyLong_AsUnsignedLong(o) : PyLong_AsUnsignedLongLong(o))

tests/test_python_types.cpp

@@ -17,6 +17,11 @@
 #  include <fcntl.h>
 #endif
 
+#if defined(_MSC_VER)
+#  pragma warning(push)
+#  pragma warning(disable: 4127) // warning C4127: Conditional expression is constant
+#endif
+
 class ExamplePythonTypes {
 public:
     static ExamplePythonTypes *new_instance() {
@@ -427,6 +432,36 @@ test_initializer python_types([](py::module &m) {
         );
     });
 
+    // Some test characters in utf16 and utf32 encodings.  The last one (the 𝐀) contains a null byte
+    // (Note also that all of these need to be in Unicode 5.2, since that the last version Python
+    // 5.2.0 supports; sadly this means 💩 and 🎂 can't be used in Python 2.7.
+    char32_t a32 = 0x61 /*a*/, z32 = 0x7a /*z*/, ib32 = 0x203d /*‽*/, clef32 = 0x1d11e /*𝄞*/,               mathbfA32 = 0x1d400 /*𝐀*/;
+    char16_t b16 = 0x62 /*b*/, z16 = 0x7a,       ib16 = 0x203d,       clef16_1 = 0xd834, clef16_2 = 0xdd1e, mathbfA16_1 = 0xd835, mathbfA16_2 = 0xdc00;
+    std::wstring wstr;
+    wstr.push_back(0x61); // a
+    wstr.push_back(0x2e18); // ⸘
+    if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // 𝐀, utf16
+    else { wstr.push_back((wchar_t) mathbfA32); } // 𝐀, utf32
+    wstr.push_back(0x7a); // z
+
+    m.def("good_utf8_string", []() { return std::string(u8"Say utf8\u203d \U0001d11e \U0001d400"); }); // Say utf8‽ 𝄞 𝐀
+    m.def("good_utf16_string", [=]() { return std::u16string({ b16, ib16, clef16_1, clef16_2, mathbfA16_1, mathbfA16_2, z16 }); }); // b‽𝄞𝐀z
+    m.def("good_utf32_string", [=]() { return std::u32string({ a32, mathbfA32, clef32, ib32, z32 }); }); // a𝐀𝄞‽z
+    m.def("good_wchar_string", [=]() { return wstr; }); // a‽𝐀z
     m.def("bad_utf8_string", []()  { return std::string("abc\xd0" "def"); });
-    m.def("good_utf8_string", []() { return std::string(u8"Say what‽ 🎂"); });
+    m.def("bad_utf16_string", [=]() { return std::u16string({ b16, char16_t(0xd800), z16 }); });
+    // Under Python 2.7, and invalid unicode UTF-32 characters don't appear to trigger UnicodeDecodeError
+    if (PY_MAJOR_VERSION >= 3)
+        m.def("bad_utf32_string", [=]() { return std::u32string({ a32, char32_t(0xd800), z32 }); });
+    if (PY_MAJOR_VERSION >= 3 || sizeof(wchar_t) == 2)
+        m.def("bad_wchar_string", [=]() { return std::wstring({ wchar_t(0x61), wchar_t(0xd800) }); });
+    m.def("u8_Z", []() -> char { return 'Z'; });
+    m.def("u8_eacute", []() -> char { return '\xe9'; });
+    m.def("u16_ibang", [=]() -> char16_t { return ib16; });
+    m.def("u32_mathbfA", [=]() -> char32_t { return mathbfA32; });
+    m.def("wchar_heart", []() -> wchar_t { return 0x2665; });
 });
+
+#if defined(_MSC_VER)
+#  pragma warning(pop)
+#endif

tests/test_python_types.py

@@ -415,8 +415,34 @@ def test_implicit_casting():
 
 def test_unicode_conversion():
     """Tests unicode conversion and error reporting."""
-    from pybind11_tests import (good_utf8_string, bad_utf8_string)
+    import pybind11_tests
+    from pybind11_tests import (good_utf8_string, bad_utf8_string,
+                                good_utf16_string, bad_utf16_string,
+                                good_utf32_string,  # bad_utf32_string,
+                                good_wchar_string,  # bad_wchar_string,
+                                u8_Z, u8_eacute, u16_ibang, u32_mathbfA, wchar_heart)
+
+    assert good_utf8_string() == u"Say utf8‽ 𝄞 𝐀"
+    assert good_utf16_string() == u"b‽𝄞𝐀z"
+    assert good_utf32_string() == u"a𝐀𝄞‽z"
+    assert good_wchar_string() == u"a⸘𝐀z"
 
-    assert good_utf8_string() == "Say what‽ 🎂"
     with pytest.raises(UnicodeDecodeError):
         bad_utf8_string()
+
+    with pytest.raises(UnicodeDecodeError):
+        bad_utf16_string()
+
+    # These are provided only if they actually fail (they don't when 32-bit and under Python 2.7)
+    if hasattr(pybind11_tests, "bad_utf32_string"):
+        with pytest.raises(UnicodeDecodeError):
+            pybind11_tests.bad_utf32_string()
+    if hasattr(pybind11_tests, "bad_wchar_string"):
+        with pytest.raises(UnicodeDecodeError):
+            pybind11_tests.bad_wchar_string()
+
+    assert u8_Z() == 'Z'
+    assert u8_eacute() == u'é'
+    assert u16_ibang() == u'‽'
+    assert u32_mathbfA() == u'𝐀'
+    assert wchar_heart() == u'♥'