diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d6989a1a56..5fb2c5a50c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,7 @@ Other """"" - ADIOS2: require version 2.7.0+ #927 +- pybind11: require version 2.6.2+ #977 0.13.4 diff --git a/CMakeLists.txt b/CMakeLists.txt index 24677bb043..2bc85a97ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -302,9 +302,9 @@ if(openPMD_USE_PYTHON STREQUAL AUTO) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") set(openPMD_HAVE_PYTHON TRUE) - message(STATUS "pybind11: Using INTERNAL version 2.6.1") + message(STATUS "pybind11: Using INTERNAL version 2.6.2") else() - find_package(pybind11 2.6.1 CONFIG) + find_package(pybind11 2.6.2 CONFIG) if(pybind11_FOUND) set(openPMD_HAVE_PYTHON TRUE) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") @@ -320,9 +320,9 @@ elseif(openPMD_USE_PYTHON) if(openPMD_USE_INTERNAL_PYBIND11) add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11") set(openPMD_HAVE_PYTHON TRUE) - message(STATUS "pybind11: Using INTERNAL version 2.6.1") + message(STATUS "pybind11: Using INTERNAL version 2.6.2") else() - find_package(pybind11 2.6.1 REQUIRED CONFIG) + find_package(pybind11 2.6.2 REQUIRED CONFIG) set(openPMD_HAVE_PYTHON TRUE) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() diff --git a/NEWS.rst b/NEWS.rst index 327ebea428..aa56a9d03b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,7 @@ Upgrade Guide ------ ADIOS 2.7.0 is now the minimally supported version for ADIOS2 support. +pybind11 2.6.2 is now the minimally supported version for Python support. 0.13.0 diff --git a/README.md b/README.md index d995a91362..7596775a86 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Required: Shipped internally in `share/openPMD/thirdParty/`: * [MPark.Variant](https://github.com/mpark/variant) 1.4.0+ ([BSL-1.0](https://github.com/mpark/variant/blob/master/LICENSE.md)) * [Catch2](https://github.com/catchorg/Catch2) 2.13.4+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) -* [pybind11](https://github.com/pybind/pybind11) 2.6.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) +* [pybind11](https://github.com/pybind/pybind11) 2.6.2+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) I/O backends: @@ -120,7 +120,7 @@ while those can be built either with or without: Optional language bindings: * Python: * Python 3.6 - 3.9 - * pybind11 2.6.1+ + * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) * pandas 1.0+ (optional, for dataframes) @@ -262,7 +262,7 @@ The following options allow to switch to external installs: |---------------------------------|------------|---------------|---------| | `openPMD_USE_INTERNAL_VARIANT` | **ON**/OFF | MPark.Variant | 1.4.0+ | | `openPMD_USE_INTERNAL_CATCH` | **ON**/OFF | Catch2 | 2.13.4+ | -| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.6.1+ | +| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11 | 2.6.2+ | | `openPMD_USE_INTERNAL_JSON` | **ON**/OFF | NLohmann-JSON | 3.9.1+ | By default, this will build as a shared library (`libopenPMD.[so|dylib|dll]`) and installs also its headers. diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index 0e955ad51e..7d1f335da8 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -69,7 +69,7 @@ CMake Option Values Installs Library Version ================================= =========== ======== ============= ======== ``openPMD_USE_INTERNAL_VARIANT`` **ON**/OFF Yes MPark.Variant 1.4.0+ ``openPMD_USE_INTERNAL_CATCH`` **ON**/OFF No Catch2 2.13.4+ -``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.6.1+ +``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF No pybind11 2.6.2+ ``openPMD_USE_INTERNAL_JSON`` **ON**/OFF No NLohmann-JSON 3.9.1+ ================================= =========== ======== ============= ======== diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index 1779d1dd7a..d1f3d11ad1 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -19,7 +19,7 @@ The following libraries are shipped internally in ``share/openPMD/thirdParty/`` * `MPark.Variant `_ 1.4.0+ (`BSL-1.0 `_) * `Catch2 `_ 2.13.4+ (`BSL-1.0 `__) -* `pybind11 `_ 2.6.1+ (`new BSD `_) +* `pybind11 `_ 2.6.2+ (`new BSD `_) * `NLohmann-JSON `_ 3.9.1+ (`MIT `_) Optional: I/O backends @@ -40,7 +40,7 @@ Optional: language bindings * Python: * Python 3.6 - 3.9 - * pybind11 2.6.1+ + * pybind11 2.6.2+ * numpy 1.15+ * mpi4py 2.1+ (optional, for MPI) * pandas 1.0+ (optional, for dataframes) diff --git a/pyproject.toml b/pyproject.toml index fc338ffe0e..702b476ec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,6 @@ requires = [ "setuptools>=42", "wheel", "cmake>=3.15.0,<4.0.0", - "pybind11>=2.6.1,<3.0.0" + "pybind11>=2.6.2,<3.0.0" ] build-backend = "setuptools.build_meta" diff --git a/share/openPMD/thirdParty/pybind11/CMakeLists.txt b/share/openPMD/thirdParty/pybind11/CMakeLists.txt index 2c08ff0b98..ded4dad9c9 100644 --- a/share/openPMD/thirdParty/pybind11/CMakeLists.txt +++ b/share/openPMD/thirdParty/pybind11/CMakeLists.txt @@ -45,8 +45,13 @@ if(NOT pybind11_FIND_QUIETLY) message(STATUS "pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}") endif() +# Avoid infinite recursion if tests include this as a subdirectory +if(DEFINED PYBIND11_MASTER_PROJECT) + set(PYBIND11_TEST OFF) +endif() + # Check if pybind11 is being used directly or via add_subdirectory -if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND NOT DEFINED PYBIND11_MASTER_PROJECT) ### Warn if not an out-of-source builds if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) set(lines @@ -73,6 +78,8 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) endif() + + set(pybind11_system "") else() set(PYBIND11_MASTER_PROJECT OFF) set(pybind11_system SYSTEM) @@ -159,12 +166,24 @@ endif() # You can also place ifs *in* the Config.in, but not here. # This section builds targets, but does *not* touch Python - -# Build the headers-only target (no Python included): -# (long name used here to keep this from clashing in subdirectory mode) -add_library(pybind11_headers INTERFACE) -add_library(pybind11::pybind11_headers ALIAS pybind11_headers) # to match exported target -add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember +# Non-IMPORT targets cannot be defined twice +if(NOT TARGET pybind11_headers) + # Build the headers-only target (no Python included): + # (long name used here to keep this from clashing in subdirectory mode) + add_library(pybind11_headers INTERFACE) + add_library(pybind11::pybind11_headers ALIAS pybind11_headers) # to match exported target + add_library(pybind11::headers ALIAS pybind11_headers) # easier to use/remember + + target_include_directories( + pybind11_headers ${pybind11_system} INTERFACE $ + $) + + target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals + cxx_right_angle_brackets) +else() + # It is invalid to install a target twice, too. + set(PYBIND11_INSTALL OFF) +endif() include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") @@ -175,14 +194,6 @@ elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR) file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) endif() -# Fill in headers target -target_include_directories( - pybind11_headers ${pybind11_system} INTERFACE $ - $) - -target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals - cxx_right_angle_brackets) - if(PYBIND11_INSTALL) install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) set(PYBIND11_CMAKECONFIG_INSTALL_DIR diff --git a/share/openPMD/thirdParty/pybind11/README.rst b/share/openPMD/thirdParty/pybind11/README.rst index fc9126da37..85eb96ce56 100644 --- a/share/openPMD/thirdParty/pybind11/README.rst +++ b/share/openPMD/thirdParty/pybind11/README.rst @@ -5,18 +5,27 @@ |Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |CI| |Build status| +|Repology| |PyPI package| |Conda-forge| |Python Versions| + +`Setuptools example `_ +• `Scikit-build example `_ +• `CMake example `_ + +.. start + .. warning:: - Combining older versions of pybind11 (< 2.6.0) with the brand-new Python - 3.9.0 will trigger undefined behavior that typically manifests as crashes - during interpreter shutdown (but could also destroy your data. **You have been + Combining older versions of pybind11 (< 2.6.0) with Python 3.9.0 will + trigger undefined behavior that typically manifests as crashes during + interpreter shutdown (but could also destroy your data. **You have been warned.**) - We recommend that you wait for Python 3.9.1 slated for release in December, - which will include a `fix `_ - that resolves this problem. In the meantime, please update to the latest - version of pybind11 (2.6.0 or newer), which includes a temporary workaround - specifically when Python 3.9.0 is detected at runtime. + We recommend that you update to the latest patch release of Python (3.9.1), + which includes a `fix `_ + that resolves this problem. If you do use Python 3.9.0, please update to + the latest version of pybind11 (2.6.0 or newer), which includes a temporary + workaround specifically when Python 3.9.0 is detected at runtime. + **pybind11** is a lightweight header-only library that exposes C++ types in Python and vice versa, mainly to create Python bindings of existing @@ -36,7 +45,7 @@ this heavy machinery has become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python -with everything stripped away that isn’t relevant for binding +with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K lines of code and depend on Python (2.7 or 3.5+, or PyPy) and the C++ standard library. This compact implementation was possible thanks to @@ -93,7 +102,7 @@ goodies: whenever possible to efficiently transfer custom data types. - It's easy to expose the internal storage of custom data types through - Python's buffer protocols. THis is handy e.g. for fast conversion + Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -101,7 +110,7 @@ goodies: transparently applied to all entries of one or more NumPy array arguments. -- Python’s slice-based access and assignment operations can be +- Python's slice-based access and assignment operations can be supported with just a few lines of code. - Everything is contained in just a few header files; there is no need @@ -123,15 +132,14 @@ goodies: Supported compilers ------------------- -1. Clang/LLVM 3.3 or newer (for Apple Xcode’s clang, this is 5.0.0 or +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) 2. GCC 4.8 or newer 3. Microsoft Visual Studio 2015 Update 3 or newer -4. Intel C++ compiler 18 or newer - (`possible issue `_ on 20.2) -5. Cygwin/GCC (tested on 2.5.1) -6. NVCC (CUDA 11.0 tested) -7. NVIDIA PGI (20.7 and 20.9 tested) +4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) +5. Cygwin/GCC (previously tested on 2.5.1) +6. NVCC (CUDA 11.0 tested in CI) +7. NVIDIA PGI (20.9 tested in CI) About ----- @@ -165,7 +173,7 @@ to the terms and conditions of this license. .. |Latest Documentation Status| image:: https://readthedocs.org/projects/pybind11/badge?version=latest :target: http://pybind11.readthedocs.org/en/latest -.. |Stable Documentation Status| image:: https://img.shields.io/badge/docs-stable-blue +.. |Stable Documentation Status| image:: https://img.shields.io/badge/docs-stable-blue.svg :target: http://pybind11.readthedocs.org/en/stable .. |Gitter chat| image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg :target: https://gitter.im/pybind/Lobby @@ -173,3 +181,11 @@ to the terms and conditions of this license. :target: https://github.com/pybind/pybind11/actions .. |Build status| image:: https://ci.appveyor.com/api/projects/status/riaj54pn4h08xy40?svg=true :target: https://ci.appveyor.com/project/wjakob/pybind11 +.. |PyPI package| image:: https://img.shields.io/pypi/v/pybind11.svg + :target: https://pypi.org/project/pybind11/ +.. |Conda-forge| image:: https://img.shields.io/conda/vn/conda-forge/pybind11.svg + :target: https://github.com/conda-forge/pybind11-feedstock +.. |Repology| image:: https://repology.org/badge/latest-versions/python:pybind11.svg + :target: https://repology.org/project/python:pybind11/versions +.. |Python Versions| image:: https://img.shields.io/pypi/pyversions/pybind11.svg + :target: https://pypi.org/project/pybind11/ diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h b/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h index 0c41670926..50efdc7cec 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/attr.h @@ -544,7 +544,7 @@ template ::value...), size_t self = constexpr_sum(std::is_same::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { - return named == 0 || (self + named + has_args + has_kwargs) == nargs; + return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; } PYBIND11_NAMESPACE_END(detail) diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h b/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h index 11c61a441f..0caccdb25b 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/cast.h @@ -960,9 +960,14 @@ template class type_caster> { private: using caster_t = make_caster; caster_t subcaster; - using subcaster_cast_op_type = typename caster_t::template cast_op_type; - static_assert(std::is_same::type &, subcaster_cast_op_type>::value, - "std::reference_wrapper caster requires T to have a caster with an `T &` operator"); + using reference_t = type&; + using subcaster_cast_op_type = + typename caster_t::template cast_op_type; + + static_assert(std::is_same::type &, subcaster_cast_op_type>::value || + std::is_same::value, + "std::reference_wrapper caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; @@ -973,7 +978,7 @@ template class type_caster> { return caster_t::cast(&src.get(), policy, parent); } template using cast_op_type = std::reference_wrapper; - operator std::reference_wrapper() { return subcaster.operator subcaster_cast_op_type&(); } + operator std::reference_wrapper() { return cast_op(subcaster); } }; #define PYBIND11_TYPE_CASTER(type, py_name) \ @@ -1020,6 +1025,14 @@ struct type_caster::value && !is_std_char_t if (!src) return false; +#if !defined(PYPY_VERSION) + auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; +#else + // In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, + // while CPython only considers the existence of `nb_index`/`__index__`. + auto index_check = [](PyObject *o) { return hasattr(o, "__index__"); }; +#endif + if (std::is_floating_point::value) { if (convert || PyFloat_Check(src.ptr())) py_value = (py_type) PyFloat_AsDouble(src.ptr()); @@ -1027,12 +1040,31 @@ struct type_caster::value && !is_std_char_t return false; } else if (PyFloat_Check(src.ptr())) { return false; - } else if (std::is_unsigned::value) { - py_value = as_unsigned(src.ptr()); - } else { // signed integer: - py_value = sizeof(T) <= sizeof(long) - ? (py_type) PyLong_AsLong(src.ptr()) - : (py_type) PYBIND11_LONG_AS_LONGLONG(src.ptr()); + } else if (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr())) { + return false; + } else { + handle src_or_index = src; +#if PY_VERSION_HEX < 0x03080000 + object index; + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + index = reinterpret_steal(PyNumber_Index(src.ptr())); + if (!index) { + PyErr_Clear(); + if (!convert) + return false; + } + else { + src_or_index = index; + } + } +#endif + if (std::is_unsigned::value) { + py_value = as_unsigned(src_or_index.ptr()); + } else { // signed integer: + py_value = sizeof(T) <= sizeof(long) + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + } } // Python API reported an error @@ -1041,15 +1073,8 @@ struct type_caster::value && !is_std_char_t // Check to see if the conversion is valid (integers should match exactly) // Signed/unsigned checks happen elsewhere if (py_err || (std::is_integral::value && sizeof(py_type) != sizeof(T) && py_value != (py_type) (T) py_value)) { - bool type_error = py_err && PyErr_ExceptionMatches( -#if PY_VERSION_HEX < 0x03000000 && !defined(PYPY_VERSION) - PyExc_SystemError -#else - PyExc_TypeError -#endif - ); PyErr_Clear(); - if (type_error && convert && PyNumber_Check(src.ptr())) { + if (py_err && convert && PyNumber_Check(src.ptr())) { auto tmp = reinterpret_steal(std::is_floating_point::value ? PyNumber_Float(src.ptr()) : PyNumber_Long(src.ptr())); @@ -1870,7 +1895,14 @@ struct arg_v : arg { #if !defined(NDEBUG) , type(type_id()) #endif - { } + { + // Workaround! See: + // https://github.com/pybind/pybind11/issues/2336 + // https://github.com/pybind/pybind11/pull/2685#issuecomment-731286700 + if (PyErr_Occurred()) { + PyErr_Clear(); + } + } public: /// Direct construction with name, default, and description @@ -2160,16 +2192,26 @@ class unpacking_collector { dict m_kwargs; }; +// [workaround(intel)] Separate function required here +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_are_all_positional() +{ + return all_of...>::value; +} + /// Collect only positional arguments for a Python function call template ...>::value>> + typename = enable_if_t()>> simple_collector collect_arguments(Args &&...args) { return simple_collector(std::forward(args)...); } /// Collect all arguments, including keywords and unpacking (only instantiated when needed) template ...>::value>> + typename = enable_if_t()>> unpacking_collector collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 static_assert( diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/class.h b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/class.h index 65dad5a57e..2f414e5c7c 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/class.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/class.h @@ -340,8 +340,6 @@ inline PyObject *make_new_instance(PyTypeObject *type) { // Allocate the value/holder internals: inst->allocate_layout(); - inst->owned = true; - return self; } @@ -552,6 +550,12 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla } std::memset(view, 0, sizeof(Py_buffer)); buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { + delete info; + // view->obj = nullptr; // Was just memset to 0, so not necessary + PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + return -1; + } view->obj = obj; view->ndim = 1; view->internal = info; @@ -561,12 +565,6 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla for (auto s : info->shape) view->len *= s; view->readonly = info->readonly; - if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { - if (view) - view->obj = nullptr; - PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); - return -1; - } if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) view->format = const_cast(info->format.c_str()); if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/common.h b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/common.h index 751141a927..de495e4f9e 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/common.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/common.h @@ -11,7 +11,7 @@ #define PYBIND11_VERSION_MAJOR 2 #define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH 1 +#define PYBIND11_VERSION_PATCH 2 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } @@ -27,7 +27,7 @@ # endif #endif -#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) +#if !(defined(_MSC_VER) && __cplusplus == 199711L) # if __cplusplus >= 201402L # define PYBIND11_CPP14 # if __cplusplus >= 201703L @@ -49,6 +49,8 @@ #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 # error pybind11 requires Intel C++ compiler v18 or newer +# elif __INTEL_COMPILER < 1900 && defined(PYBIND11_CPP14) +# error pybind11 supports only C++11 with Intel C++ compiler v18. Use v19 or newer for C++14. # endif #elif defined(__clang__) && !defined(__apple_build_version__) # if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3) @@ -663,6 +665,10 @@ template using is_function_pointer = bool_constant< std::is_pointer::value && std::is_function::type>::value>; template struct strip_function_object { + // If you are encountering an + // 'error: name followed by "::" must be a class or namespace name' + // with the Intel compiler and a noexcept function here, + // try to use noexcept(true) instead of plain noexcept. using type = typename remove_class::type; }; @@ -687,8 +693,10 @@ template using is_lambda = satisfies_none_of, /// Ignore that a variable is unused in compiler warnings inline void ignore_unused(const int *) { } +// [workaround(intel)] Internal error on fold expression /// Apply a function over each element of a parameter pack -#ifdef __cpp_fold_expressions +#if defined(__cpp_fold_expressions) && !defined(__INTEL_COMPILER) +// Intel compiler produces an internal error on this fold expression (tested with ICC 19.0.2) #define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) (((PATTERN), void()), ...) #else using expand_side_effects = bool[]; diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/internals.h b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/internals.h index a455715bfd..75fcd3c208 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/detail/internals.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/detail/internals.h @@ -112,7 +112,7 @@ struct internals { PyInterpreterState *istate = nullptr; ~internals() { // This destructor is called *after* Py_Finalize() in finalize_interpreter(). - // That *SHOULD BE* fine. The following details what happens whe PyThread_tss_free is called. + // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is called. // PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does nothing. // PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). Neither @@ -266,7 +266,7 @@ PYBIND11_NOINLINE inline internals &get_internals() { const PyGILState_STATE state; } gil; - constexpr auto *id = PYBIND11_INTERNALS_ID; + PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); auto builtins = handle(PyEval_GetBuiltins()); if (builtins.contains(id) && isinstance(builtins[id])) { internals_pp = static_cast(capsule(builtins[id])); diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/iostream.h b/share/openPMD/thirdParty/pybind11/include/pybind11/iostream.h index 5e9a8143d0..9dee755431 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/iostream.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/iostream.h @@ -43,16 +43,20 @@ class pythonbuf : public std::streambuf { // simplified to a fully qualified call. int _sync() { if (pbase() != pptr()) { - // This subtraction cannot be negative, so dropping the sign - str line(pbase(), static_cast(pptr() - pbase())); { gil_scoped_acquire tmp; + + // This subtraction cannot be negative, so dropping the sign. + str line(pbase(), static_cast(pptr() - pbase())); + pywrite(line); pyflush(); + + // Placed inside gil_scoped_aquire as a mutex to avoid a race + setp(pbase(), epptr()); } - setp(pbase(), epptr()); } return 0; } @@ -102,8 +106,8 @@ PYBIND11_NAMESPACE_END(detail) .. code-block:: cpp { - py::scoped_ostream_redirect output{std::cerr, py::module_::import("sys").attr("stderr")}; - std::cerr << "Hello, World!"; + py::scoped_ostream_redirect output{std::cerr, py::module::import("sys").attr("stderr")}; + std::cout << "Hello, World!"; } \endrst */ class scoped_ostream_redirect { diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/pybind11.h b/share/openPMD/thirdParty/pybind11/include/pybind11/pybind11.h index e2ddda020b..3bffbb28d2 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/pybind11.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/pybind11.h @@ -114,9 +114,16 @@ class cpp_function : public function { object name() const { return attr("__name__"); } protected: + struct InitializingFunctionRecordDeleter { + // `destruct(function_record, false)`: `initialize_generic` copies strings and + // takes care of cleaning up in case of exceptions. So pass `false` to `free_strings`. + void operator()(detail::function_record * rec) { destruct(rec, false); } + }; + using unique_function_record = std::unique_ptr; + /// Space optimization: don't inline this frequently instantiated fragment - PYBIND11_NOINLINE detail::function_record *make_function_record() { - return new detail::function_record(); + PYBIND11_NOINLINE unique_function_record make_function_record() { + return unique_function_record(new detail::function_record()); } /// Special internal constructor for functors, lambda functions, etc. @@ -126,7 +133,9 @@ class cpp_function : public function { struct capture { remove_reference_t f; }; /* Store the function including any extra state it might have (e.g. a lambda capture object) */ - auto rec = make_function_record(); + // The unique_ptr makes sure nothing is leaked in case of an exception. + auto unique_rec = make_function_record(); + auto rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ if (sizeof(capture) <= sizeof(rec->data)) { @@ -207,7 +216,8 @@ class cpp_function : public function { PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ - initialize_generic(rec, signature.text, types.data(), sizeof...(Args)); + // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid. + initialize_generic(std::move(unique_rec), signature.text, types.data(), sizeof...(Args)); if (cast_in::has_args) rec->has_args = true; if (cast_in::has_kwargs) rec->has_kwargs = true; @@ -223,20 +233,51 @@ class cpp_function : public function { } } + // Utility class that keeps track of all duplicated strings, and cleans them up in its destructor, + // unless they are released. Basically a RAII-solution to deal with exceptions along the way. + class strdup_guard { + public: + ~strdup_guard() { + for (auto s : strings) + std::free(s); + } + char *operator()(const char *s) { + auto t = strdup(s); + strings.push_back(t); + return t; + } + void release() { + strings.clear(); + } + private: + std::vector strings; + }; + /// Register a function call with Python (generic non-templated code goes here) - void initialize_generic(detail::function_record *rec, const char *text, + void initialize_generic(unique_function_record &&unique_rec, const char *text, const std::type_info *const *types, size_t args) { + // Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr, + // we do not want this to destuct the pointer. `initialize` (the caller) still relies on the + // pointee being alive after this call. Only move out if a `capsule` is going to keep it alive. + auto rec = unique_rec.get(); + + // Keep track of strdup'ed strings, and clean them up as long as the function's capsule + // has not taken ownership yet (when `unique_rec.release()` is called). + // Note: This cannot easily be fixed by a `unique_ptr` with custom deleter, because the strings + // are only referenced before strdup'ing. So only *after* the following block could `destruct` + // safely be called, but even then, `repr` could still throw in the middle of copying all strings. + strdup_guard guarded_strdup; /* Create copies of all referenced C-style strings */ - rec->name = strdup(rec->name ? rec->name : ""); - if (rec->doc) rec->doc = strdup(rec->doc); + rec->name = guarded_strdup(rec->name ? rec->name : ""); + if (rec->doc) rec->doc = guarded_strdup(rec->doc); for (auto &a: rec->args) { if (a.name) - a.name = strdup(a.name); + a.name = guarded_strdup(a.name); if (a.descr) - a.descr = strdup(a.descr); + a.descr = guarded_strdup(a.descr); else if (a.value) - a.descr = strdup(repr(a.value).cast().c_str()); + a.descr = guarded_strdup(repr(a.value).cast().c_str()); } rec->is_constructor = !strcmp(rec->name, "__init__") || !strcmp(rec->name, "__setstate__"); @@ -319,13 +360,13 @@ class cpp_function : public function { #if PY_MAJOR_VERSION < 3 if (strcmp(rec->name, "__next__") == 0) { std::free(rec->name); - rec->name = strdup("next"); + rec->name = guarded_strdup("next"); } else if (strcmp(rec->name, "__bool__") == 0) { std::free(rec->name); - rec->name = strdup("__nonzero__"); + rec->name = guarded_strdup("__nonzero__"); } #endif - rec->signature = strdup(signature.c_str()); + rec->signature = guarded_strdup(signature.c_str()); rec->args.shrink_to_fit(); rec->nargs = (std::uint16_t) args; @@ -356,9 +397,10 @@ class cpp_function : public function { rec->def->ml_meth = reinterpret_cast(reinterpret_cast(*dispatcher)); rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; - capsule rec_capsule(rec, [](void *ptr) { + capsule rec_capsule(unique_rec.release(), [](void *ptr) { destruct((detail::function_record *) ptr); }); + guarded_strdup.release(); object scope_module; if (rec->scope) { @@ -393,13 +435,15 @@ class cpp_function : public function { chain_start = rec; rec->next = chain; auto rec_capsule = reinterpret_borrow(((PyCFunctionObject *) m_ptr)->m_self); - rec_capsule.set_pointer(rec); + rec_capsule.set_pointer(unique_rec.release()); + guarded_strdup.release(); } else { // Or end of chain (normal behavior) chain_start = chain; while (chain->next) chain = chain->next; - chain->next = rec; + chain->next = unique_rec.release(); + guarded_strdup.release(); } } @@ -439,9 +483,9 @@ class cpp_function : public function { /* Install docstring */ auto *func = (PyCFunctionObject *) m_ptr; - if (func->m_ml->ml_doc) - std::free(const_cast(func->m_ml->ml_doc)); - func->m_ml->ml_doc = strdup(signatures.c_str()); + std::free(const_cast(func->m_ml->ml_doc)); + // Install docstring if it's non-empty (when at least one option is enabled) + func->m_ml->ml_doc = signatures.empty() ? nullptr : strdup(signatures.c_str()); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); @@ -452,7 +496,7 @@ class cpp_function : public function { } /// When a cpp_function is GCed, release any memory allocated by pybind11 - static void destruct(detail::function_record *rec) { + static void destruct(detail::function_record *rec, bool free_strings = true) { // If on Python 3.9, check the interpreter "MICRO" (patch) version. // If this is running on 3.9.0, we have to work around a bug. #if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 @@ -463,14 +507,20 @@ class cpp_function : public function { detail::function_record *next = rec->next; if (rec->free_data) rec->free_data(rec); - std::free((char *) rec->name); - std::free((char *) rec->doc); - std::free((char *) rec->signature); - for (auto &arg: rec->args) { - std::free(const_cast(arg.name)); - std::free(const_cast(arg.descr)); - arg.value.dec_ref(); + // During initialization, these strings might not have been copied yet, + // so they cannot be freed. Once the function has been created, they can. + // Check `make_function_record` for more details. + if (free_strings) { + std::free((char *) rec->name); + std::free((char *) rec->doc); + std::free((char *) rec->signature); + for (auto &arg: rec->args) { + std::free(const_cast(arg.name)); + std::free(const_cast(arg.descr)); + } } + for (auto &arg: rec->args) + arg.value.dec_ref(); if (rec->def) { std::free(const_cast(rec->def->ml_doc)); // Python 3.9.0 decref's these in the wrong order; rec->def @@ -504,15 +554,15 @@ class cpp_function : public function { auto self_value_and_holder = value_and_holder(); if (overloads->is_constructor) { - const auto tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); - const auto pi = reinterpret_cast(parent.ptr()); - self_value_and_holder = pi->get_value_and_holder(tinfo, false); - - if (!self_value_and_holder.type || !self_value_and_holder.inst) { + if (!PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { PyErr_SetString(PyExc_TypeError, "__init__(self, ...) called with invalid `self` argument"); return nullptr; } + const auto tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); + const auto pi = reinterpret_cast(parent.ptr()); + self_value_and_holder = pi->get_value_and_holder(tinfo, true); + // If this value is already registered it must mean __init__ is invoked multiple times; // we really can't support that in C++, so just ignore the second __init__. if (self_value_and_holder.instance_registered()) @@ -977,7 +1027,7 @@ class module_ : public object { /** \rst Create a new top-level module that can be used as the main module of a C extension. - For Python 3, ``def`` should point to a staticly allocated module_def. + For Python 3, ``def`` should point to a statically allocated module_def. For Python 2, ``def`` can be a nullptr and is completely ignored. \endrst */ static module_ create_extension_module(const char *name, const char *doc, module_def *def) { @@ -1005,7 +1055,7 @@ class module_ : public object { throw error_already_set(); pybind11_fail("Internal error in module_::create_extension_module()"); } - // TODO: Sould be reinterpret_steal for Python 3, but Python also steals it again when returned from PyInit_... + // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when returned from PyInit_... // For Python 2, reinterpret_borrow is correct. return reinterpret_borrow(m); } @@ -1731,6 +1781,7 @@ template class enum_ : public class_ { m_base.init(is_arithmetic, is_convertible); def(init([](Scalar i) { return static_cast(i); }), arg("value")); + def_property_readonly("value", [](Type value) { return (Scalar) value; }); def("__int__", [](Type value) { return (Scalar) value; }); #if PY_MAJOR_VERSION < 3 def("__long__", [](Type value) { return (Scalar) value; }); @@ -1909,7 +1960,7 @@ template void implicitly_convertible() { struct set_flag { bool &flag; - set_flag(bool &flag) : flag(flag) { flag = true; } + set_flag(bool &flag_) : flag(flag_) { flag_ = true; } ~set_flag() { flag = false; } }; auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * { @@ -2089,15 +2140,7 @@ class gil_scoped_acquire { } if (release) { - /* Work around an annoying assertion in PyThreadState_Swap */ - #if defined(Py_DEBUG) - PyInterpreterState *interp = tstate->interp; - tstate->interp = nullptr; - #endif PyEval_AcquireThread(tstate); - #if defined(Py_DEBUG) - tstate->interp = interp; - #endif } inc_ref(); @@ -2121,12 +2164,22 @@ class gil_scoped_acquire { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); #endif PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); + if (active) + PyThreadState_DeleteCurrent(); PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); release = false; } } + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { + active = false; + } + PYBIND11_NOINLINE ~gil_scoped_acquire() { dec_ref(); if (release) @@ -2135,6 +2188,7 @@ class gil_scoped_acquire { private: PyThreadState *tstate = nullptr; bool release = true; + bool active = true; }; class gil_scoped_release { @@ -2150,10 +2204,22 @@ class gil_scoped_release { PYBIND11_TLS_DELETE_VALUE(key); } } + + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { + active = false; + } + ~gil_scoped_release() { if (!tstate) return; - PyEval_RestoreThread(tstate); + // `PyEval_RestoreThread()` should not be called if runtime is finalizing + if (active) + PyEval_RestoreThread(tstate); if (disassoc) { auto key = detail::get_internals().tstate; PYBIND11_TLS_REPLACE_VALUE(key, tstate); @@ -2162,6 +2228,7 @@ class gil_scoped_release { private: PyThreadState *tstate; bool disassoc; + bool active = true; }; #elif defined(PYPY_VERSION) class gil_scoped_acquire { @@ -2169,6 +2236,7 @@ class gil_scoped_acquire { public: gil_scoped_acquire() { state = PyGILState_Ensure(); } ~gil_scoped_acquire() { PyGILState_Release(state); } + void disarm() {} }; class gil_scoped_release { @@ -2176,10 +2244,15 @@ class gil_scoped_release { public: gil_scoped_release() { state = PyEval_SaveThread(); } ~gil_scoped_release() { PyEval_RestoreThread(state); } + void disarm() {} }; #else -class gil_scoped_acquire { }; -class gil_scoped_release { }; +class gil_scoped_acquire { + void disarm() {} +}; +class gil_scoped_release { + void disarm() {} +}; #endif error_already_set::~error_already_set() { @@ -2254,7 +2327,7 @@ PYBIND11_NAMESPACE_END(detail) /** \rst Try to retrieve a python method by the provided name from the instance pointed to by the this_ptr. - :this_ptr: The pointer to the object the overriden method should be retrieved for. This should be + :this_ptr: The pointer to the object the overridden method should be retrieved for. This should be the first non-trampoline class encountered in the inheritance chain. :name: The name of the overridden Python method to retrieve. :return: The Python method by this name from the object or an empty function wrapper. diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/pytypes.h b/share/openPMD/thirdParty/pybind11/include/pybind11/pytypes.h index 1010ad713c..78db7947d9 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/pytypes.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/pytypes.h @@ -812,18 +812,18 @@ PYBIND11_NAMESPACE_END(detail) : Parent(check_(o) ? o.release().ptr() : ConvertFun(o.ptr()), stolen_t{}) \ { if (!m_ptr) throw error_already_set(); } -#define PYBIND11_OBJECT_CHECK_FAILED(Name, o) \ +#define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ ::pybind11::type_error("Object of type '" + \ - ::pybind11::detail::get_fully_qualified_tp_name(Py_TYPE(o.ptr())) + \ + ::pybind11::detail::get_fully_qualified_tp_name(Py_TYPE(o_ptr)) + \ "' is not an instance of '" #Name "'") #define PYBIND11_OBJECT(Name, Parent, CheckFun) \ PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ Name(const object &o) : Parent(o) \ - { if (o && !check_(o)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, o); } \ + { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } \ Name(object &&o) : Parent(std::move(o)) \ - { if (o && !check_(o)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, o); } + { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ PYBIND11_OBJECT(Name, Parent, CheckFun) \ @@ -1271,6 +1271,15 @@ class tuple : public object { detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> part below +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_are_all_keyword_or_ds() +{ + return detail::all_of...>::value; +} + class dict : public object { public: PYBIND11_OBJECT_CVT(dict, object, PyDict_Check, raw_dict) @@ -1278,7 +1287,7 @@ class dict : public object { if (!m_ptr) pybind11_fail("Could not allocate dict object!"); } template ...>::value>, + typename = detail::enable_if_t()>, // MSVC workaround: it can't compile an out-of-line definition, so defer the collector typename collector = detail::deferred_t, Args...>> explicit dict(Args &&...args) : dict(collector(std::forward(args)...).kwargs()) { } diff --git a/share/openPMD/thirdParty/pybind11/include/pybind11/stl_bind.h b/share/openPMD/thirdParty/pybind11/include/pybind11/stl_bind.h index 9d8ed0c825..83195ee496 100644 --- a/share/openPMD/thirdParty/pybind11/include/pybind11/stl_bind.h +++ b/share/openPMD/thirdParty/pybind11/include/pybind11/stl_bind.h @@ -375,10 +375,20 @@ struct vector_has_data_and_format : std::false_type {}; template struct vector_has_data_and_format::format(), std::declval().data()), typename Vector::value_type*>::value>> : std::true_type {}; +// [workaround(intel)] Separate function required here +// Workaround as the Intel compiler does not compile the enable_if_t part below +// (tested with icc (ICC) 2021.1 Beta 20200827) +template +constexpr bool args_any_are_buffer() { + return detail::any_of...>::value; +} + +// [workaround(intel)] Separate function required here +// [workaround(msvc)] Can't use constexpr bool in return type + // Add the buffer interface to a vector template -enable_if_t...>::value> -vector_buffer(Class_& cl) { +void vector_buffer_impl(Class_& cl, std::true_type) { using T = typename Vector::value_type; static_assert(vector_has_data_and_format::value, "There is not an appropriate format descriptor for this vector"); @@ -416,7 +426,12 @@ vector_buffer(Class_& cl) { } template -enable_if_t...>::value> vector_buffer(Class_&) {} +void vector_buffer_impl(Class_&, std::false_type) {} + +template +void vector_buffer(Class_& cl) { + vector_buffer_impl(cl, detail::any_of...>{}); +} PYBIND11_NAMESPACE_END(detail) diff --git a/share/openPMD/thirdParty/pybind11/pybind11/_version.py b/share/openPMD/thirdParty/pybind11/pybind11/_version.py index d18535cd7b..f8b795eebc 100644 --- a/share/openPMD/thirdParty/pybind11/pybind11/_version.py +++ b/share/openPMD/thirdParty/pybind11/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s): return s -__version__ = "2.6.1" +__version__ = "2.6.2" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/share/openPMD/thirdParty/pybind11/pybind11/setup_helpers.py b/share/openPMD/thirdParty/pybind11/pybind11/setup_helpers.py index 33605ddfd1..c69064ca50 100644 --- a/share/openPMD/thirdParty/pybind11/pybind11/setup_helpers.py +++ b/share/openPMD/thirdParty/pybind11/pybind11/setup_helpers.py @@ -99,15 +99,14 @@ class Pybind11Extension(_Extension): this is an ugly old-style class due to Distutils. """ - def _add_cflags(self, *flags): - for flag in flags: - if flag not in self.extra_compile_args: - self.extra_compile_args.append(flag) + # flags are prepended, so that they can be further overridden, e.g. by + # ``extra_compile_args=["-g"]``. - def _add_lflags(self, *flags): - for flag in flags: - if flag not in self.extra_link_args: - self.extra_link_args.append(flag) + def _add_cflags(self, flags): + self.extra_compile_args[:0] = flags + + def _add_ldflags(self, flags): + self.extra_link_args[:0] = flags def __init__(self, *args, **kwargs): @@ -139,13 +138,17 @@ def __init__(self, *args, **kwargs): # Have to use the accessor manually to support Python 2 distutils Pybind11Extension.cxx_std.__set__(self, cxx_std) + cflags = [] + ldflags = [] if WIN: - self._add_cflags("/EHsc", "/bigobj") + cflags += ["/EHsc", "/bigobj"] else: - self._add_cflags("-fvisibility=hidden", "-g0") + cflags += ["-fvisibility=hidden", "-g0"] if MACOS: - self._add_cflags("-stdlib=libc++") - self._add_lflags("-stdlib=libc++") + cflags += ["-stdlib=libc++"] + ldflags += ["-stdlib=libc++"] + self._add_cflags(cflags) + self._add_ldflags(ldflags) @property def cxx_std(self): @@ -174,7 +177,8 @@ def cxx_std(self, level): if not level: return - self.extra_compile_args.append(STD_TMPL.format(level)) + cflags = [STD_TMPL.format(level)] + ldflags = [] if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ: # C++17 requires a higher min version of macOS. An earlier version @@ -186,18 +190,21 @@ def cxx_std(self, level): desired_macos = (10, 9) if level < 17 else (10, 14) macos_string = ".".join(str(x) for x in min(current_macos, desired_macos)) macosx_min = "-mmacosx-version-min=" + macos_string - self.extra_compile_args.append(macosx_min) - self.extra_link_args.append(macosx_min) + cflags += [macosx_min] + ldflags += [macosx_min] if PY2: if WIN: # Will be ignored on MSVC 2015, where C++17 is not supported so # this flag is not valid. - self.extra_compile_args.append("/wd5033") + cflags += ["/wd5033"] elif level >= 17: - self.extra_compile_args.append("-Wno-register") + cflags += ["-Wno-register"] elif level >= 14: - self.extra_compile_args.append("-Wno-deprecated-register") + cflags += ["-Wno-deprecated-register"] + + self._add_cflags(cflags) + self._add_ldflags(ldflags) # Just in case someone clever tries to multithread @@ -232,7 +239,8 @@ def has_flag(compiler, flag): with tmp_chdir(): fname = "flagcheck.cpp" with open(fname, "w") as f: - f.write("int main (int argc, char **argv) { return 0; }") + # Don't trigger -Wunused-parameter. + f.write("int main (int, char **) { return 0; }") try: compiler.compile([fname], extra_postargs=[flag]) diff --git a/share/openPMD/thirdParty/pybind11/tools/FindEigen3.cmake b/share/openPMD/thirdParty/pybind11/tools/FindEigen3.cmake index 98ab43d9e6..83625d92e0 100644 --- a/share/openPMD/thirdParty/pybind11/tools/FindEigen3.cmake +++ b/share/openPMD/thirdParty/pybind11/tools/FindEigen3.cmake @@ -64,6 +64,9 @@ if(EIGEN3_INCLUDE_DIR) set(EIGEN3_FOUND ${EIGEN3_VERSION_OK}) else(EIGEN3_INCLUDE_DIR) + if(NOT DEFINED KDE4_INCLUDE_DIR) + set(KDE4_INCLUDE_DIR "") + endif() find_path( EIGEN3_INCLUDE_DIR diff --git a/share/openPMD/thirdParty/pybind11/tools/FindPythonLibsNew.cmake b/share/openPMD/thirdParty/pybind11/tools/FindPythonLibsNew.cmake index b7125912c8..3605aebcf3 100644 --- a/share/openPMD/thirdParty/pybind11/tools/FindPythonLibsNew.cmake +++ b/share/openPMD/thirdParty/pybind11/tools/FindPythonLibsNew.cmake @@ -57,6 +57,8 @@ endif() if(PythonLibsNew_FIND_QUIETLY) set(_pythonlibs_quiet QUIET) +else() + set(_pythonlibs_quiet "") endif() if(PythonLibsNew_FIND_REQUIRED) diff --git a/share/openPMD/thirdParty/pybind11/tools/pybind11Config.cmake.in b/share/openPMD/thirdParty/pybind11/tools/pybind11Config.cmake.in index 9808f3d255..9921aeb355 100644 --- a/share/openPMD/thirdParty/pybind11/tools/pybind11Config.cmake.in +++ b/share/openPMD/thirdParty/pybind11/tools/pybind11Config.cmake.in @@ -87,7 +87,7 @@ you can either use the basic targets, or use the FindPython tools: target_link_libraries(MyModule2 pybind11::headers) set_target_properties(MyModule2 PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON - CXX__VISIBILITY_PRESET ON + CXX_VISIBILITY_PRESET ON VISIBLITY_INLINES_HIDDEN ON) If you build targets yourself, you may be interested in stripping the output diff --git a/share/openPMD/thirdParty/pybind11/tools/pybind11NewTools.cmake b/share/openPMD/thirdParty/pybind11/tools/pybind11NewTools.cmake index 357cc61c13..18da8be170 100644 --- a/share/openPMD/thirdParty/pybind11/tools/pybind11NewTools.cmake +++ b/share/openPMD/thirdParty/pybind11/tools/pybind11NewTools.cmake @@ -12,6 +12,8 @@ get_property( if(pybind11_FIND_QUIETLY) set(_pybind11_quiet QUIET) +else() + set(_pybind11_quiet "") endif() if(CMAKE_VERSION VERSION_LESS 3.12) @@ -127,10 +129,20 @@ endif() # Check on every access - since Python2 and Python3 could have been used - do nothing in that case. if(DEFINED ${_Python}_INCLUDE_DIRS) + # Only add Python for build - must be added during the import for config + # since it has to be re-discovered. + # + # This needs to be a target to be included after the local pybind11 + # directory, just in case there there is an installed pybind11 sitting + # next to Python's includes. It also ensures Python is a SYSTEM library. + add_library(pybind11::python_headers INTERFACE IMPORTED) + set_property( + TARGET pybind11::python_headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES + "$") set_property( TARGET pybind11::pybind11 APPEND - PROPERTY INTERFACE_INCLUDE_DIRECTORIES $) + PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python_headers) set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" "${${_Python}_INCLUDE_DIRS}" CACHE INTERNAL "Directories where pybind11 and possibly Python headers are located") @@ -144,11 +156,11 @@ if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS 3) endif() # In CMake 3.18+, you can find these separately, so include an if -if(TARGET ${_Python}::${_Python}) +if(TARGET ${_Python}::Python) set_property( TARGET pybind11::embed APPEND - PROPERTY INTERFACE_LINK_LIBRARIES ${_Python}::${_Python}) + PROPERTY INTERFACE_LINK_LIBRARIES ${_Python}::Python) endif() # CMake 3.15+ has this @@ -170,27 +182,27 @@ function(pybind11_add_module target_name) cmake_parse_arguments(PARSE_ARGV 1 ARG "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI" "" "") - if(ARG_ADD_LIBRARY_STATIC) - set(type STATIC) - elseif(ARG_ADD_LIBRARY_SHARED) - set(type SHARED) + if(ARG_STATIC) + set(lib_type STATIC) + elseif(ARG_SHARED) + set(lib_type SHARED) else() - set(type MODULE) + set(lib_type MODULE) endif() if("${_Python}" STREQUAL "Python") - python_add_library(${target_name} ${type} ${ARG_UNPARSED_ARGUMENTS}) + python_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") - python3_add_library(${target_name} ${type} ${ARG_UNPARSED_ARGUMENTS}) + python3_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python2") - python2_add_library(${target_name} ${type} ${ARG_UNPARSED_ARGUMENTS}) + python2_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) else() message(FATAL_ERROR "Cannot detect FindPython version: ${_Python}") endif() target_link_libraries(${target_name} PRIVATE pybind11::headers) - if(type STREQUAL "MODULE") + if(lib_type STREQUAL "MODULE") target_link_libraries(${target_name} PRIVATE pybind11::module) else() target_link_libraries(${target_name} PRIVATE pybind11::embed) @@ -204,12 +216,21 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::python2_no_register) endif() - set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden" - CUDA_VISIBILITY_PRESET "hidden") + # -fvisibility=hidden is required to allow multiple modules compiled against + # different pybind versions to work properly, and for some features (e.g. + # py::module_local). We force it on everything inside the `pybind11` + # namespace; also turning it on for a pybind module compilation here avoids + # potential warnings or issues from having mixed hidden/non-hidden types. + if(NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET) + set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden") + endif() + + if(NOT DEFINED CMAKE_CUDA_VISIBILITY_PRESET) + set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden") + endif() # If we don't pass a WITH_SOABI or WITHOUT_SOABI, use our own default handling of extensions - if("${type}" STREQUAL "MODULE" AND (NOT ARG_WITHOUT_SOABI OR NOT "WITH_SOABI" IN_LIST - ARG_UNPARSED_ARGUMENTS)) + if(NOT ARG_WITHOUT_SOABI OR NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) pybind11_extension(${target_name}) endif() diff --git a/share/openPMD/thirdParty/pybind11/tools/pybind11Tools.cmake b/share/openPMD/thirdParty/pybind11/tools/pybind11Tools.cmake index 23cff98e89..3231353998 100644 --- a/share/openPMD/thirdParty/pybind11/tools/pybind11Tools.cmake +++ b/share/openPMD/thirdParty/pybind11/tools/pybind11Tools.cmake @@ -10,6 +10,8 @@ include(CMakeParseArguments) if(pybind11_FIND_QUIETLY) set(_pybind11_quiet QUIET) +else() + set(_pybind11_quiet "") endif() # If this is the first run, PYTHON_VERSION can stand in for PYBIND11_PYTHON_VERSION @@ -22,16 +24,21 @@ if(NOT DEFINED PYBIND11_PYTHON_VERSION AND DEFINED PYTHON_VERSION) CACHE STRING "Python version to use for compiling modules") unset(PYTHON_VERSION) unset(PYTHON_VERSION CACHE) -else() - # If this is set as a normal variable, promote it, otherwise, make an empty cache variable. +elseif(DEFINED PYBIND11_PYTHON_VERSION) + # If this is set as a normal variable, promote it set(PYBIND11_PYTHON_VERSION "${PYBIND11_PYTHON_VERSION}" CACHE STRING "Python version to use for compiling modules") +else() + # Make an empty cache variable. + set(PYBIND11_PYTHON_VERSION + "" + CACHE STRING "Python version to use for compiling modules") endif() # A user can set versions manually too set(Python_ADDITIONAL_VERSIONS - "3.9;3.8;3.7;3.6;3.5;3.4" + "3.10;3.9;3.8;3.7;3.6;3.5;3.4" CACHE INTERNAL "") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") @@ -81,11 +88,19 @@ if(PYBIND11_MASTER_PROJECT) endif() endif() -# Only add Python for build - must be added during the import for config since it has to be re-discovered. +# Only add Python for build - must be added during the import for config since +# it has to be re-discovered. +# +# This needs to be an target to it is included after the local pybind11 +# directory, just in case there are multiple versions of pybind11, we want the +# one we expect. +add_library(pybind11::python_headers INTERFACE IMPORTED) +set_property(TARGET pybind11::python_headers PROPERTY INTERFACE_INCLUDE_DIRECTORIES + "$") set_property( TARGET pybind11::pybind11 APPEND - PROPERTY INTERFACE_INCLUDE_DIRECTORIES $) + PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python_headers) set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" "${PYTHON_INCLUDE_DIRS}" @@ -166,8 +181,13 @@ function(pybind11_add_module target_name) # py::module_local). We force it on everything inside the `pybind11` # namespace; also turning it on for a pybind module compilation here avoids # potential warnings or issues from having mixed hidden/non-hidden types. - set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden" - CUDA_VISIBILITY_PRESET "hidden") + if(NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET) + set_target_properties(${target_name} PROPERTIES CXX_VISIBILITY_PRESET "hidden") + endif() + + if(NOT DEFINED CMAKE_CUDA_VISIBILITY_PRESET) + set_target_properties(${target_name} PROPERTIES CUDA_VISIBILITY_PRESET "hidden") + endif() if(ARG_NO_EXTRAS) return()