From 024932b3799044f8b259d6d4f2760f3e900c1f06 Mon Sep 17 00:00:00 2001 From: Dean Moldovan Date: Sun, 20 Aug 2017 16:15:33 +0200 Subject: [PATCH] Move everything related to `internals` into a separate detail header --- CMakeLists.txt | 1 + include/pybind11/cast.h | 103 +----------- include/pybind11/detail/common.h | 108 ------------- include/pybind11/detail/internals.h | 232 ++++++++++++++++++++++++++++ setup.py | 1 + 5 files changed, 235 insertions(+), 210 deletions(-) create mode 100644 include/pybind11/detail/internals.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ac4e9921..cad29a9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ set(PYBIND11_HEADERS include/pybind11/detail/common.h include/pybind11/detail/descr.h include/pybind11/detail/init.h + include/pybind11/detail/internals.h include/pybind11/detail/typeid.h include/pybind11/attr.h include/pybind11/buffer_info.h diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 805fea6f..4c3abc1c 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -13,6 +13,7 @@ #include "pytypes.h" #include "detail/typeid.h" #include "detail/descr.h" +#include "detail/internals.h" #include #include #include @@ -32,108 +33,6 @@ NAMESPACE_BEGIN(PYBIND11_NAMESPACE) NAMESPACE_BEGIN(detail) -// Forward declarations: -inline PyTypeObject *make_static_property_type(); -inline PyTypeObject *make_default_metaclass(); -inline PyObject *make_object_base_type(PyTypeObject *metaclass); -struct value_and_holder; - -/// Additional type information which does not fit into the PyTypeObject -struct type_info { - PyTypeObject *type; - const std::type_info *cpptype; - size_t type_size, holder_size_in_ptrs; - void *(*operator_new)(size_t); - void (*init_instance)(instance *, const void *); - void (*dealloc)(value_and_holder &v_h); - std::vector implicit_conversions; - std::vector> implicit_casts; - std::vector *direct_conversions; - buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; - void *get_buffer_data = nullptr; - void *(*module_local_load)(PyObject *, const type_info *) = nullptr; - /* A simple type never occurs as a (direct or indirect) parent - * of a class that makes use of multiple inheritance */ - bool simple_type : 1; - /* True if there is no multiple inheritance in this type's inheritance tree */ - bool simple_ancestors : 1; - /* for base vs derived holder_type checks */ - bool default_holder : 1; - /* true if this is a type registered with py::module_local */ - bool module_local : 1; -}; - -PYBIND11_NOINLINE inline internals *&get_internals_ptr() { - static internals *internals_ptr = nullptr; - return internals_ptr; -} - -PYBIND11_NOINLINE inline internals &get_internals() { - internals *&internals_ptr = get_internals_ptr(); - if (internals_ptr) - return *internals_ptr; - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr = *static_cast(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our error_already_set and - // builtin_exception may be different local classes than the ones set up in the initial - // exception translator, below, so add another for our local exception classes. - // - // stdlibc++ doesn't require this (types there are identified only by name) - #if !defined(__GLIBCXX__) - internals_ptr->registered_exception_translators.push_front( - [](std::exception_ptr p) -> void { - try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; - } - } - ); - #endif - } else { - internals_ptr = new internals(); - #if defined(WITH_THREAD) - PyEval_InitThreads(); - PyThreadState *tstate = PyThreadState_Get(); - internals_ptr->tstate = PyThread_create_key(); - PyThread_set_key_value(internals_ptr->tstate, tstate); - internals_ptr->istate = tstate->interp; - #endif - builtins[id] = capsule(&internals_ptr); - internals_ptr->registered_exception_translators.push_front( - [](std::exception_ptr p) -> void { - try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; - } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; - } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::length_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::out_of_range &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; - } catch (const std::range_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::exception &e) { PyErr_SetString(PyExc_RuntimeError, e.what()); return; - } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!"); - return; - } - } - ); - internals_ptr->static_property_type = make_static_property_type(); - internals_ptr->default_metaclass = make_default_metaclass(); - internals_ptr->instance_base = make_object_base_type(internals_ptr->default_metaclass); - } - return *internals_ptr; -} - -// Works like internals.registered_types_cpp, but for module-local registered types: -PYBIND11_NOINLINE inline type_map ®istered_local_types_cpp() { - static type_map locals{}; - return locals; -} /// A life support system for temporary objects created by `type_caster::load()`. /// Adding a patient will keep it alive up until the enclosing function returns. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6c914db2..feefc128 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -205,8 +205,6 @@ extern "C" { #define PYBIND11_TRY_NEXT_OVERLOAD ((PyObject *) 1) // special failure return code #define PYBIND11_STRINGIFY(x) #x #define PYBIND11_TOSTRING(x) PYBIND11_STRINGIFY(x) -#define PYBIND11_INTERNALS_ID "__pybind11_" \ - PYBIND11_TOSTRING(PYBIND11_VERSION_MAJOR) "_" PYBIND11_TOSTRING(PYBIND11_VERSION_MINOR) "__" /** \rst ***Deprecated in favor of PYBIND11_MODULE*** @@ -442,73 +440,6 @@ struct instance { static_assert(std::is_standard_layout::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); -struct overload_hash { - inline size_t operator()(const std::pair& v) const { - size_t value = std::hash()(v.first); - value ^= std::hash()(v.second) + 0x9e3779b9 + (value<<6) + (value>>2); - return value; - } -}; - -// Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly -// other stls, this means `typeid(A)` from one module won't equal `typeid(A)` from another module -// even when `A` is the same, non-hidden-visibility type (e.g. from a common include). Under -// stdlibc++, this doesn't happen: equality and the type_index hash are based on the type name, -// which works. If not under a known-good stl, provide our own name-based hasher and equality -// functions that use the type name. -#if defined(__GLIBCXX__) -inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } -using type_hash = std::hash; -using type_equal_to = std::equal_to; -#else -inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { - return lhs.name() == rhs.name() || - std::strcmp(lhs.name(), rhs.name()) == 0; -} -struct type_hash { - size_t operator()(const std::type_index &t) const { - size_t hash = 5381; - const char *ptr = t.name(); - while (auto c = static_cast(*ptr++)) - hash = (hash * 33) ^ c; - return hash; - } -}; -struct type_equal_to { - bool operator()(const std::type_index &lhs, const std::type_index &rhs) const { - return lhs.name() == rhs.name() || - std::strcmp(lhs.name(), rhs.name()) == 0; - } -}; -#endif - -template -using type_map = std::unordered_map; - -/// Internal data structure used to track registered instances and types -struct internals { - type_map registered_types_cpp; // std::type_index -> type_info - std::unordered_map> registered_types_py; // PyTypeObject* -> base type_info(s) - std::unordered_multimap registered_instances; // void * -> instance* - std::unordered_set, overload_hash> inactive_overload_cache; - type_map> direct_conversions; - std::unordered_map> patients; - std::forward_list registered_exception_translators; - std::unordered_map shared_data; // Custom data to be shared across extensions - std::vector loader_patient_stack; // Used by `loader_life_support` - std::forward_list static_strings; // Stores the std::strings backing detail::c_str() - PyTypeObject *static_property_type; - PyTypeObject *default_metaclass; - PyObject *instance_base; -#if defined(WITH_THREAD) - decltype(PyThread_create_key()) tstate = 0; // Usually an int but a long on Cygwin64 with Python 3.x - PyInterpreterState *istate = nullptr; -#endif -}; - -/// Return a reference to the current 'internals' information -inline internals &get_internals(); - /// from __cpp_future__ import (convenient aliases from C++14/17) #if defined(PYBIND11_CPP14) && (!defined(_MSC_VER) || _MSC_VER >= 1910) using std::enable_if_t; @@ -716,47 +647,8 @@ using expand_side_effects = bool[]; #define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) pybind11::detail::expand_side_effects{ ((PATTERN), void(), false)..., false } #endif -/// Constructs a std::string with the given arguments, stores it in `internals`, and returns its -/// `c_str()`. Such strings objects have a long storage duration -- the internal strings are only -/// cleared when the program exits or after interpreter shutdown (when embedding), and so are -/// suitable for c-style strings needed by Python internals (such as PyTypeObject's tp_name). -template const char *c_str(Args &&...args) { - auto &strings = get_internals().static_strings; - strings.emplace_front(std::forward(args)...); - return strings.front().c_str(); -} - NAMESPACE_END(detail) -/// Returns a named pointer that is shared among all extension modules (using the same -/// pybind11 version) running in the current interpreter. Names starting with underscores -/// are reserved for internal usage. Returns `nullptr` if no matching entry was found. -inline PYBIND11_NOINLINE void* get_shared_data(const std::string& name) { - auto& internals = detail::get_internals(); - auto it = internals.shared_data.find(name); - return it != internals.shared_data.end() ? it->second : nullptr; -} - -/// Set the shared data that can be later recovered by `get_shared_data()`. -inline PYBIND11_NOINLINE void *set_shared_data(const std::string& name, void *data) { - detail::get_internals().shared_data[name] = data; - return data; -} - -/// Returns a typed reference to a shared data entry (by using `get_shared_data()`) if -/// such entry exists. Otherwise, a new object of default-constructible type `T` is -/// added to the shared data under the given name and a reference to it is returned. -template T& get_or_create_shared_data(const std::string& name) { - auto& internals = detail::get_internals(); - auto it = internals.shared_data.find(name); - T* ptr = (T*) (it != internals.shared_data.end() ? it->second : nullptr); - if (!ptr) { - ptr = new T(); - internals.shared_data[name] = ptr; - } - return *ptr; -} - /// C++ bindings of builtin Python exceptions class builtin_exception : public std::runtime_error { public: diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h new file mode 100644 index 00000000..73f34df5 --- /dev/null +++ b/include/pybind11/detail/internals.h @@ -0,0 +1,232 @@ +/* + pybind11/detail/internals.h: Internal data structure and related functions + + Copyright (c) 2017 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../pytypes.h" + +NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +NAMESPACE_BEGIN(detail) +// Forward declarations +inline PyTypeObject *make_static_property_type(); +inline PyTypeObject *make_default_metaclass(); +inline PyObject *make_object_base_type(PyTypeObject *metaclass); + +// Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly +// other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module +// even when `A` is the same, non-hidden-visibility type (e.g. from a common include). Under +// libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, +// which works. If not under a known-good stl, provide our own name-based hash and equality +// functions that use the type name. +#if defined(__GLIBCXX__) +inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } +using type_hash = std::hash; +using type_equal_to = std::equal_to; +#else +inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { + return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; +} + +struct type_hash { + size_t operator()(const std::type_index &t) const { + size_t hash = 5381; + const char *ptr = t.name(); + while (auto c = static_cast(*ptr++)) + hash = (hash * 33) ^ c; + return hash; + } +}; + +struct type_equal_to { + bool operator()(const std::type_index &lhs, const std::type_index &rhs) const { + return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; + } +}; +#endif + +template +using type_map = std::unordered_map; + +struct overload_hash { + inline size_t operator()(const std::pair& v) const { + size_t value = std::hash()(v.first); + value ^= std::hash()(v.second) + 0x9e3779b9 + (value<<6) + (value>>2); + return value; + } +}; + +/// Internal data structure used to track registered instances and types. +struct internals { + type_map registered_types_cpp; // std::type_index -> type_info + std::unordered_map> registered_types_py; // PyTypeObject* -> base type_info(s) + std::unordered_multimap registered_instances; // void * -> instance* + std::unordered_set, overload_hash> inactive_overload_cache; + type_map> direct_conversions; + std::unordered_map> patients; + std::forward_list registered_exception_translators; + std::unordered_map shared_data; // Custom data to be shared across extensions + std::vector loader_patient_stack; // Used by `loader_life_support` + std::forward_list static_strings; // Stores the std::strings backing detail::c_str() + PyTypeObject *static_property_type; + PyTypeObject *default_metaclass; + PyObject *instance_base; +#if defined(WITH_THREAD) + decltype(PyThread_create_key()) tstate = 0; // Usually an int but a long on Cygwin64 with Python 3.x + PyInterpreterState *istate = nullptr; +#endif +}; + +/// Additional type information which does not fit into the PyTypeObject. +struct type_info { + PyTypeObject *type; + const std::type_info *cpptype; + size_t type_size, holder_size_in_ptrs; + void *(*operator_new)(size_t); + void (*init_instance)(instance *, const void *); + void (*dealloc)(value_and_holder &v_h); + std::vector implicit_conversions; + std::vector> implicit_casts; + std::vector *direct_conversions; + buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; + void *get_buffer_data = nullptr; + void *(*module_local_load)(PyObject *, const type_info *) = nullptr; + /* A simple type never occurs as a (direct or indirect) parent + * of a class that makes use of multiple inheritance */ + bool simple_type : 1; + /* True if there is no multiple inheritance in this type's inheritance tree */ + bool simple_ancestors : 1; + /* for base vs derived holder_type checks */ + bool default_holder : 1; + /* true if this is a type registered with py::module_local */ + bool module_local : 1; +}; + +#define PYBIND11_INTERNALS_ID "__pybind11_" \ + PYBIND11_TOSTRING(PYBIND11_VERSION_MAJOR) "_" PYBIND11_TOSTRING(PYBIND11_VERSION_MINOR) "__" + +/// Each module locally stores a pointer to the `internals` data. The data +/// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. +PYBIND11_NOINLINE inline internals *&get_internals_ptr() { + static internals *internals_ptr = nullptr; + return internals_ptr; +} + +/// Return a reference to the current `internals` data +PYBIND11_NOINLINE inline internals &get_internals() { + auto *&internals_ptr = get_internals_ptr(); + if (internals_ptr) + return *internals_ptr; + + constexpr auto *id = PYBIND11_INTERNALS_ID; + auto builtins = handle(PyEval_GetBuiltins()); + if (builtins.contains(id) && isinstance(builtins[id])) { + internals_ptr = *static_cast(capsule(builtins[id])); + + // We loaded builtins through python's builtins, which means that our `error_already_set` + // and `builtin_exception` may be different local classes than the ones set up in the + // initial exception translator, below, so add another for our local exception classes. + // + // libstdc++ doesn't require this (types there are identified only by name) +#if !defined(__GLIBCXX__) + internals_ptr->registered_exception_translators.push_front( + [](std::exception_ptr p) -> void { + try { + if (p) std::rethrow_exception(p); + } catch (error_already_set &e) { e.restore(); return; + } catch (const builtin_exception &e) { e.set_error(); return; + } + } + ); +#endif + } else { + internals_ptr = new internals(); +#if defined(WITH_THREAD) + PyEval_InitThreads(); + PyThreadState *tstate = PyThreadState_Get(); + internals_ptr->tstate = PyThread_create_key(); + PyThread_set_key_value(internals_ptr->tstate, tstate); + internals_ptr->istate = tstate->interp; +#endif + builtins[id] = capsule(&internals_ptr); + internals_ptr->registered_exception_translators.push_front( + [](std::exception_ptr p) -> void { + try { + if (p) std::rethrow_exception(p); + } catch (error_already_set &e) { e.restore(); return; + } catch (const builtin_exception &e) { e.set_error(); return; + } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; + } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::length_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::out_of_range &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; + } catch (const std::range_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; + } catch (const std::exception &e) { PyErr_SetString(PyExc_RuntimeError, e.what()); return; + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!"); + return; + } + } + ); + internals_ptr->static_property_type = make_static_property_type(); + internals_ptr->default_metaclass = make_default_metaclass(); + internals_ptr->instance_base = make_object_base_type(internals_ptr->default_metaclass); + } + return *internals_ptr; +} + +/// Works like `internals.registered_types_cpp`, but for module-local registered types: +PYBIND11_NOINLINE inline type_map ®istered_local_types_cpp() { + static type_map locals{}; + return locals; +} + +/// Constructs a std::string with the given arguments, stores it in `internals`, and returns its +/// `c_str()`. Such strings objects have a long storage duration -- the internal strings are only +/// cleared when the program exits or after interpreter shutdown (when embedding), and so are +/// suitable for c-style strings needed by Python internals (such as PyTypeObject's tp_name). +template +const char *c_str(Args &&...args) { + auto &strings = get_internals().static_strings; + strings.emplace_front(std::forward(args)...); + return strings.front().c_str(); +} + +NAMESPACE_END(detail) + +/// Returns a named pointer that is shared among all extension modules (using the same +/// pybind11 version) running in the current interpreter. Names starting with underscores +/// are reserved for internal usage. Returns `nullptr` if no matching entry was found. +inline PYBIND11_NOINLINE void *get_shared_data(const std::string &name) { + auto &internals = detail::get_internals(); + auto it = internals.shared_data.find(name); + return it != internals.shared_data.end() ? it->second : nullptr; +} + +/// Set the shared data that can be later recovered by `get_shared_data()`. +inline PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *data) { + detail::get_internals().shared_data[name] = data; + return data; +} + +/// Returns a typed reference to a shared data entry (by using `get_shared_data()`) if +/// such entry exists. Otherwise, a new object of default-constructible type `T` is +/// added to the shared data under the given name and a reference to it is returned. +template +T &get_or_create_shared_data(const std::string &name) { + auto &internals = detail::get_internals(); + auto it = internals.shared_data.find(name); + T *ptr = (T *) (it != internals.shared_data.end() ? it->second : nullptr); + if (!ptr) { + ptr = new T(); + internals.shared_data[name] = ptr; + } + return *ptr; +} + +NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/setup.py b/setup.py index c67fa027..51ab47d1 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ else: 'include/pybind11/detail/common.h', 'include/pybind11/detail/descr.h', 'include/pybind11/detail/init.h', + 'include/pybind11/detail/internals.h', 'include/pybind11/detail/typeid.h' 'include/pybind11/attr.h', 'include/pybind11/buffer_info.h',