diff --git a/docs/classes.rst b/docs/classes.rst index 4270b8d6..300816d4 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -165,6 +165,66 @@ the setter and getter functions: static variables and properties. Please also see the section on :ref:`static_properties` in the advanced part of the documentation. +Dynamic attributes +================== + +Native Python classes can pick up new attributes dynamically: + +.. code-block:: pycon + + >>> class Pet: + ... name = 'Molly' + ... + >>> p = Pet() + >>> p.name = 'Charly' # overwrite existing + >>> p.age = 2 # dynamically add a new attribute + +By default, classes exported from C++ do not support this and the only writable +attributes are the ones explicitly defined using :func:`class_::def_readwrite` +or :func:`class_::def_property`. + +.. code-block:: cpp + + py::class_(m, "Pet") + .def(py::init<>()) + .def_readwrite("name", &Pet::name); + +Trying to set any other attribute results in an error: + +.. code-block:: pycon + + >>> p = example.Pet() + >>> p.name = 'Charly' # OK, attribute defined in C++ + >>> p.age = 2 # fail + AttributeError: 'Pet' object has no attribute 'age' + +To enable dynamic attributes for C++ classes, the :class:`py::dynamic_attr` tag +must be added to the :class:`py::class_` constructor: + +.. code-block:: cpp + + py::class_(m, "Pet", py::dynamic_attr()) + .def(py::init<>()) + .def_readwrite("name", &Pet::name); + +Now everything works as expected: + +.. code-block:: pycon + + >>> p = example.Pet() + >>> p.name = 'Charly' # OK, overwrite value in C++ + >>> p.age = 2 # OK, dynamically add a new attribute + >>> p.__dict__ # just like a native Python class + {'age': 2} + +Note that there is a small runtime cost for a class with dynamic attributes. +Not only because of the addition of a ``__dict__``, but also because of more +expensive garbage collection tracking which must be activated to resolve +possible circular references. Native Python classes incur this same cost by +default, so this is not anything to worry about. By default, pybind11 classes +are more efficient than native Python classes. Enabling dynamic attributes +just brings them on par. + .. _inheritance: Inheritance diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index f37e8627..15bb2e4a 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -44,6 +44,9 @@ template struct keep_alive { }; /// Annotation indicating that a class is involved in a multiple inheritance relationship struct multiple_inheritance { }; +/// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class +struct dynamic_attr { }; + NAMESPACE_BEGIN(detail) /* Forward declarations */ enum op_id : int; @@ -162,6 +165,9 @@ struct type_record { /// Multiple inheritance marker bool multiple_inheritance = false; + /// Does the class manage a __dict__? + bool dynamic_attr = false; + PYBIND11_NOINLINE void add_base(const std::type_info *base, void *(*caster)(void *)) { auto base_info = detail::get_type_info(*base, false); if (!base_info) { @@ -292,6 +298,11 @@ struct process_attribute : process_attribute_defaultmultiple_inheritance = true; } }; +template <> +struct process_attribute : process_attribute_default { + static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } +}; + /*** * Process a keep_alive call policy -- invokes keep_alive_impl during the * pre-call handler if both Nurse, Patient != 0 and use the post-call handler diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index f7d46cf3..f19435d3 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -573,6 +573,33 @@ public: }; NAMESPACE_BEGIN(detail) +extern "C" inline PyObject *get_dict(PyObject *op, void *) { + PyObject *&dict = *_PyObject_GetDictPtr(op); + if (!dict) { + dict = PyDict_New(); + } + Py_XINCREF(dict); + return dict; +} + +extern "C" inline int set_dict(PyObject *op, PyObject *new_dict, void *) { + if (!PyDict_Check(new_dict)) { + PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'", + Py_TYPE(new_dict)->tp_name); + return -1; + } + PyObject *&dict = *_PyObject_GetDictPtr(op); + Py_INCREF(new_dict); + Py_CLEAR(dict); + dict = new_dict; + return 0; +} + +static PyGetSetDef generic_getset[] = { + {const_cast("__dict__"), get_dict, set_dict, nullptr, nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr} +}; + /// Generic support for creating new Python heap types class generic_type : public object { template friend class class_; @@ -684,6 +711,16 @@ protected: #endif type->ht_type.tp_flags &= ~Py_TPFLAGS_HAVE_GC; + /* Support dynamic attributes */ + if (rec->dynamic_attr) { + type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_GC; + type->ht_type.tp_dictoffset = type->ht_type.tp_basicsize; // place the dict at the end + type->ht_type.tp_basicsize += sizeof(PyObject *); // and allocate enough space for it + type->ht_type.tp_getset = generic_getset; + type->ht_type.tp_traverse = traverse; + type->ht_type.tp_clear = clear; + } + type->ht_type.tp_doc = tp_doc; if (PyType_Ready(&type->ht_type) < 0) @@ -785,10 +822,27 @@ protected: if (self->weakrefs) PyObject_ClearWeakRefs((PyObject *) self); + + PyObject **dict_ptr = _PyObject_GetDictPtr((PyObject *) self); + if (dict_ptr) { + Py_CLEAR(*dict_ptr); + } } Py_TYPE(self)->tp_free((PyObject*) self); } + static int traverse(PyObject *op, visitproc visit, void *arg) { + PyObject *&dict = *_PyObject_GetDictPtr(op); + Py_VISIT(dict); + return 0; + } + + static int clear(PyObject *op) { + PyObject *&dict = *_PyObject_GetDictPtr(op); + Py_CLEAR(dict); + return 0; + } + void install_buffer_funcs( buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index 8b0351e4..4948dc09 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -53,6 +53,12 @@ public: int value = 0; }; +class DynamicClass { +public: + DynamicClass() { print_default_created(this); } + ~DynamicClass() { print_destroyed(this); } +}; + test_initializer methods_and_attributes([](py::module &m) { py::class_(m, "ExampleMandA") .def(py::init<>()) @@ -81,4 +87,7 @@ test_initializer methods_and_attributes([](py::module &m) { .def("__str__", &ExampleMandA::toString) .def_readwrite("value", &ExampleMandA::value) ; + + py::class_(m, "DynamicClass", py::dynamic_attr()) + .def(py::init()); }); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 9340e6fb..04f2d12a 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -1,3 +1,4 @@ +import pytest from pybind11_tests import ExampleMandA, ConstructorStats @@ -44,3 +45,67 @@ def test_methods_and_attributes(): assert cstats.move_constructions >= 1 assert cstats.copy_assignments == 0 assert cstats.move_assignments == 0 + + +def test_dynamic_attributes(): + from pybind11_tests import DynamicClass + + instance = DynamicClass() + assert not hasattr(instance, "foo") + assert "foo" not in dir(instance) + + # Dynamically add attribute + instance.foo = 42 + assert hasattr(instance, "foo") + assert instance.foo == 42 + assert "foo" in dir(instance) + + # __dict__ should be accessible and replaceable + assert "foo" in instance.__dict__ + instance.__dict__ = {"bar": True} + assert not hasattr(instance, "foo") + assert hasattr(instance, "bar") + + with pytest.raises(TypeError) as excinfo: + instance.__dict__ = [] + assert str(excinfo.value) == "__dict__ must be set to a dictionary, not a 'list'" + + cstats = ConstructorStats.get(DynamicClass) + assert cstats.alive() == 1 + del instance + assert cstats.alive() == 0 + + # Derived classes should work as well + class Derived(DynamicClass): + pass + + derived = Derived() + derived.foobar = 100 + assert derived.foobar == 100 + + assert cstats.alive() == 1 + del derived + assert cstats.alive() == 0 + + +def test_cyclic_gc(): + from pybind11_tests import DynamicClass + + # One object references itself + instance = DynamicClass() + instance.circular_reference = instance + + cstats = ConstructorStats.get(DynamicClass) + assert cstats.alive() == 1 + del instance + assert cstats.alive() == 0 + + # Two object reference each other + i1 = DynamicClass() + i2 = DynamicClass() + i1.cycle = i2 + i2.cycle = i1 + + assert cstats.alive() == 2 + del i1, i2 + assert cstats.alive() == 0 diff --git a/tests/test_pickling.cpp b/tests/test_pickling.cpp index 4494c24d..3941dc59 100644 --- a/tests/test_pickling.cpp +++ b/tests/test_pickling.cpp @@ -24,6 +24,14 @@ private: int m_extra2 = 0; }; +class PickleableWithDict { +public: + PickleableWithDict(const std::string &value) : value(value) { } + + std::string value; + int extra; +}; + test_initializer pickling([](py::module &m) { py::class_(m, "Pickleable") .def(py::init()) @@ -48,4 +56,26 @@ test_initializer pickling([](py::module &m) { p.setExtra1(t[1].cast()); p.setExtra2(t[2].cast()); }); + + py::class_(m, "PickleableWithDict", py::dynamic_attr()) + .def(py::init()) + .def_readwrite("value", &PickleableWithDict::value) + .def_readwrite("extra", &PickleableWithDict::extra) + .def("__getstate__", [](py::object self) { + /* Also include __dict__ in state */ + return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__")); + }) + .def("__setstate__", [](py::object self, py::tuple t) { + if (t.size() != 3) + throw std::runtime_error("Invalid state!"); + /* Cast and construct */ + auto& p = self.cast(); + new (&p) Pickleable(t[0].cast()); + + /* Assign C++ state */ + p.extra = t[1].cast(); + + /* Assign Python state */ + self.attr("__dict__") = t[2]; + }); }); diff --git a/tests/test_pickling.py b/tests/test_pickling.py index f6e4c04f..5e62e1fc 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -3,10 +3,10 @@ try: except ImportError: import pickle -from pybind11_tests import Pickleable - def test_roundtrip(): + from pybind11_tests import Pickleable + p = Pickleable("test_value") p.setExtra1(15) p.setExtra2(48) @@ -16,3 +16,17 @@ def test_roundtrip(): assert p2.value() == p.value() assert p2.extra1() == p.extra1() assert p2.extra2() == p.extra2() + + +def test_roundtrip_with_dict(): + from pybind11_tests import PickleableWithDict + + p = PickleableWithDict("test_value") + p.extra = 15 + p.dynamic = "Attribute" + + data = pickle.dumps(p, pickle.HIGHEST_PROTOCOL) + p2 = pickle.loads(data) + assert p2.value == p.value + assert p2.extra == p.extra + assert p2.dynamic == p.dynamic