From 5cd3311c6c7eeea09947cac2bc2902e919f36667 Mon Sep 17 00:00:00 2001 From: Wenzel Jakob Date: Tue, 20 Oct 2015 00:58:59 +0200 Subject: [PATCH] added benchmark --- docs/benchmark.py | 90 ++++++ docs/benchmark.rst | 70 +++++ docs/index.rst | 1 + docs/pybind11_vs_boost_python1.svg | 439 +++++++++++++++++++++++++++++ docs/pybind11_vs_boost_python2.svg | 427 ++++++++++++++++++++++++++++ 5 files changed, 1027 insertions(+) create mode 100644 docs/benchmark.py create mode 100644 docs/benchmark.rst create mode 100644 docs/pybind11_vs_boost_python1.svg create mode 100644 docs/pybind11_vs_boost_python2.svg diff --git a/docs/benchmark.py b/docs/benchmark.py new file mode 100644 index 00000000..efa82847 --- /dev/null +++ b/docs/benchmark.py @@ -0,0 +1,90 @@ +import random +import os +import time +import datetime as dt + +nfns = 4 # Functions per class +nargs = 4 # Arguments per function + + +def generate_dummy_code_pybind11(nclasses=10): + decl = "" + bindings = "" + + for cl in range(nclasses): + decl += "class cl%03i;\n" % cl + decl += '\n' + + for cl in range(nclasses): + decl += "class cl%03i {\n" % cl + decl += "public:\n" + bindings += ' py::class_(m, "cl%03i")\n' % (cl, cl) + for fn in range(nfns): + ret = random.randint(0, nclasses - 1) + params = [random.randint(0, nclasses - 1) for i in range(nargs)] + decl += " cl%03i *fn_%03i(" % (ret, fn) + decl += ", ".join("cl%03i *" % p for p in params) + decl += ");\n" + bindings += ' .def("fn_%03i", &cl%03i::fn_%03i)\n' % \ + (fn, cl, fn) + decl += "};\n\n" + bindings += ' ;\n' + + result = "#include \n\n" + result += "namespace py = pybind11;\n\n" + result += decl + '\n' + result += "PYBIND11_PLUGIN(example) {\n" + result += " py::module m(\"example\");" + result += bindings + result += " return m.ptr();" + result += "}" + return result + + +def generate_dummy_code_boost(nclasses=10): + decl = "" + bindings = "" + + for cl in range(nclasses): + decl += "class cl%03i;\n" % cl + decl += '\n' + + for cl in range(nclasses): + decl += "class cl%03i {\n" % cl + decl += "public:\n" + bindings += ' py::class_("cl%03i")\n' % (cl, cl) + for fn in range(nfns): + ret = random.randint(0, nclasses - 1) + params = [random.randint(0, nclasses - 1) for i in range(nargs)] + decl += " cl%03i *fn_%03i(" % (ret, fn) + decl += ", ".join("cl%03i *" % p for p in params) + decl += ");\n" + bindings += ' .def("fn_%03i", &cl%03i::fn_%03i, py::return_value_policy())\n' % \ + (fn, cl, fn) + decl += "};\n\n" + bindings += ' ;\n' + + result = "#include \n\n" + result += "namespace py = boost::python;\n\n" + result += decl + '\n' + result += "BOOST_PYTHON_MODULE(example) {\n" + result += bindings + result += "}" + return result + + +for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: + print ("{") + for i in range(0, 10): + nclasses = 2 ** i + with open("test.cpp", "w") as f: + f.write(codegen(nclasses)) + n1 = dt.datetime.now() + os.system("g++ -Os -shared -rdynamic -undefined dynamic_lookup " + "-fvisibility=hidden -std=c++11 test.cpp -I include " + "-I /System/Library/Frameworks/Python.framework/Headers -o test.so") + n2 = dt.datetime.now() + elapsed = (n2 - n1).total_seconds() + size = os.stat('test.so').st_size + print(" {%i, %f, %i}," % (nclasses * nfns, elapsed, size)) + print ("}") diff --git a/docs/benchmark.rst b/docs/benchmark.rst new file mode 100644 index 00000000..c06bfb39 --- /dev/null +++ b/docs/benchmark.rst @@ -0,0 +1,70 @@ +Benchmark +========= + +The following is the result of a synthetic benchmark comparing both compilation +time and module size of pybind11 against Boost.Python. + +A python script (see the ``docs/benchmark.py`` file) was used to generate a +set of dummy classes whose count increases for each successive benchmark +(between 1 and 512 classes in powers of two). Each class has four methods with +a randomly generated signature with a return value and four arguments. (There +was no particular reason for this setup other than the desire to generate many +unique function signatures whose count could be controlled in a simple way.) + +Here is an example of the binding code for one class: + +.. code-block:: cpp + + ... + class cl034 { + public: + cl279 *fn_000(cl084 *, cl057 *, cl065 *, cl042 *); + cl025 *fn_001(cl098 *, cl262 *, cl414 *, cl121 *); + cl085 *fn_002(cl445 *, cl297 *, cl145 *, cl421 *); + cl470 *fn_003(cl200 *, cl323 *, cl332 *, cl492 *); + }; + ... + + PYBIND11_PLUGIN(example) { + py::module m("example"); + ... + py::class_(m, "cl034") + .def("fn_000", &cl034::fn_000) + .def("fn_001", &cl034::fn_001) + .def("fn_002", &cl034::fn_002) + .def("fn_003", &cl034::fn_003) + ... + return m.ptr(); + } + +The Boost.Python version looks almost identical except that a return value +policy had to be specified as an argument to ``def()``. For both libraries, +compilation was done with + +.. code-block:: bash + + Apple LLVM version 7.0.0 (clang-700.0.72) + +and the following compilation flags + +.. code-block:: bash + + g++ -Os -shared -rdynamic -undefined dynamic_lookup -fvisibility=hidden -std=c++11 + +The following log-log plot shows how the compilation time grows for an +increasing number of class and function declarations. pybind11 includes fewer +headers, which initially leads to shorter compilation times, but the +performance is ultimately very similar (pybind11 is 1 second faster for the +largest file, which is less than 1% of the total compilation time). + +.. image:: pybind11_vs_boost_python1.svg + +Differences between the two libraries become more pronounced when considering +the file size of the generated Python plugin. Note that the plot below does not +include the size of the Boost.Python shared library, hence Boost actually has a +slight advantage. + +.. image:: pybind11_vs_boost_python2.svg + +Despite this, the libraries procuced by Boost.Python for more than a few +functions are consistently larger by a factor of 1.75. diff --git a/docs/index.rst b/docs/index.rst index 21437a3a..44f47c3e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,4 +12,5 @@ Contents: classes advanced cmake + benchmark reference diff --git a/docs/pybind11_vs_boost_python1.svg b/docs/pybind11_vs_boost_python1.svg new file mode 100644 index 00000000..8ffacf1a --- /dev/null +++ b/docs/pybind11_vs_boost_python1.svg @@ -0,0 +1,439 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/pybind11_vs_boost_python2.svg b/docs/pybind11_vs_boost_python2.svg new file mode 100644 index 00000000..dbaf9aa0 --- /dev/null +++ b/docs/pybind11_vs_boost_python2.svg @@ -0,0 +1,427 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +