From 94cd7070e8ba2f45032f3af732f1c56351a3b26c Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 2 Sep 2022 08:33:50 -0700 Subject: [PATCH] moving some bootstrapping to python layer --- .efrocachemap | 58 +++--- CHANGELOG.md | 3 +- assets/.asset_manifest_public.json | 2 + assets/Makefile | 2 + assets/src/ba_data/python/ba/_bootstrap.py | 182 +++++++++++++++++++ src/ballistica/ballistica.cc | 2 +- src/meta/bameta/python_embedded/bootstrap.py | 158 +--------------- tools/efro/log.py | 36 ++-- 8 files changed, 247 insertions(+), 196 deletions(-) create mode 100644 assets/src/ba_data/python/ba/_bootstrap.py diff --git a/.efrocachemap b/.efrocachemap index 19ae2423..dd6f427f 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -3995,26 +3995,26 @@ "assets/src/ba_data/python/ba/_generated/__init__.py": "https://files.ballistica.net/cache/ba1/ee/e8/cad05aa531c7faf7ff7b96db7f6e", "assets/src/ba_data/python/ba/_generated/enums.py": "https://files.ballistica.net/cache/ba1/b2/e5/0ee0561e16257a32830645239f34", "ballisticacore-windows/Generic/BallisticaCore.ico": "https://files.ballistica.net/cache/ba1/89/c0/e32c7d2a35dc9aef57cc73b0911a", - "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/8f/35/1a6fbc2bd9d367b5b5d8350199da", - "build/prefab/full/linux_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/e8/df/e7aae0645d3813227e32628a0ff3", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ce/43/1d18f5d73d3fe5d7f1ed4fdc472c", - "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/cb/04/5dc6236fd0ebeafbd013299a4766", - "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/e1/3a/bbe527aae553058d38a89a85e6b5", - "build/prefab/full/linux_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/60/1c/2e11dded6067b1cb27e6f6d48a0a", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/59/fd/0f61ebccbcfb85d01693c431f5a6", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/9b/94/8a16341d49d6de25102c07c70675", - "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/ac/1d/d48d569a072d45d96ea86760b9f0", - "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/d7/31/f3a671560f4efb8708430f0ce985", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/6e/76/fa07d7183f1bd0d438657339d33a", - "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b0/8b/60a531e23f24bba638e3fc615ed3", - "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/9c/1b/519b7ba8f1718787d8ab62f61222", - "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/4b/a0/138ece248132798d69cc81bce581", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/a1/58/18ba7845fb9524a5cfef4d14bf7f", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/12/93/7e04d239fd333188b1412272c873", - "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/44/cb/2144fb8e2fc054d605e8e3baea77", - "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/89/90/a98081fbe24f8d062443ce84f3cc", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/c9/80/1de60807e22d9a46c6902badbe7f", - "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/d7/f1/e2d6d8fdedd4ec4f3a6c0cc6bc14", + "build/prefab/full/linux_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/16/68/50bab5698a6f581fb56ba314d953", + "build/prefab/full/linux_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/88/e0/d5043781c54e6eceb273b6df0d38", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/da/dd/d847840917c2b44c3faa7f360133", + "build/prefab/full/linux_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/73/06/99a81805a891faed7f126e795dda", + "build/prefab/full/linux_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/e3/45/1cba0d26da1b275b36e6dd5e2487", + "build/prefab/full/linux_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/28/8b/40a34cf4436f6644b4c60dd75ea5", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/d0/51/3a2ca66c96aca2f9649de78fa076", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c3/b3/4900e33f2a75408842f568a53f6b", + "build/prefab/full/mac_arm64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/32/26/b90ad9ea1ed8c6fc32684b52f741", + "build/prefab/full/mac_arm64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/8c/3c/77eaf5714f5fb4c793b8e5d2b7cf", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ff/9b/d13d185b8091790b28804d918d07", + "build/prefab/full/mac_arm64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c1/39/b3aeb79b72a36df1069c495ec8fd", + "build/prefab/full/mac_x86_64_gui/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/b9/8f/5e01bcc248400251f6a1ac45ca96", + "build/prefab/full/mac_x86_64_gui/release/ballisticacore": "https://files.ballistica.net/cache/ba1/14/73/29cc3cba5c54c6f29700e9392f30", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/e4/60/b20fbb2f034ba8f0d250f287bad5", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/89/14/5529452723c8477b8882b1aecd81", + "build/prefab/full/windows_x86_gui/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/07/08/5abca1559160b52c5cb51366b9e5", + "build/prefab/full/windows_x86_gui/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/25/03/978fc62e0298b7d8322ade9fca18", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/bd/31/d8242ecff0934e358fb8fcb2675a", + "build/prefab/full/windows_x86_server/release/dist/BallisticaCoreHeadless.exe": "https://files.ballistica.net/cache/ba1/7c/7d/6c02413bb4bd8ed58a067c78884f", "build/prefab/lib/linux_arm64_gui/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3b/0c/2f4061ab877d415a1c30e0e736db", "build/prefab/lib/linux_arm64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/3c/5a/2b0714af254c64954ccfe51c70b3", "build/prefab/lib/linux_arm64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1f/ae/c8a885b1a1868b6846b606cdb456", @@ -4031,14 +4031,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/44/df/efb51d1c226eac613d48e2cbf0b8", "build/prefab/lib/mac_x86_64_server/debug/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/1c/f6/357fe951c86c9fc5b1b737cd91ae", "build/prefab/lib/mac_x86_64_server/release/libballisticacore_internal.a": "https://files.ballistica.net/cache/ba1/04/17/e2de0ab5df6b938d828e8662ce6d", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/97/5d/5255b7a90235bc570e71bfaf9f60", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/07/0b/c65c6f6b009633a7cf66ea1c9e08", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/81/09/10f7873ec315479806a9daa6e100", - "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/0f/09/ab4219ed9d6b6b63439a33f38aac", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/fd/ae/d0a4fb20969028322bab2ae2365d", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/0b/cf/4b7529302b842bc75695f93c4172", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/ad/b0/7d2ca14baad3fdb2aac296352570", - "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/68/f8/156cbf9f5cf0fcbae9f12e8b18e8", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/34/77/1986ffe869aca7b8aee6b24ea64b", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/0f/af/5430940c906f3d0c6e0983b9a2b9", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/22/84/02b9109e1449f2acca49f4f9b934", + "build/prefab/lib/windows/Debug_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/2b/22/b32d8e18c6a258929f091a14419d", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.lib": "https://files.ballistica.net/cache/ba1/a2/08/ecf905f1c6ede831e66e8d84c6f6", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreGenericInternal.pdb": "https://files.ballistica.net/cache/ba1/83/d7/01a034b1d9e2f028cb5f964396d3", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.lib": "https://files.ballistica.net/cache/ba1/81/a5/e780126b52d530cce18a64ae65e4", + "build/prefab/lib/windows/Release_Win32/BallisticaCoreHeadlessInternal.pdb": "https://files.ballistica.net/cache/ba1/d8/39/51a851a77b6ce36073e9d190b9bf", "src/ballistica/generated/python_embedded/binding.inc": "https://files.ballistica.net/cache/ba1/7d/3e/229a581cb2454ed856f1d8b564a7", - "src/ballistica/generated/python_embedded/bootstrap.inc": "https://files.ballistica.net/cache/ba1/aa/a5/3ddc86d1789b2bf1d376b7671a3d" + "src/ballistica/generated/python_embedded/bootstrap.inc": "https://files.ballistica.net/cache/ba1/98/12/571b2160d69d42580e8f31fa6a8d" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ca34f7b6..c17744a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -### 1.7.7 (build 20725, api 7, 2022-09-01) +### 1.7.7 (build 20728, api 7, 2022-09-02) - Added `ba.app.meta.load_exported_classes()` for loading classes discovered by the meta subsystem cleanly in a background thread. - Improved logging of missing playlist game types. - Some ba.Lstr functionality can now be used in background threads. @@ -8,6 +8,7 @@ - Added support for the console tool in the new devices section on ballistica.net. - Increased timeouts in net-testing gui and a few other places to be able to better diagnose/handle places with very poor connectivity. - Removed `Platform::SetLastPyCall()` which was just for debugging and which has not been useful in a while. +- Moved some app bootstrapping from the C++ layer to the ba._bootstrap module. ### 1.7.6 (build 20687, api 7, 2022-08-11) - Cleaned up da MetaSubsystem code. diff --git a/assets/.asset_manifest_public.json b/assets/.asset_manifest_public.json index b447e3ff..075dc768 100644 --- a/assets/.asset_manifest_public.json +++ b/assets/.asset_manifest_public.json @@ -17,6 +17,7 @@ "ba_data/python/ba/__pycache__/_assetmanager.cpython-310.opt-1.pyc", "ba_data/python/ba/__pycache__/_asyncio.cpython-310.opt-1.pyc", "ba_data/python/ba/__pycache__/_benchmark.cpython-310.opt-1.pyc", + "ba_data/python/ba/__pycache__/_bootstrap.cpython-310.opt-1.pyc", "ba_data/python/ba/__pycache__/_campaign.cpython-310.opt-1.pyc", "ba_data/python/ba/__pycache__/_cloud.cpython-310.opt-1.pyc", "ba_data/python/ba/__pycache__/_collision.cpython-310.opt-1.pyc", @@ -82,6 +83,7 @@ "ba_data/python/ba/_assetmanager.py", "ba_data/python/ba/_asyncio.py", "ba_data/python/ba/_benchmark.py", + "ba_data/python/ba/_bootstrap.py", "ba_data/python/ba/_campaign.py", "ba_data/python/ba/_cloud.py", "ba_data/python/ba/_collision.py", diff --git a/assets/Makefile b/assets/Makefile index a8898ae9..88f68306 100644 --- a/assets/Makefile +++ b/assets/Makefile @@ -150,6 +150,7 @@ SCRIPT_TARGETS_PY_PUBLIC = \ build/ba_data/python/ba/_assetmanager.py \ build/ba_data/python/ba/_asyncio.py \ build/ba_data/python/ba/_benchmark.py \ + build/ba_data/python/ba/_bootstrap.py \ build/ba_data/python/ba/_campaign.py \ build/ba_data/python/ba/_cloud.py \ build/ba_data/python/ba/_collision.py \ @@ -399,6 +400,7 @@ SCRIPT_TARGETS_PYC_PUBLIC = \ build/ba_data/python/ba/__pycache__/_assetmanager.cpython-310.opt-1.pyc \ build/ba_data/python/ba/__pycache__/_asyncio.cpython-310.opt-1.pyc \ build/ba_data/python/ba/__pycache__/_benchmark.cpython-310.opt-1.pyc \ + build/ba_data/python/ba/__pycache__/_bootstrap.cpython-310.opt-1.pyc \ build/ba_data/python/ba/__pycache__/_campaign.cpython-310.opt-1.pyc \ build/ba_data/python/ba/__pycache__/_cloud.cpython-310.opt-1.pyc \ build/ba_data/python/ba/__pycache__/_collision.cpython-310.opt-1.pyc \ diff --git a/assets/src/ba_data/python/ba/_bootstrap.py b/assets/src/ba_data/python/ba/_bootstrap.py new file mode 100644 index 00000000..b1e2c765 --- /dev/null +++ b/assets/src/ba_data/python/ba/_bootstrap.py @@ -0,0 +1,182 @@ +# Released under the MIT License. See LICENSE for details. +# +"""Bootstrapping.""" +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +import _ba + +if TYPE_CHECKING: + from typing import Any, TextIO, Callable + + +def bootstrap() -> None: + """Run bootstrapping logic. + + This is the very first userland code that runs. + It sets up low level environment bits and creates the app instance. + """ + import os + import sys + import signal + + # The first thing we set up is capturing/redirecting Python + # stdout/stderr so we can at least debug problems on systems where + # native stdout/stderr is not easily accessible (looking at you, Android). + sys.stdout = _Redirect(sys.stdout, _ba.print_stdout) # type: ignore + sys.stderr = _Redirect(sys.stderr, _ba.print_stderr) # type: ignore + + # Give a soft warning if we're being used with a different binary + # version than we expect. + expected_build = 20728 + running_build = _ba.env().get('build_number') + if running_build != expected_build: + print( + f'WARNING: These script files are meant to be used with' + f' Ballistica build {expected_build}.\n' + f' You are running build {running_build}.' + f' This might cause the app to error or misbehave.', + file=sys.stderr) + + # Tell Python to not handle SIGINT itself (it normally generates + # KeyboardInterrupts which make a mess; we want to intercept them + # for simple clean exit). We capture interrupts per-platform in + # the C++ layer. + # Note: I tried creating a handler in Python but it seemed to often have + # a delay of up to a second before getting called. (not a huge deal + # but I'm picky). + signal.signal(signal.SIGINT, signal.SIG_DFL) # Do default handling. + + # ..though it turns out we need to set up our C signal handling AFTER + # we've told Python to disable its own; otherwise (on Mac at least) it + # wipes out our existing C handler. + _ba.setup_sigint() + + # Sanity check: we should always be run in UTF-8 mode. + if sys.flags.utf8_mode != 1: + print('ERROR: Python\'s UTF-8 mode is not set.' + ' This will likely result in errors.') + + debug_build = _ba.env()['debug_build'] + + # We expect dev_mode on in debug builds and off otherwise. + if debug_build != sys.flags.dev_mode: + print(f'WARNING: Mismatch in debug_build {debug_build}' + f' and sys.flags.dev_mode {sys.flags.dev_mode}') + + # In embedded situations (when we're providing our own Python) let's + # also provide our own root certs so ssl works. We can consider overriding + # this in particular embedded cases if we can verify that system certs + # are working. + # (We also allow forcing this via an env var if the user desires) + if (_ba.contains_python_dist() + or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'): + import certifi + + # Let both OpenSSL and requests (if present) know to use this. + os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = ( + certifi.where()) + + # FIXME: I think we should init Python in the main thread, which should + # also avoid these issues. (and also might help us play better with + # Python debuggers?) + + # Gloriously hacky workaround here: + # Our 'main' Python thread is the game thread (not the app's main + # thread) which means it has a small stack compared to the main + # thread (at least on apple). Sadly it turns out this causes the + # debug build of Python to blow its stack immediately when doing + # some big imports. + # Normally we'd just give the game thread the same stack size as + # the main thread and that'd be the end of it. However + # we're using std::threads which it turns out have no way to set + # the stack size (as of fall '19). Grumble. + # + # However python threads *can* take custom stack sizes. + # (and it appears they might use the main thread's by default?..) + # ...so as a workaround in the debug version, we can run problematic + # heavy imports here in another thread and all is well. + # If we ever see stack overflows in our release build we'll have + # to take more drastic measures like switching from std::threads + # to pthreads. + + if debug_build: + + # noinspection PyUnresolvedReferences + def _thread_func() -> None: + # pylint: disable=unused-import + import json + import urllib.request + + testthread = threading.Thread(target=_thread_func) + testthread.start() + testthread.join() + del testthread + + # Clear out the standard quit/exit messages since they don't work for us. + # pylint: disable=c-extension-no-member + if not TYPE_CHECKING: + import __main__ + del __main__.__builtins__.quit + del __main__.__builtins__.exit + + # Now spin up our App instance, store it on both _ba and ba, + # and return it to the C++ layer. + # noinspection PyProtectedMember + from ba._app import App + import ba + + _ba.app = ba.app = App() + + +class _Redirect: + + def __init__(self, original: TextIO, call: Callable[[str], None]) -> None: + self._lock = threading.Lock() + self._linebits: list[str] = [] + self._original = original + self._call = call + self._pending_ship = False + + def write(self, sval: Any) -> None: + """Override standard write call.""" + + self._call(sval) + + # Now do logging: + # Add it to our accumulated line. + # If the message ends in a newline, we can ship it + # immediately as a log entry. Otherwise, schedule a ship + # next cycle (if it hasn't yet at that point) so that we + # can accumulate subsequent prints. + # (so stuff like print('foo', 123, 'bar') will ship as one entry) + with self._lock: + self._linebits.append(sval) + if sval.endswith('\n'): + self._shiplog() + else: + _ba.pushcall(self._shiplog, + from_other_thread=True, + suppress_other_thread_warning=True) + + def _shiplog(self) -> None: + with self._lock: + line = ''.join(self._linebits) + if not line: + return + self._linebits = [] + + # Log messages aren't expected to have trailing newlines. + if line.endswith('\n'): + line = line[:-1] + _ba.log(line, to_stdout=False) + + def flush(self) -> None: + """Flush the file.""" + self._original.flush() + + def isatty(self) -> bool: + """Are we a terminal?""" + return self._original.isatty() diff --git a/src/ballistica/ballistica.cc b/src/ballistica/ballistica.cc index 1c7d2d9f..98908462 100644 --- a/src/ballistica/ballistica.cc +++ b/src/ballistica/ballistica.cc @@ -21,7 +21,7 @@ namespace ballistica { // These are set automatically via script; don't modify them here. -const int kAppBuildNumber = 20725; +const int kAppBuildNumber = 20728; const char* kAppVersion = "1.7.7"; // Our standalone globals. diff --git a/src/meta/bameta/python_embedded/bootstrap.py b/src/meta/bameta/python_embedded/bootstrap.py index 26e409ca..d84cd45d 100644 --- a/src/meta/bameta/python_embedded/bootstrap.py +++ b/src/meta/bameta/python_embedded/bootstrap.py @@ -1,78 +1,20 @@ # Released under the MIT License. See LICENSE for details. +# """Initial ballistica bootstrapping.""" from __future__ import annotations -import os import sys -import signal -import threading from typing import TYPE_CHECKING import _ba if TYPE_CHECKING: - from typing import Any, TextIO, Callable + pass +# All we do here is make our script files accessible and then hand it off +# to them. -class _BAConsoleRedirect: - - def __init__(self, original: TextIO, call: Callable[[str], None]) -> None: - self._lock = threading.Lock() - self._linebits: list[str] = [] - self._original = original - self._call = call - self._pending_ship = False - - def write(self, sval: Any) -> None: - """Override standard write call.""" - - self._call(sval) - - # Now do logging: - # Add it to our accumulated line. - # If the message ends in a newline, we can ship it - # immediately as a log entry. Otherwise, schedule a ship - # next cycle (if it hasn't yet at that point) so that we - # can accumulate subsequent prints. - # (so stuff like print('foo', 123, 'bar') will ship as one entry) - with self._lock: - self._linebits.append(sval) - if sval.endswith('\n'): - self._shiplog() - else: - _ba.pushcall(self._shiplog, - from_other_thread=True, - suppress_other_thread_warning=True) - - def _shiplog(self) -> None: - with self._lock: - line = ''.join(self._linebits) - if not line: - return - self._linebits = [] - - # Log messages aren't expected to have trailing newlines. - if line.endswith('\n'): - line = line[:-1] - _ba.log(line, to_stdout=False) - - def flush(self) -> None: - """Flush the file.""" - self._original.flush() - - def isatty(self) -> bool: - """Are we a terminal?""" - return self._original.isatty() - - -# The very first thing we set up is redirecting Python stdout/stderr so -# we can at least debug problems on systems where native stdout/stderr -# is not easily accessible (looking at you, Android). -sys.stdout = _BAConsoleRedirect(sys.stdout, _ba.print_stdout) # type: ignore -sys.stderr = _BAConsoleRedirect(sys.stderr, _ba.print_stderr) # type: ignore - -# Now get access to our various script files. # Let's lookup mods first (so users can do whatever they want). # and then our bundled scripts last (don't want bundled site-package # stuff overwriting system versions) @@ -80,93 +22,9 @@ sys.path.insert(0, _ba.env()['python_directory_user']) sys.path.append(_ba.env()['python_directory_app']) sys.path.append(_ba.env()['python_directory_app_site']) -# Tell Python to not handle SIGINT itself (it normally generates -# KeyboardInterrupts which make a mess; we want to intercept them -# for simple clean exit). We capture interrupts per-platform in -# the C++ layer. -# Note: I tried creating a handler in Python but it seemed to often have -# a delay of up to a second before getting called. (not a huge deal -# but I'm picky). -signal.signal(signal.SIGINT, signal.SIG_DFL) # Do default handling. - -# ..though it turns out we need to set up our C signal handling AFTER -# we've told Python to disable its own; otherwise (on Mac at least) it -# wipes out our existing C handler. -_ba.setup_sigint() - -# Sanity check: we should always be run in UTF-8 mode. -if sys.flags.utf8_mode != 1: - print('ERROR: Python\'s UTF-8 mode is not set.' - ' This will likely result in errors.') - -debug_build = _ba.env()['debug_build'] - -# We expect dev_mode on in debug builds and off otherwise. -if debug_build != sys.flags.dev_mode: - print(f'WARNING: Mismatch in debug_build {debug_build}' - f' and sys.flags.dev_mode {sys.flags.dev_mode}') - -# In embedded situations (when we're providing our own Python) let's -# also provide our own root certs so ssl works. We can consider overriding -# this in particular embedded cases if we can verify that system certs -# are working. -# (We also allow forcing this via an env var if the user desires) -# pylint: disable=wrong-import-position -if (_ba.contains_python_dist() - or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'): - import certifi - - # Let both OpenSSL and requests (if present) know to use this. - os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = ( - certifi.where()) - -# FIXME: I think we should init Python in the main thread, which should -# also avoid these issues. (and also might help us play better with -# Python debuggers?) - -# Gloriously hacky workaround here: -# Our 'main' Python thread is the game thread (not the app's main -# thread) which means it has a small stack compared to the main -# thread (at least on apple). Sadly it turns out this causes the -# debug build of Python to blow its stack immediately when doing -# some big imports. -# Normally we'd just give the game thread the same stack size as -# the main thread and that'd be the end of it. However -# we're using std::threads which it turns out have no way to set -# the stack size (as of fall '19). Grumble. -# -# However python threads *can* take custom stack sizes. -# (and it appears they might use the main thread's by default?..) -# ...so as a workaround in the debug version, we can run problematic -# heavy imports here in another thread and all is well. -# If we ever see stack overflows in our release build we'll have -# to take more drastic measures like switching from std::threads -# to pthreads. - -if debug_build: - - # noinspection PyUnresolvedReferences - def _thread_func() -> None: - # pylint: disable=unused-import - import json - import urllib.request - - testthread = threading.Thread(target=_thread_func) - testthread.start() - testthread.join() - del testthread - -# Clear out the standard quit/exit messages since they don't work for us. -# pylint: disable=c-extension-no-member -if not TYPE_CHECKING: - import __main__ - del __main__.__builtins__.quit - del __main__.__builtins__.exit - -# Now spin up our App instance, store it on both _ba and ba, -# and return it to the C++ layer. +# The import is down here since it won't work until we muck with paths. # noinspection PyProtectedMember -from ba._app import App -import ba +# pylint: disable=wrong-import-position +from ba._bootstrap import bootstrap -_ba.app = ba.app = App() +bootstrap() diff --git a/tools/efro/log.py b/tools/efro/log.py index a2e78637..af647f56 100644 --- a/tools/efro/log.py +++ b/tools/efro/log.py @@ -9,22 +9,24 @@ import logging import datetime import threading from enum import Enum -from typing import TYPE_CHECKING, Annotated from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated from efro.util import utc_now -from efro.dataclassio import ioprepped, IOAttrs, dataclass_to_json from efro.terminal import TerminalColor +from efro.dataclassio import ioprepped, IOAttrs, dataclass_to_json if TYPE_CHECKING: - from typing import Any, Callable from pathlib import Path + from typing import Any, Callable class LogLevel(Enum): """Severity level for a log entry. - Note: these are numeric values so they can be compared in severity. + These enums have numeric values so they can be compared in severity. + Note that these values are not currently interchangeable with the + logging.ERROR, logging.DEBUG, etc. values. """ DEBUG = 0 INFO = 1 @@ -36,7 +38,7 @@ class LogLevel(Enum): @ioprepped @dataclass class LogEntry: - """Structured log entry.""" + """Single logged message.""" name: Annotated[str, IOAttrs('n', soft_default='root', store_default=False)] message: Annotated[str, IOAttrs('m')] @@ -44,7 +46,7 @@ class LogEntry: time: Annotated[datetime.datetime, IOAttrs('t')] -class StructuredLogHandler(logging.StreamHandler): +class LogHandler(logging.Handler): """Fancy-pants handler for logging output. Writes logs to disk in structured json format and echoes them @@ -64,9 +66,14 @@ class StructuredLogHandler(logging.StreamHandler): self._suppress_non_root_debug = suppress_non_root_debug def emit(self, record: logging.LogRecord) -> None: + + # Special case - filter out this common extra-chatty category. + # TODO - should use a standard logging.Filter for this. if (self._suppress_non_root_debug and record.name != 'root' and record.levelname == 'DEBUG'): return + + # Bake down all log formatting into a simple string. msg = self.format(record) # Translate Python log levels to our own. @@ -87,7 +94,8 @@ class StructuredLogHandler(logging.StreamHandler): for call in self._callbacks: call(entry) - # Also route log entries to the echo file (generally stdout) + # Also route log entries to the echo file (generally stdout/stderr) + # with pretty colors. if self._echofile is not None: cbegin: str cend: str @@ -123,7 +131,6 @@ class StructuredLogHandler(logging.StreamHandler): level=level, time=utc_now()) - # Inform anyone who wants to know about this log's level. for call in self._callbacks: call(entry) @@ -140,8 +147,8 @@ class StructuredLogHandler(logging.StreamHandler): class LogRedirect: """A file-like object for redirecting stdout/stderr to our log.""" - def __init__(self, name: str, orig_out: Any, - log_handler: StructuredLogHandler, log_level: LogLevel): + def __init__(self, name: str, orig_out: Any, log_handler: LogHandler, + log_level: LogLevel): self._name = name self._orig_out = orig_out self._log_handler = log_handler @@ -196,10 +203,9 @@ class LogRedirect: self._chunk = '' -def setup_logging( - log_path: str | Path | None, - level: LogLevel, - suppress_non_root_debug: bool = False) -> StructuredLogHandler: +def setup_logging(log_path: str | Path | None, + level: LogLevel, + suppress_non_root_debug: bool = False) -> LogHandler: """Set up our logging environment. Returns the custom handler which can be used to fetch information @@ -216,7 +222,7 @@ def setup_logging( # Wire logger output to go to a structured log file. # Also echo it to stderr IF we're running in a terminal. - loghandler = StructuredLogHandler( + loghandler = LogHandler( log_path, echofile=sys.stderr if sys.stderr.isatty() else None, suppress_non_root_debug=suppress_non_root_debug)