diff --git a/.efrocachemap b/.efrocachemap index b5174a7b..5f73998c 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4068,26 +4068,26 @@ "build/assets/windows/Win32/ucrtbased.dll": "2def5335207d41b21b9823f6805997f1", "build/assets/windows/Win32/vc_redist.x86.exe": "b08a55e2e77623fe657bea24f223a3ae", "build/assets/windows/Win32/vcruntime140d.dll": "865b2af4d1e26a1a8073c89acb06e599", - "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "6c5410dd5457a13813493d7ea2d5562b", - "build/prefab/full/linux_arm64_gui/release/ballisticakit": "e69bb296bf91b09a0e231dfa8ed0b7e2", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "86c5d40f241ddaa93388ea0f8a80082c", - "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "5706bd9259808872b6f72782fc55b455", - "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "e7783c8332d719d1c3f267f373db5475", - "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "7b5a0ecb206e6e99bc226b450947f574", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "3ec06df77ccbd6692f70627590839015", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "6d0aaabc7b99cc3ede5e929243978996", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "da63fc9c35a40a5dd00b635a25f8c6f3", - "build/prefab/full/mac_arm64_gui/release/ballisticakit": "3ccba6e3f58feaa072e586a931e4fd66", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "731a2915a6171ef062731c955ce08e88", - "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "98a5504c530c7e7332c7a36e74768cda", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "fc296240aea46b34c2cbc025469c0b35", - "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "2de6690d43b4e0ca1f025db203e61105", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "907aa9778e4d5de273c8ab0371693973", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "f5931ba9f402cb06ae3fac1435fe373b", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "80044ca1925fda3eff45413fd95dfcd5", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "2ce86b7c958c9e7c020aa45bac5c6add", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "179578c9032fce4a42c525b1cc5f076f", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "09fa90d666a704c13090e77f840dbd9f", + "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "e90f3bcd2f94dd910220e15a42550966", + "build/prefab/full/linux_arm64_gui/release/ballisticakit": "969a2f87c4e7863dac65b26269101099", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "5514b6c86406ddddf5b651f8f3f55fe7", + "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "1aae03c6da0ba129356a2cad35a7f662", + "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "927adc17b55984179e79c513dd02f255", + "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "c14f956a37c6c4b854528431f69d97a3", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "fd57912de1a8c12d23b9f19729b963d8", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "3e21f3f57cd594dcd4e4b4ed14546cd8", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "c4db68443f31c70b27511c89c0042ff6", + "build/prefab/full/mac_arm64_gui/release/ballisticakit": "f3d980a36605c34dca3b12184326163a", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "fb034ad5af059f2b19d42fff4c465ece", + "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "badf5f08cbcb64742114ca048c6b7727", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "9c074fac35c13f29f55a1f913307aba3", + "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "8405f44936e40e5b03f43be9767dae08", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "555a3eac0feed1ae6f86326082019d2c", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "82b73ef479f177a3f96d3fc521fd986b", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "cb138fb8ec019892df1d29bfa86e7b46", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "5710f9d700298af5fe062af043aa9a61", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "6b2b5e81a81e82140b5637ae8d76810b", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "999459515be3dcd49b3b4268298eab6d", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "85ba4e81a1f7ae2cff4b1355eb49904f", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "498921f7eb2afd327d4b900cb70e31f9", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "85ba4e81a1f7ae2cff4b1355eb49904f", @@ -4104,14 +4104,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "0ab638b6602610bdaf432e3cc2464080", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "92394eb19387c363471ce134ac9e6a1b", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "0ab638b6602610bdaf432e3cc2464080", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "829a4527bff626c891712fc68ecd0b2f", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "192469453f6c0e5fb54ed2286c44319f", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "5f3e5de6cc2b29d2183d4ba725a03dc4", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "d0d0e606a7b5e8ee5f463ae2589f8054", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "6e6e70046ff50d7080395f9b00fcb73f", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "77b5b5d744c23471a640f85e611f8ab0", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "d84545b72fd39dbc4cad3956141e725f", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "646f9d970cdb97c002ac5deded02f934", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "7e7dd154962326d6c3b6089774da2fd3", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "887664acc51488a1e5bda23297e2ede0", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "4057bf7feb1d96987f571d52c482fd9d", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "1f3d4445f1854bbfed32d164e83c01cd", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "6273ffb3c4160d960a0be12dee00b78a", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "38c0a19337162817680cc0ff34279618", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "ebc7699242e51224623f5dd8f6685ead", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "35008f5db3de1717c8008762ea12e964", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "f8cd3af311ac63147882590123b78318", "src/ballistica/base/mgen/pyembed/binding_base.inc": "eeddad968b176000e31c65be6206a2bc", diff --git a/CHANGELOG.md b/CHANGELOG.md index a9ee45d4..8c448db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,11 @@ -### 1.7.26 (build 21213, api 8, 2023-08-17) +### 1.7.26 (build 21215, api 8, 2023-08-18) - Various general improvements to the pcommand (project command) system. - Modules containing pcommand functions are now named with an 's' - so `pcommands.py` instead of `pcommand.py`. `pcommand.py` in efrotools is now solely related to the functioning of the pcommand system. +- Switched various build stuff to use `pcommandbatch` instead of regular + pcommand. In some cases this can speed things up by 5x or so. ### 1.7.25 (build 21211, api 8, 2023-08-03) diff --git a/Makefile b/Makefile index 61e08cc6..1b737de0 100644 --- a/Makefile +++ b/Makefile @@ -29,17 +29,16 @@ help: @$(PCOMMAND) makefile_target_list Makefile # Set env-var BA_ENABLE_COMPILE_COMMANDS_DB=1 to enable creating/updating a -# cmake compile-commands database for use with irony for emacs (and possibly -# other tools). +# cmake compile-commands database for use with things like clangd. ifeq ($(BA_ENABLE_COMPILE_COMMANDS_DB),1) PREREQ_COMPILE_COMMANDS_DB = .cache/compile_commands_db/compile_commands.json endif +# pcommandbatch can be much faster when running hundreds or thousands of +# commands, but has some downsides and limitations compared to regular +# pcommand. See tools/efrotools/pcommandbatch.py for more info on when to use +# which. PCOMMAND = tools/pcommand -# Support for running pcommands in 'batch' mode in which a simple local server -# handles command requests from a lightweight client binary. This largely -# takes Python's startup time out of the equation, which can add up when -# running lots of small pcommands in cases such as asset builds. PCOMMANDBATCHBIN = .cache/pcommandbatch/pcommandbatch ifeq ($(BA_PCOMMANDBATCH_DISABLE),1) PCOMMANDBATCH = $(PCOMMAND) @@ -49,8 +48,8 @@ endif # Prereq targets that should be safe to run anytime; even if project-files # are out of date. -PREREQS_SAFE = .cache/checkenv $(PCOMMANDBATCH) .dir-locals.el .mypy.ini \ - .pyrightconfig.json .pycheckers .pylintrc .style.yapf .clang-format \ +PREREQS_SAFE = .cache/checkenv $(PCOMMANDBATCHBIN) .dir-locals.el .mypy.ini \ + .pyrightconfig.json .pycheckers .pylintrc .style.yapf .clang-format \ ballisticakit-cmake/.clang-format .editorconfig # Prereq targets that may break if the project needs updating should go here. @@ -203,11 +202,11 @@ pcommandbatch_speed_test: prereqs # Assemble & run a gui debug build for this platform. prefab-gui-debug: prefab-gui-debug-build - $($(shell $(PCOMMANDBATCH) prefab_run_var gui-debug)) + $($(shell $(PCOMMAND) prefab_run_var gui-debug)) # Assemble & run a gui release build for this platform. prefab-gui-release: prefab-gui-release-build - $($(shell $(PCOMMANDBATCH) prefab_run_var gui-release)) + $($(shell $(PCOMMAND) prefab_run_var gui-release)) # Assemble a debug build for this platform. prefab-gui-debug-build: @@ -219,11 +218,11 @@ prefab-gui-release-build: # Assemble & run a server debug build for this platform. prefab-server-debug: prefab-server-debug-build - $($(shell $(PCOMMANDBATCH) prefab_run_var server-debug)) + $($(shell $(PCOMMAND) prefab_run_var server-debug)) # Assemble & run a server release build for this platform. prefab-server-release: prefab-server-release-build - $($(shell $(PCOMMANDBATCH) prefab_run_var server-release)) + $($(shell $(PCOMMAND) prefab_run_var server-release)) # Assemble a server debug build for this platform. prefab-server-debug-build: diff --git a/src/assets/Makefile b/src/assets/Makefile index c097d56c..dfc5c99a 100644 --- a/src/assets/Makefile +++ b/src/assets/Makefile @@ -23,11 +23,11 @@ BUILD_DIR = $(PROJ_DIR)/build/assets PCOMMAND = $(TOOLS_DIR)/pcommand -# Support for running pcommands in 'batch' mode in which a simple local server -# handles command requests from a lightweight client binary. This largely -# takes Python's startup time out of the equation, which can add up when -# running lots of small pcommands in cases such as asset builds. -PCOMMANDBATCHBIN := $(PROJ_DIR)/.cache/pcommandbatch/pcommandbatch +# pcommandbatch can be much faster when running hundreds or thousands of +# commands, but has some downsides and limitations compared to regular +# pcommand. See tools/efrotools/pcommandbatch.py for more info on when to use +# which. +PCOMMANDBATCHBIN = $(PROJ_DIR)/.cache/pcommandbatch/pcommandbatch ifeq ($(BA_PCOMMANDBATCH_DISABLE),1) PCOMMANDBATCH = $(TOOLS_DIR)/pcommand PCOMMANDBATCHFROMROOT = tools/pcommand diff --git a/src/assets/ba_data/python/baenv.py b/src/assets/ba_data/python/baenv.py index 04bf2e92..d020e38e 100644 --- a/src/assets/ba_data/python/baenv.py +++ b/src/assets/ba_data/python/baenv.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: # Build number and version of the ballistica binary we expect to be # using. -TARGET_BALLISTICA_BUILD = 21213 +TARGET_BALLISTICA_BUILD = 21215 TARGET_BALLISTICA_VERSION = '1.7.26' diff --git a/src/ballistica/shared/ballistica.cc b/src/ballistica/shared/ballistica.cc index 83e3476a..b9d00891 100644 --- a/src/ballistica/shared/ballistica.cc +++ b/src/ballistica/shared/ballistica.cc @@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int { namespace ballistica { // These are set automatically via script; don't modify them here. -const int kEngineBuildNumber = 21213; +const int kEngineBuildNumber = 21215; const char* kEngineVersion = "1.7.26"; #if BA_MONOLITHIC_BUILD diff --git a/tools/batools/pcommands.py b/tools/batools/pcommands.py index 32e9bb24..c7f1e267 100644 --- a/tools/batools/pcommands.py +++ b/tools/batools/pcommands.py @@ -602,11 +602,10 @@ def ensure_prefab_platform() -> None: import batools.build from efro.error import CleanError - pcommand.disallow_in_batch() - - if len(sys.argv) != 3: + args = pcommand.get_args() + if len(args) != 1: raise CleanError('Expected 1 platform name arg.') - needed = sys.argv[2] + needed = args[0] current = batools.build.get_current_prefab_platform() if current != needed: raise CleanError( diff --git a/tools/batools/pcommands2.py b/tools/batools/pcommands2.py index 69236812..7f6b0141 100644 --- a/tools/batools/pcommands2.py +++ b/tools/batools/pcommands2.py @@ -420,6 +420,8 @@ def wsl_build_check_win_drive() -> None: import textwrap from efro.error import CleanError + # We use env vars to influence our behavior and thus can't support + # batch. pcommand.disallow_in_batch() if ( diff --git a/tools/efrotools/pcommandbatch.py b/tools/efrotools/pcommandbatch.py index b716a561..f703ac46 100644 --- a/tools/efrotools/pcommandbatch.py +++ b/tools/efrotools/pcommandbatch.py @@ -12,28 +12,31 @@ hundreds or thousands of pcommands are being run. To help fight that problem, pcommandbatch introduces a way to run pcommands by submitting requests to temporary local server daemons. -This allows individual pcommand calls to go through a very lightweight -client binary that simply forwards the command to a running server. -This cuts minimal client runtime down greatly. Building and managing -the server and client are handled automatically, and systems which are -unable to compile a client binary can fall back to using vanilla -pcommand in those cases. +This allows individual pcommand calls to go through a lightweight client +binary that simply forwards the command to a running server. This cuts +minimum pcommand runtimes down greatly. Building and managing the server +and client are handled automatically, and systems which are unable to +compile a client binary can fall back to using vanilla pcommand in those +cases. -A few considerations must be made when using pcommands with batch mode. -By default, all existing pcommands have been fitted with a -disallow_in_batch() call which triggers an error under batch mode. -These calls should be removed if/when each call is updated to work -cleanly in batch mode. Requirements for batch-friendly pcommands follow: +A few considerations must be made when using pcommandbatch. By default, +all existing pcommands have been fitted with a disallow_in_batch() call +which triggers an error under batch mode. These calls should be removed +if/when each call is updated to work cleanly in batch mode. Guidelines +for batch-friendly pcommands follow: - Batch mode runs parallel pcommands in different background threads - and may run thousands of commands throughout the duration of the - server process. Batch-friendly pcommands must behave reasonably in - such an environment. + and may process thousands of commands in a single process. + Batch-friendly pcommands must behave reasonably in such an environment. - Batch-enabled pcommands must not call os.chdir() or sys.exit() or anything else having global effects. This should be self-explanatory considering the shared server model in use. +- Batch-enabled pcommands must not use environment-variables to + influence their behavior. In batch mode this would unintuitively use + the environment of the server and not of the client. + - Batch-enabled pcommands should not look at sys.argv. They should instead use pcommand.get_args(). Be aware that this value does not include the first two values from sys.argv (executable path and