From 7643ebce10754f9994a8de3f1d541184e2fa0e75 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 18 Aug 2023 19:23:09 -0700 Subject: [PATCH] polishing up pcommandbatch on my flight home --- .efrocachemap | 32 +- .idea/dictionaries/ericf.xml | 3 + Makefile | 277 +++++++++--------- .../.idea/dictionaries/ericf.xml | 3 + src/tools/pcommandbatch/pcommandbatch.c | 198 +++++++++---- tools/batools/pcommands.py | 28 +- tools/efro/error.py | 19 +- tools/efrotools/pcommand.py | 184 +++++++----- tools/efrotools/pcommandbatch.py | 96 +++--- tools/efrotools/pcommands.py | 22 +- tools/pcommand | 7 + 11 files changed, 510 insertions(+), 359 deletions(-) diff --git a/.efrocachemap b/.efrocachemap index 152bea3c..b5174a7b 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4076,18 +4076,18 @@ "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "7b5a0ecb206e6e99bc226b450947f574", "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "3ec06df77ccbd6692f70627590839015", "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "6d0aaabc7b99cc3ede5e929243978996", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "6d1fe60a2a3ec603957cbc7423eae8bc", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "da63fc9c35a40a5dd00b635a25f8c6f3", "build/prefab/full/mac_arm64_gui/release/ballisticakit": "3ccba6e3f58feaa072e586a931e4fd66", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "83d14fc9336afa5dc41ec366b137203e", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "731a2915a6171ef062731c955ce08e88", "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "98a5504c530c7e7332c7a36e74768cda", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "d0cfd400a198391887c9254f80acef48", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "fc296240aea46b34c2cbc025469c0b35", "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "2de6690d43b4e0ca1f025db203e61105", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "a6f8edb46a3651a609575d07b0df2c7f", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "907aa9778e4d5de273c8ab0371693973", "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "f5931ba9f402cb06ae3fac1435fe373b", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "df37f9ec11f003f192b3f58c071393f0", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "f7b674637dc6e633b37118bd19eb3f6a", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "04900388d6ee576fe6ca7004f03754c5", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "48c7804a59abb43e7e7c7c54fce11dc0", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "80044ca1925fda3eff45413fd95dfcd5", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "2ce86b7c958c9e7c020aa45bac5c6add", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "179578c9032fce4a42c525b1cc5f076f", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "09fa90d666a704c13090e77f840dbd9f", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "85ba4e81a1f7ae2cff4b1355eb49904f", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "498921f7eb2afd327d4b900cb70e31f9", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "85ba4e81a1f7ae2cff4b1355eb49904f", @@ -4104,14 +4104,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "0ab638b6602610bdaf432e3cc2464080", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "92394eb19387c363471ce134ac9e6a1b", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "0ab638b6602610bdaf432e3cc2464080", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "1d8634f8423145911a8db28caffa7022", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "12191e80087d08e5def19f9bbdc84b9e", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "4abf18c2871cf29ba5c2ed466cd60ce7", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "d267014b78a59c519af3c9c4769372c3", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "78f112cec4040a2c478793ee93936c8d", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "9b5b639759109353e848a67eb092ba8d", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "83ea8101969c9c030c97f9e529da6ae8", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "54c7978cfc2a1870a8b23aa18b913ebc", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "829a4527bff626c891712fc68ecd0b2f", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "192469453f6c0e5fb54ed2286c44319f", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "5f3e5de6cc2b29d2183d4ba725a03dc4", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "d0d0e606a7b5e8ee5f463ae2589f8054", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "6e6e70046ff50d7080395f9b00fcb73f", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "77b5b5d744c23471a640f85e611f8ab0", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "d84545b72fd39dbc4cad3956141e725f", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "646f9d970cdb97c002ac5deded02f934", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "f8cd3af311ac63147882590123b78318", "src/ballistica/base/mgen/pyembed/binding_base.inc": "eeddad968b176000e31c65be6206a2bc", diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index ab6e0ed0..a956c19e 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -515,6 +515,8 @@ clearsign clientid clientlist + clientprint + clientprints clienttobasn clionbin clioncode @@ -532,6 +534,7 @@ clrhdr clrnames clrred + clrtp cmakelist cmakelists cmakemodular diff --git a/Makefile b/Makefile index 5c855502..61e08cc6 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ # List targets in this Makefile and basic descriptions for them. help: - @tools/pcommand makefile_target_list Makefile + @$(PCOMMAND) makefile_target_list Makefile # Set env-var BA_ENABLE_COMPILE_COMMANDS_DB=1 to enable creating/updating a # cmake compile-commands database for use with irony for emacs (and possibly @@ -35,13 +35,14 @@ ifeq ($(BA_ENABLE_COMPILE_COMMANDS_DB),1) PREREQ_COMPILE_COMMANDS_DB = .cache/compile_commands_db/compile_commands.json endif +PCOMMAND = tools/pcommand # Support for running pcommands in 'batch' mode in which a simple local server # handles command requests from a lightweight client binary. This largely # takes Python's startup time out of the equation, which can add up when # running lots of small pcommands in cases such as asset builds. -PCOMMANDBATCHBIN := .cache/pcommandbatch/pcommandbatch +PCOMMANDBATCHBIN = .cache/pcommandbatch/pcommandbatch ifeq ($(BA_PCOMMANDBATCH_DISABLE),1) - PCOMMANDBATCH = tools/pcommand + PCOMMANDBATCH = $(PCOMMAND) else PCOMMANDBATCH = $(PCOMMANDBATCHBIN) endif @@ -71,52 +72,52 @@ prereqs-clean: # Build all assets for all platforms. assets: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) # Build assets required for cmake builds (linux, mac). assets-cmake: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) cmake # Build only script assets for cmake builds (linux, mac). assets-cmake-scripts: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) scripts-cmake # Build assets required for server builds. assets-server: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) server # Build assets required for WINDOWS_PLATFORM windows builds. assets-windows: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) win-$(WINDOWS_PLATFORM) # Build assets required for Win32 windows builds. assets-windows-Win32: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) win-Win32 # Build assets required for x64 windows builds. assets-windows-x64: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) win-x64 # Build assets required for mac xcode builds assets-mac: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) mac # Build assets required for ios. assets-ios: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) ios # Build assets required for android. assets-android: prereqs meta - @tools/pcommand lazybuild assets_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild assets_src $(LAZYBUILDDIR)/$@ \ cd src/assets \&\& $(MAKE) -j$(CPUS) android # Clean all assets. @@ -126,7 +127,7 @@ assets-clean: # Build resources. resources: prereqs meta - @tools/pcommand lazybuild resources_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild resources_src $(LAZYBUILDDIR)/$@ \ cd src/resources \&\& $(MAKE) -j$(CPUS) # Clean resources. @@ -138,7 +139,7 @@ resources-clean: # Meta builds can affect sources used by asset builds, resource builds, and # compiles, so it should be listed as a dependency of any of those. meta: prereqs - @tools/pcommand lazybuild meta_src $(LAZYBUILDDIR)/$@ \ + @$(PCOMMAND) lazybuild meta_src $(LAZYBUILDDIR)/$@ \ cd src/meta \&\& $(MAKE) -j$(CPUS) # Clean our generated sources. @@ -164,8 +165,8 @@ clean-list: # it should not be built in parallel with other targets. # See py_check_prereqs target for more info. dummymodules: prereqs meta - @tools/pcommand lazybuild dummymodules_src $(LAZYBUILDDIR)/$@ \ - rm -rf build/dummymodules \&\& ./tools/pcommand gen_dummy_modules + @$(PCOMMAND) lazybuild dummymodules_src $(LAZYBUILDDIR)/$@ \ + rm -rf build/dummymodules \&\& $(PCOMMAND) gen_dummy_modules dummymodules-clean: rm -f $(LAZYBUILDDIR)/dummymodules @@ -179,10 +180,10 @@ docs: $(MAKE) docs-pdoc docs-pdoc: - @tools/pcommand gen_docs_pdoc + @$(PCOMMAND) gen_docs_pdoc pcommandbatch_speed_test: prereqs - @tools/pcommand pcommandbatch_speed_test $(PCOMMANDBATCH) + @$(PCOMMAND) pcommandbatch_speed_test $(PCOMMANDBATCH) # Tell make which of these targets don't represent files. .PHONY: help prereqs prereqs-pre-update prereqs-clean assets assets-cmake \ @@ -202,35 +203,35 @@ pcommandbatch_speed_test: prereqs # Assemble & run a gui debug build for this platform. prefab-gui-debug: prefab-gui-debug-build - $($(shell tools/pcommand prefab_run_var gui-debug)) + $($(shell $(PCOMMANDBATCH) prefab_run_var gui-debug)) # Assemble & run a gui release build for this platform. prefab-gui-release: prefab-gui-release-build - $($(shell tools/pcommand prefab_run_var gui-release)) + $($(shell $(PCOMMANDBATCH) prefab_run_var gui-release)) # Assemble a debug build for this platform. prefab-gui-debug-build: - @tools/pcommand make_prefab gui-debug + @$(PCOMMAND) make_prefab gui-debug # Assemble a release build for this platform. prefab-gui-release-build: - @tools/pcommand make_prefab gui-release + @$(PCOMMAND) make_prefab gui-release # Assemble & run a server debug build for this platform. prefab-server-debug: prefab-server-debug-build - $($(shell tools/pcommand prefab_run_var server-debug)) + $($(shell $(PCOMMANDBATCH) prefab_run_var server-debug)) # Assemble & run a server release build for this platform. prefab-server-release: prefab-server-release-build - $($(shell tools/pcommand prefab_run_var server-release)) + $($(shell $(PCOMMANDBATCH) prefab_run_var server-release)) # Assemble a server debug build for this platform. prefab-server-debug-build: - @tools/pcommand make_prefab server-debug + @$(PCOMMAND) make_prefab server-debug # Assemble a server release build for this platform. prefab-server-release-build: - @tools/pcommand make_prefab server-release + @$(PCOMMAND) make_prefab server-release # Clean all prefab builds. prefab-clean: @@ -250,11 +251,11 @@ RUN_PREFAB_MAC_ARM64_GUI_DEBUG = cd build/prefab/full/mac_arm64_gui/debug \ && ./ballisticakit prefab-mac-x86-64-gui-debug: prefab-mac-x86-64-gui-debug-build - @tools/pcommand ensure_prefab_platform mac_x86_64 + @$(PCOMMAND) ensure_prefab_platform mac_x86_64 @$(RUN_PREFAB_MAC_X86_64_GUI_DEBUG) prefab-mac-arm64-gui-debug: prefab-mac-arm64-gui-debug-build - @tools/pcommand ensure_prefab_platform mac_arm64 + @$(PCOMMAND) ensure_prefab_platform mac_arm64 @$(RUN_PREFAB_MAC_ARM64_GUI_DEBUG) prefab-mac-x86-64-gui-debug-build: prereqs assets-cmake \ @@ -280,11 +281,11 @@ RUN_PREFAB_MAC_ARM64_GUI_RELEASE = cd build/prefab/full/mac_arm64_gui/release \ && ./ballisticakit prefab-mac-x86-64-gui-release: prefab-mac-x86-64-gui-release-build - @tools/pcommand ensure_prefab_platform mac_x86_64 + @$(PCOMMAND) ensure_prefab_platform mac_x86_64 @$(RUN_PREFAB_MAC_X86_64_GUI_RELEASE) prefab-mac-arm64-gui-release: prefab-mac-arm64-gui_release-build - @tools/pcommand ensure_prefab_platform mac_arm64 + @$(PCOMMAND) ensure_prefab_platform mac_arm64 @$(RUN_PREFAB_MAC_ARM64_GUI_RELEASE) prefab-mac-x86-64-gui-release-build: prereqs assets-cmake \ @@ -310,11 +311,11 @@ RUN_PREFAB_MAC_ARM64_SERVER_DEBUG = cd \ build/prefab/full/mac_arm64_server/debug && ./ballisticakit_server prefab-mac-x86-64-server-debug: prefab-mac-x86-64-server-debug-build - @tools/pcommand ensure_prefab_platform mac_x86_64 + @$(PCOMMAND) ensure_prefab_platform mac_x86_64 @$(RUN_PREFAB_MAC_X86_64_SERVER_DEBUG) prefab-mac-arm64-server-debug: prefab-mac-arm64-server-debug-build - @tools/pcommand ensure_prefab_platform mac_arm64 + @$(PCOMMAND) ensure_prefab_platform mac_arm64 @$(RUN_PREFAB_MAC_ARM64_SERVER_DEBUG) prefab-mac-x86-64-server-debug-build: prereqs assets-server \ @@ -340,11 +341,11 @@ RUN_PREFAB_MAC_ARM64_SERVER_RELEASE = cd \ build/prefab/full/mac_arm64_server/release && ./ballisticakit_server prefab-mac-x86-64-server-release: prefab-mac-x86-64-server-release-build - @tools/pcommand ensure_prefab_platform mac_x86_64 + @$(PCOMMAND) ensure_prefab_platform mac_x86_64 @$(RUN_PREFAB_MAC_X86_64_SERVER_RELEASE) prefab-mac-arm64-server-release: prefab-mac-arm64-server-release-build - @tools/pcommand ensure_prefab_platform mac_arm64 + @$(PCOMMAND) ensure_prefab_platform mac_arm64 @$(RUN_PREFAB_MAC_ARM64_SERVER_RELEASE) prefab-mac-x86-64-server-release-build: prereqs assets-server \ @@ -372,11 +373,11 @@ RUN_PREFAB_LINUX_ARM64_GUI_DEBUG = cd \ build/prefab/full/linux_arm64_gui/debug && ./ballisticakit prefab-linux-x86-64-gui-debug: prefab-linux-x86-64-gui-debug-build - @tools/pcommand ensure_prefab_platform linux_x86_64 + @$(PCOMMAND) ensure_prefab_platform linux_x86_64 @$(RUN_PREFAB_LINUX_X86_64_GUI_DEBUG) prefab-linux-arm64-gui-debug: prefab-linux-arm64-gui-debug-build - @tools/pcommand ensure_prefab_platform linux_arm64 + @$(PCOMMAND) ensure_prefab_platform linux_arm64 @$(RUN_PREFAB_LINUX_ARM64_GUI_DEBUG) prefab-linux-x86-64-gui-debug-build: prereqs assets-cmake \ @@ -402,11 +403,11 @@ RUN_PREFAB_LINUX_ARM64_GUI_RELEASE = cd \ build/prefab/full/linux_arm64_gui/release && ./ballisticakit prefab-linux-x86-64-gui-release: prefab-linux-x86-64-gui-release-build - @tools/pcommand ensure_prefab_platform linux_x86_64 + @$(PCOMMAND) ensure_prefab_platform linux_x86_64 @$(RUN_PREFAB_LINUX_X86_64_GUI_RELEASE) prefab-linux-arm64-gui-release: prefab-linux-arm64-gui-release-build - @tools/pcommand ensure_prefab_platform linux_arm64 + @$(PCOMMAND) ensure_prefab_platform linux_arm64 @$(RUN_PREFAB_LINUX_ARM64_GUI_RELEASE) prefab-linux-x86-64-gui-release-build: prereqs assets-cmake \ @@ -432,11 +433,11 @@ RUN_PREFAB_LINUX_ARM64_SERVER_DEBUG = cd \ build/prefab/full/linux_arm64_server/debug && ./ballisticakit_server prefab-linux-x86-64-server-debug: prefab-linux-x86-64-server-debug-build - @tools/pcommand ensure_prefab_platform linux_x86_64 + @$(PCOMMAND) ensure_prefab_platform linux_x86_64 @$(RUN_PREFAB_LINUX_X86_64_SERVER_DEBUG) prefab-linux-arm64-server-debug: prefab-linux-arm64-server-debug-build - @tools/pcommand ensure_prefab_platform linux_arm64 + @$(PCOMMAND) ensure_prefab_platform linux_arm64 @$(RUN_PREFAB_LINUX_ARM64_SERVER_DEBUG) prefab-linux-x86-64-server-debug-build: prereqs assets-server \ @@ -464,11 +465,11 @@ RUN_PREFAB_LINUX_ARM64_SERVER_RELEASE = cd \ build/prefab/full/linux_arm64_server/release && ./ballisticakit_server prefab-linux-x86-64-server-release: prefab-linux-x86-64-server-release-build - @tools/pcommand ensure_prefab_platform linux_x86_64 + @$(PCOMMAND) ensure_prefab_platform linux_x86_64 @$(RUN_PREFAB_LINUX_X86_64_SERVER_RELEASE) prefab-linux-arm64-server-release: prefab-linux-arm64-server-release-build - @tools/pcommand ensure_prefab_platform linux_arm64 + @$(PCOMMAND) ensure_prefab_platform linux_arm64 @$(RUN_PREFAB_LINUX_ARM64_SERVER_RELEASE) prefab-linux-x86-64-server-release-build: prereqs assets-server \ @@ -493,7 +494,7 @@ RUN_PREFAB_WINDOWS_X86_GUI_DEBUG = cd build/prefab/full/windows_x86_gui/debug \ && ./BallisticaKit.exe prefab-windows-x86-gui-debug: prefab-windows-x86-gui-debug-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 @$(RUN_PREFAB_WINDOWS_X86_GUI_DEBUG) prefab-windows-x86-gui-debug-build: prereqs assets-windows-$(WINPLAT_X86) \ @@ -516,7 +517,7 @@ RUN_PREFAB_WINDOWS_X86_GUI_RELEASE = cd \ build/prefab/full/windows_x86_gui/release && ./BallisticaKit.exe prefab-windows-x86-gui-release: prefab-windows-x86-gui-release-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 @$(RUN_PREFAB_WINDOWS_X86_GUI_RELEASE) prefab-windows-x86-gui-release-build: prereqs \ @@ -541,7 +542,7 @@ RUN_PREFAB_WINDOWS_X86_SERVER_DEBUG = cd \ && dist/python_d.exe ballisticakit_server.py prefab-windows-x86-server-debug: prefab-windows-x86-server-debug-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 @$(RUN_PREFAB_WINDOWS_X86_SERVER_DEBUG) prefab-windows-x86-server-debug-build: prereqs \ @@ -566,7 +567,7 @@ RUN_PREFAB_WINDOWS_X86_SERVER_RELEASE = cd \ && dist/python.exe -O ballisticakit_server.py prefab-windows-x86-server-release: prefab-windows-x86-server-release-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 @$(RUN_PREFAB_WINDOWS_X86_SERVER_RELEASE) prefab-windows-x86-server-release-build: prereqs \ @@ -622,43 +623,43 @@ SPINOFF_TEST_TARGET ?= core # Run a given spinoff test. spinoff-test: - tools/pcommand spinoff_test $(SPINOFF_TEST_TARGET) $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test $(SPINOFF_TEST_TARGET) $(SPINOFF_TEST_EXTRA_ARGS) # Build and check core feature set alone. spinoff-test-core: - tools/pcommand spinoff_test core $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test core $(SPINOFF_TEST_EXTRA_ARGS) # Build and check base feature set alone. spinoff-test-base: - tools/pcommand spinoff_test base $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test base $(SPINOFF_TEST_EXTRA_ARGS) # Build and check plus feature set alone. spinoff-test-plus: - tools/pcommand spinoff_test plus $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test plus $(SPINOFF_TEST_EXTRA_ARGS) # Build and check classic feature set alone. spinoff-test-classic: - tools/pcommand spinoff_test classic $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test classic $(SPINOFF_TEST_EXTRA_ARGS) # Build and check template_fs feature set alone. spinoff-test-template_fs: - tools/pcommand spinoff_test template_fs $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test template_fs $(SPINOFF_TEST_EXTRA_ARGS) # Build and check ui_v1 feature set alone. spinoff-test-ui_v1: - tools/pcommand spinoff_test ui_v1 $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test ui_v1 $(SPINOFF_TEST_EXTRA_ARGS) # Build and check ui_v1_lib feature set alone. spinoff-test-ui_v1_lib: - tools/pcommand spinoff_test ui_v1_lib $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test ui_v1_lib $(SPINOFF_TEST_EXTRA_ARGS) # Build and check scene_v1 feature set alone. spinoff-test-scene_v1: - tools/pcommand spinoff_test scene_v1 $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test scene_v1 $(SPINOFF_TEST_EXTRA_ARGS) # Build and check scene_v1_lib feature set alone. spinoff-test-scene_v1_lib: - tools/pcommand spinoff_test scene_v1_lib $(SPINOFF_TEST_EXTRA_ARGS) + $(PCOMMAND) spinoff_test scene_v1_lib $(SPINOFF_TEST_EXTRA_ARGS) # Blow away all spinoff-test builds. spinoff-test-clean: @@ -666,25 +667,25 @@ spinoff-test-clean: # Grab the current parent project and sync it into ourself. spinoff-update: - @tools/pcommand spinoff_check_submodule_parent + @$(PCOMMAND) spinoff_check_submodule_parent $(MAKE) update - @tools/pcommand echo BLU Pulling current parent project... + @$(PCOMMANDBATCH) echo BLU Pulling current parent project... git submodule update - @tools/pcommand echo BLU Syncing parent into current project... + @$(PCOMMANDBATCH) echo BLU Syncing parent into current project... tools/spinoff update @$(MAKE) update-check # Make sure spinoff didn't break anything. - @tools/pcommand echo GRN Spinoff update successful! + @$(PCOMMANDBATCH) echo GRN Spinoff update successful! # Upgrade to latest parent project and sync it into ourself. spinoff-upgrade: - @tools/pcommand spinoff_check_submodule_parent + @$(PCOMMAND) spinoff_check_submodule_parent $(MAKE) update - @tools/pcommand echo BLU Pulling latest parent project... + @$(PCOMMANDBATCH) echo BLU Pulling latest parent project... cd submodules/ballistica && git checkout master && git pull - @tools/pcommand echo BLU Syncing parent into current project... + @$(PCOMMANDBATCH) echo BLU Syncing parent into current project... tools/spinoff update @$(MAKE) update-check # Make sure spinoff didn't break anything. - @tools/pcommand echo GRN Spinoff upgrade successful! + @$(PCOMMANDBATCH) echo GRN Spinoff upgrade successful! # Tell make which of these targets don't represent files. .PHONY: spinoff-test-core spinoff-test-base spinoff-test-plus \ @@ -700,16 +701,16 @@ spinoff-upgrade: # Update any project files that need it (does NOT build projects). update: prereqs-pre-update - @tools/pcommand update_project + @$(PCOMMAND) update_project # Though not technically necessary, let's keep things like tool-configs # immediately updated so our editors/etc. better reflect the current state. @$(MAKE) -j$(CPUS) prereqs - @tools/pcommand echo GRN Update-Project: SUCCESS! + @$(PCOMMANDBATCH) echo GRN Update-Project: SUCCESS! # Don't update but fail if anything needs it. update-check: prereqs-pre-update - @tools/pcommand update_project --check - @tools/pcommand echo GRN Check-Project: Everything up to date. + @$(PCOMMAND) update_project --check + @$(PCOMMANDBATCH) echo GRN Check-Project: Everything up to date. # Tell make which of these targets don't represent files. .PHONY: update update-check @@ -724,32 +725,32 @@ update-check: prereqs-pre-update # Run formatting on all files in the project considered 'dirty'. format: @$(MAKE) -j$(CPUS) format-code format-scripts format-makefile - @tools/pcommand echo BLD Formatting complete for $(notdir $(CURDIR))! + @$(PCOMMANDBATCH) echo BLD Formatting complete for $(notdir $(CURDIR))! # Same but always formats; ignores dirty state. format-full: @$(MAKE) -j$(CPUS) format-code-full format-scripts-full format-makefile - @tools/pcommand echo BLD Formatting complete for $(notdir $(CURDIR))! + @$(PCOMMANDBATCH) echo BLD Formatting complete for $(notdir $(CURDIR))! # Run formatting for compiled code sources (.cc, .h, etc.). format-code: prereqs - @tools/pcommand formatcode + @$(PCOMMAND) formatcode # Same but always formats; ignores dirty state. format-code-full: prereqs - @tools/pcommand formatcode -full + @$(PCOMMAND) formatcode -full # Runs formatting for scripts (.py, etc). format-scripts: prereqs - @tools/pcommand formatscripts + @$(PCOMMAND) formatscripts # Same but always formats; ignores dirty state. format-scripts-full: prereqs - @tools/pcommand formatscripts -full + @$(PCOMMAND) formatscripts -full # Runs formatting on the project Makefile. format-makefile: prereqs - @tools/pcommand formatmakefile + @$(PCOMMAND) formatmakefile .PHONY: format format-full format-code format-code-full format-scripts \ format-scripts-full @@ -764,67 +765,67 @@ format-makefile: prereqs # Run all project checks. (static analysis) check: py_check_prereqs @$(DMAKE) -j$(CPUS) update-check cpplint pylint mypy - @tools/pcommand echo SGRN BLD ALL CHECKS PASSED! + @$(PCOMMANDBATCH) echo SGRN BLD ALL CHECKS PASSED! # Same as check but no caching (all files are checked). check-full: py_check_prereqs @$(DMAKE) -j$(CPUS) update-check cpplint-full pylint-full mypy-full - @tools/pcommand echo SGRN BLD ALL CHECKS PASSED! + @$(PCOMMANDBATCH) echo SGRN BLD ALL CHECKS PASSED! # Same as 'check' plus optional/slow extra checks. check2: py_check_prereqs @$(DMAKE) -j$(CPUS) update-check cpplint pylint mypy pycharm - @tools/pcommand echo SGRN BLD ALL CHECKS PASSED! + @$(PCOMMANDBATCH) echo SGRN BLD ALL CHECKS PASSED! # Same as check2 but no caching (all files are checked). check2-full: py_check_prereqs @$(DMAKE) -j$(CPUS) update-check cpplint-full pylint-full mypy-full \ pycharm-full - @tools/pcommand echo SGRN BLD ALL CHECKS PASSED! + @$(PCOMMANDBATCH) echo SGRN BLD ALL CHECKS PASSED! # Run Cpplint checks on all C/C++ code. cpplint: prereqs meta - @tools/pcommand cpplint + @$(PCOMMAND) cpplint # Run Cpplint checks without caching (all files are checked). cpplint-full: prereqs meta - @tools/pcommand cpplint -full + @$(PCOMMAND) cpplint -full # Run Pylint checks on all Python Code. pylint: py_check_prereqs - @tools/pcommand pylint + @$(PCOMMAND) pylint # Run Pylint checks without caching (all files are checked). pylint-full: py_check_prereqs - @tools/pcommand pylint -full + @$(PCOMMAND) pylint -full # Run Mypy checks on all Python code. mypy: py_check_prereqs - @tools/pcommand mypy + @$(PCOMMAND) mypy # Run Mypy checks without caching (all files are checked). mypy-full: py_check_prereqs - @tools/pcommand mypy -full + @$(PCOMMAND) mypy -full # Run Mypy checks on all Python code using daemon mode. dmypy: py_check_prereqs - @tools/pcommand dmypy + @$(PCOMMAND) dmypy # Stop the mypy daemon dmypy-stop: py_check_prereqs - @tools/pcommand dmypy -stop + @$(PCOMMAND) dmypy -stop # Run Pyright checks on all Python code. pyright: py_check_prereqs - @tools/pcommand pyright + @$(PCOMMAND) pyright # Run PyCharm checks on all Python code. pycharm: py_check_prereqs - @tools/pcommand pycharm + @$(PCOMMAND) pycharm # Run PyCharm checks without caching (all files are checked). pycharm-full: py_check_prereqs - @tools/pcommand pycharm -full + @$(PCOMMAND) pycharm -full # Build prerequisites needed for python checks. # @@ -862,14 +863,14 @@ TEST_TARGET ?= tests # Run all tests. (live execution verification) test: py_check_prereqs - @tools/pcommand echo BLU Running all tests... - @tools/pcommand tests_warm_start - @tools/pcommand pytest -v $(TEST_TARGET) + @$(PCOMMANDBATCH) echo BLU Running all tests... + @$(PCOMMAND) tests_warm_start + @$(PCOMMAND) pytest -v $(TEST_TARGET) test-verbose: py_check_prereqs - @tools/pcommand echo BLU Running all tests... - @tools/pcommand tests_warm_start - @tools/pcommand pytest -o log_cli=true -o log_cli_level=debug \ + @$(PCOMMANDBATCH) echo BLU Running all tests... + @$(PCOMMAND) tests_warm_start + @$(PCOMMAND) pytest -o log_cli=true -o log_cli_level=debug \ -s -vv $(TEST_TARGET) # Run tests with any caching disabled. @@ -877,17 +878,17 @@ test-full: test # Shortcut to test efro.message only. test-message: - @tools/pcommand pytest -o log_cli=true -o log_cli_level=debug -s -vv \ + @$(PCOMMAND) pytest -o log_cli=true -o log_cli_level=debug -s -vv \ tests/test_efro/test_message.py # Shortcut to test efro.dataclassio only. test-dataclassio: - @tools/pcommand pytest -o log_cli=true -o log_cli_level=debug -s -vv \ + @$(PCOMMAND) pytest -o log_cli=true -o log_cli_level=debug -s -vv \ tests/test_efro/test_dataclassio.py # Shortcut to test efro.rpc only. test-rpc: - @tools/pcommand pytest -o log_cli=true -o log_cli_level=debug -s -vv \ + @$(PCOMMAND) pytest -o log_cli=true -o log_cli_level=debug -s -vv \ tests/test_efro/test_rpc.py # Tell make which of these targets don't represent files. @@ -905,28 +906,28 @@ preflight: @$(MAKE) format @$(MAKE) update @$(MAKE) -j$(CPUS) cpplint pylint mypy test - @tools/pcommand echo SGRN BLD PREFLIGHT SUCCESSFUL! + @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! # Same as 'preflight' without caching (all files are visited). preflight-full: @$(MAKE) format-full @$(MAKE) update @$(MAKE) -j$(CPUS) cpplint-full pylint-full mypy-full test-full - @tools/pcommand echo SGRN BLD PREFLIGHT SUCCESSFUL! + @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! # Same as 'preflight' plus optional/slow extra checks. preflight2: @$(MAKE) format @$(MAKE) update @$(MAKE) -j$(CPUS) cpplint pylint mypy pycharm test - @tools/pcommand echo SGRN BLD PREFLIGHT SUCCESSFUL! + @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! # Same as 'preflight2' but without caching (all files visited). preflight2-full: @$(MAKE) format-full @$(MAKE) update @$(MAKE) -j$(CPUS) cpplint-full pylint-full mypy-full pycharm-full test-full - @tools/pcommand echo SGRN BLD PREFLIGHT SUCCESSFUL! + @$(PCOMMANDBATCH) echo SGRN BLD PREFLIGHT SUCCESSFUL! # Tell make which of these targets don't represent files. .PHONY: preflight preflight-full preflight2 preflight2-full @@ -956,20 +957,20 @@ windows-staging: assets-windows resources meta # Build and run a debug windows build (from WSL). windows-debug: windows-debug-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 build/windows/Debug_Win32/BallisticaKitGeneric.exe # Build and run a release windows build (from WSL). windows-release: windows-release-build - @tools/pcommand ensure_prefab_platform windows_x86 + @$(PCOMMAND) ensure_prefab_platform windows_x86 build/windows/Release_Win32/BallisticaKitGeneric.exe # Build a debug windows build (from WSL). windows-debug-build: \ build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib \ build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb - @tools/pcommand ensure_prefab_platform windows_x86 - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) ensure_prefab_platform windows_x86 + @$(PCOMMAND) wsl_build_check_win_drive WINDOWS_CONFIGURATION=Debug WINDOWS_PLATFORM=Win32 $(MAKE) windows-staging WINDOWS_PROJECT=Generic WINDOWS_CONFIGURATION=Debug WINDOWS_PLATFORM=Win32 \ $(MAKE) _windows-wsl-build @@ -978,8 +979,8 @@ windows-debug-build: \ windows-debug-rebuild: \ build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib \ build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb - @tools/pcommand ensure_prefab_platform windows_x86 - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) ensure_prefab_platform windows_x86 + @$(PCOMMAND) wsl_build_check_win_drive WINDOWS_CONFIGURATION=Debug WINDOWS_PLATFORM=Win32 $(MAKE) windows-staging WINDOWS_PROJECT=Generic WINDOWS_CONFIGURATION=Debug WINDOWS_PLATFORM=Win32 \ $(MAKE) _windows-wsl-rebuild @@ -988,8 +989,8 @@ windows-debug-rebuild: \ windows-release-build: \ build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib \ build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb - @tools/pcommand ensure_prefab_platform windows_x86 - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) ensure_prefab_platform windows_x86 + @$(PCOMMAND) wsl_build_check_win_drive WINDOWS_CONFIGURATION=Release WINDOWS_PLATFORM=Win32 $(MAKE) windows-staging WINDOWS_PROJECT=Generic WINDOWS_CONFIGURATION=Release WINDOWS_PLATFORM=Win32 \ $(MAKE) _windows-wsl-build @@ -998,8 +999,8 @@ windows-release-build: \ windows-release-rebuild: \ build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib \ build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb - @tools/pcommand ensure_prefab_platform windows_x86 - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) ensure_prefab_platform windows_x86 + @$(PCOMMAND) wsl_build_check_win_drive WINDOWS_CONFIGURATION=Release WINDOWS_PLATFORM=Win32 $(MAKE) windows-staging WINDOWS_PROJECT=Generic WINDOWS_CONFIGURATION=Release WINDOWS_PLATFORM=Win32 \ $(MAKE) _windows-wsl-rebuild @@ -1050,10 +1051,10 @@ cmake-lldb: cmake-build cmake-build: assets-cmake resources cmake-binary @$(STAGE_BUILD) -cmake -$(CM_BT_LC) -builddir build/cmake/$(CM_BT_LC) \ build/cmake/$(CM_BT_LC)/staged - @tools/pcommand echo BLD Build complete: BLU build/cmake/$(CM_BT_LC)/staged + @$(PCOMMANDBATCH) echo BLD Build complete: BLU build/cmake/$(CM_BT_LC)/staged cmake-binary: meta - @tools/pcommand cmake_prep_dir build/cmake/$(CM_BT_LC) + @$(PCOMMAND) cmake_prep_dir build/cmake/$(CM_BT_LC) @cd build/cmake/$(CM_BT_LC) && test -f Makefile \ || cmake -DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) \ $(shell pwd)/ballisticakit-cmake @@ -1071,11 +1072,11 @@ cmake-server-build: assets-server meta cmake-server-binary @$(STAGE_BUILD) -cmakeserver -$(CM_BT_LC) \ -builddir build/cmake/server-$(CM_BT_LC) \ build/cmake/server-$(CM_BT_LC)/staged - @tools/pcommand echo BLD \ + @$(PCOMMANDBATCH) echo BLD \ Server build complete: BLU build/cmake/server-$(CM_BT_LC)/staged cmake-server-binary: meta - @tools/pcommand cmake_prep_dir build/cmake/server-$(CM_BT_LC) + @$(PCOMMAND) cmake_prep_dir build/cmake/server-$(CM_BT_LC) @cd build/cmake/server-$(CM_BT_LC) && test -f Makefile \ || cmake -DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) -DHEADLESS=true \ $(shell pwd)/ballisticakit-cmake @@ -1090,14 +1091,14 @@ cmake-modular-build: assets-cmake meta cmake-modular-binary @$(STAGE_BUILD) -cmakemodular -$(CM_BT_LC) \ -builddir build/cmake/modular-$(CM_BT_LC) \ build/cmake/modular-$(CM_BT_LC)/staged - @tools/pcommand echo BLD \ + @$(PCOMMANDBATCH) echo BLD \ Modular build complete: BLU build/cmake/modular-$(CM_BT_LC)/staged cmake-modular: cmake-modular-build cd build/cmake/modular-$(CM_BT_LC)/staged && ./ballisticakit cmake-modular-binary: meta - @tools/pcommand cmake_prep_dir build/cmake/modular-$(CM_BT_LC) + @$(PCOMMAND) cmake_prep_dir build/cmake/modular-$(CM_BT_LC) @cd build/cmake/modular-$(CM_BT_LC) && test -f Makefile \ || cmake -DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) \ $(shell pwd)/ballisticakit-cmake @@ -1115,11 +1116,11 @@ cmake-modular-server-build: assets-server meta cmake-modular-server-binary @$(STAGE_BUILD) -cmakemodularserver -$(CM_BT_LC) \ -builddir build/cmake/modular-server-$(CM_BT_LC) \ build/cmake/modular-server-$(CM_BT_LC)/staged - @tools/pcommand echo BLD \ + @$(PCOMMANDBATCH) echo BLD \ Server build complete: BLU build/cmake/modular-server-$(CM_BT_LC)/staged cmake-modular-server-binary: meta - @tools/pcommand cmake_prep_dir build/cmake/modular-server-$(CM_BT_LC) + @$(PCOMMAND) cmake_prep_dir build/cmake/modular-server-$(CM_BT_LC) @cd build/cmake/modular-server-$(CM_BT_LC) && test -f Makefile \ || cmake -DCMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) -DHEADLESS=true \ $(shell pwd)/ballisticakit-cmake @@ -1163,11 +1164,11 @@ DMAKE = $(MAKE) MAKEFLAGS= MKFLAGS= MAKELEVEL= # if using this on other platforms. CPUS = $(shell getconf _NPROCESSORS_ONLN || echo 8) PROJ_DIR = $(abspath $(CURDIR)) -VERSION = $(shell tools/pcommand version version) -BUILD_NUMBER = $(shell tools/pcommand version build) +VERSION = $(shell $(PCOMMAND) version version) +BUILD_NUMBER = $(shell $(PCOMMAND) version build) BUILD_DIR = $(PROJ_DIR)/build LAZYBUILDDIR = .cache/lazybuild -STAGE_BUILD = $(PROJ_DIR)/tools/pcommand stage_build +STAGE_BUILD = $(PROJ_DIR)/$(PCOMMAND) stage_build # Things to ignore when doing root level cleans. Note that we exclude build # and just blow that away manually; it might contain git repos or other things @@ -1176,16 +1177,16 @@ ROOT_CLEAN_IGNORES = --exclude=config/localconfig.json \ --exclude=.spinoffdata \ --exclude=/build -CHECK_CLEAN_SAFETY = tools/pcommand check_clean_safety +CHECK_CLEAN_SAFETY = $(PCOMMAND) check_clean_safety # Some tool configs that need filtering (mainly injecting projroot path). -TOOL_CFG_INST = tools/pcommand tool_config_install +TOOL_CFG_INST = $(PCOMMAND) tool_config_install # Anything that affects tool-config generation. TOOL_CFG_SRC = tools/efrotools/toolconfig.py config/projectconfig.json # Anything that should trigger an environment-check when changed. -ENV_SRC = tools/pcommand tools/batools/build.py +ENV_SRC = $(PCOMMAND) tools/batools/build.py .clang-format: config/toolconfigsrc/clang-format $(TOOL_CFG_SRC) @$(TOOL_CFG_INST) $< $@ @@ -1219,12 +1220,12 @@ SKIP_ENV_CHECKS ?= 0 .cache/checkenv: $(ENV_SRC) @if [ $(SKIP_ENV_CHECKS) -ne 1 ]; then \ - tools/pcommand checkenv && mkdir -p .cache && touch .cache/checkenv; \ + $(PCOMMAND) checkenv && mkdir -p .cache && touch .cache/checkenv; \ fi $(PCOMMANDBATCHBIN): src/tools/pcommandbatch/pcommandbatch.c \ src/tools/pcommandbatch/cJSON.c - @tools/pcommand build_pcommandbatch $^ $@ + @$(PCOMMAND) build_pcommandbatch $^ $@ # CMake build-type lowercase CM_BT_LC = $(shell echo $(CMAKE_BUILD_TYPE) | tr A-Z a-z) @@ -1253,7 +1254,7 @@ ballisticakit-cmake/.clang-format: .clang-format # whenever CMakeLists changes. .cache/compile_commands_db/compile_commands.json: \ ballisticakit-cmake/CMakeLists.txt - @tools/pcommand echo BLU Updating compile commands db... + @$(PCOMMANDBATCH) echo BLU Updating compile commands db... @mkdir -p .cache/compile_commands_db @cd .cache/compile_commands_db \ && cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug \ @@ -1262,29 +1263,29 @@ ballisticakit-cmake/.clang-format: .clang-format && rm -rf .cache/compile_commands_db \ && mkdir .cache/compile_commands_db \ && mv compile_commands.json .cache/compile_commands_db - @tools/pcommand echo BLU Created compile commands db at $@ + @$(PCOMMANDBATCH) echo BLU Created compile commands db at $@ _windows-wsl-build: - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) wsl_build_check_win_drive $(WIN_MSBUILD_EXE_B) \ - $(shell tools/pcommand wsl_path_to_win --escape \ + $(shell $(PCOMMAND) wsl_path_to_win --escape \ ballisticakit-windows/$(WINPRJ)/BallisticaKit$(WINPRJ).vcxproj) \ -target:Build \ -property:Configuration=$(WINCFG) \ -property:Platform=$(WINPLT) \ $(VISUAL_STUDIO_VERSION) - @tools/pcommand echo BLU BLD Built build/windows/BallisticaKit$(WINPRJ).exe. + @$(PCOMMAND) echo BLU BLD Built build/windows/BallisticaKit$(WINPRJ).exe. _windows-wsl-rebuild: - @tools/pcommand wsl_build_check_win_drive + @$(PCOMMAND) wsl_build_check_win_drive $(WIN_MSBUILD_EXE_B) \ - $(shell tools/pcommand wsl_path_to_win --escape \ + $(shell $(PCOMMAND) wsl_path_to_win --escape \ ballisticakit-windows/$(WINPRJ)/BallisticaKit$(WINPRJ).vcxproj) \ -target:Rebuild \ -property:Configuration=$(WINCFG) \ -property:Platform=$(WINPLT) \ $(VISUAL_STUDIO_VERSION) - @tools/pcommand echo BLU BLD Built build/windows/BallisticaKit$(WINPRJ).exe. + @$(PCOMMAND) echo BLU BLD Built build/windows/BallisticaKit$(WINPRJ).exe. # Tell make which of these targets don't represent files. .PHONY: _windows-wsl-build _windows-wsl-rebuild diff --git a/ballisticakit-cmake/.idea/dictionaries/ericf.xml b/ballisticakit-cmake/.idea/dictionaries/ericf.xml index f3549c9a..419e7d35 100644 --- a/ballisticakit-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticakit-cmake/.idea/dictionaries/ericf.xml @@ -332,9 +332,12 @@ clearsign clientid clientinfo + clientprint + clientprints clienttobasn clipcount cloudtoba + clrtp cmakelist cmakemodular cmakemodularserver diff --git a/src/tools/pcommandbatch/pcommandbatch.c b/src/tools/pcommandbatch/pcommandbatch.c index 57c4de64..a8a12dc3 100644 --- a/src/tools/pcommandbatch/pcommandbatch.c +++ b/src/tools/pcommandbatch/pcommandbatch.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,16 @@ int establish_connection_(const struct Context_* ctx); int calc_paths_(struct Context_* ctx); int send_command_(struct Context_* ctx, int argc, char** argv); int handle_response_(const struct Context_* ctx); + +// Read all data from a socket and return as a malloc'ed null-terminated +// string. +char* read_string_from_socket_(const struct Context_* ctx); + +// Tear down context (closing socket, etc.) before closing app. +void tear_down_context_(struct Context_* ctx); + +// If a valid state file is present at the provided path and not older than +// server_idle_seconds, return said port as an int. Otherwise return -1; int get_running_server_port_(const struct Context_* ctx, const char* state_file_path_full); @@ -49,6 +60,7 @@ int main(int argc, char** argv) { ctx.server_idle_seconds = 5; ctx.pid = getpid(); + ctx.sockfd = -1; // Verbose mode enables more printing here. Debug mode enables that plus // extra stuff. The extra stuff is mostly the server side though. @@ -67,36 +79,43 @@ int main(int argc, char** argv) { // Figure our which file path we'll use to get server state. if (calc_paths_(&ctx) != 0) { + tear_down_context_(&ctx); return 1; } // Establish communication with said server (spinning it up if needed). ctx.sockfd = establish_connection_(&ctx); if (ctx.sockfd == -1) { + tear_down_context_(&ctx); return 1; } if (send_command_(&ctx, argc, argv) != 0) { + tear_down_context_(&ctx); return 1; } int result_val = handle_response_(&ctx); if (result_val != 0) { + tear_down_context_(&ctx); return 1; } - if (close(ctx.sockfd) != 0) { - fprintf( - stderr, - "Error: pcommandbatch client %s_%d (pid %d): error on socket close.\n", - ctx.instance_prefix, ctx.instance_num, ctx.pid); - return 1; - } + tear_down_context_(&ctx); return result_val; } -// If a valid state file is present at the provided path and not older than -// server_idle_seconds, return said port as an int. Otherwise return -1; +void tear_down_context_(struct Context_* ctx) { + if (ctx->sockfd != -1) { + if (close(ctx->sockfd) != 0) { + fprintf(stderr, + "Error: pcommandbatch client %s_%d (pid %d): error %d closing " + "socket.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid, errno); + } + } +} + int get_running_server_port_(const struct Context_* ctx, const char* state_file_path_full) { struct stat file_stat; @@ -120,12 +139,12 @@ int get_running_server_port_(const struct Context_* ctx, int age_seconds = current_time - file_stat.st_mtime; if (ctx->verbose) { if (age_seconds <= ctx->server_idle_seconds) { - fprintf( - stderr, - "pcommandbatch client %s_%d (pid %d) found state file with age %d at " - "time %ld.\n", - ctx->instance_prefix, ctx->instance_num, ctx->pid, age_seconds, - time(NULL)); + fprintf(stderr, + "pcommandbatch client %s_%d (pid %d) found state file with age " + "%d at " + "time %ld.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid, age_seconds, + time(NULL)); } } @@ -153,6 +172,7 @@ int get_running_server_port_(const struct Context_* ctx, ctx->instance_prefix, ctx->instance_num, ctx->pid); return -1; } + // If results included output, print it. cJSON* port_obj = cJSON_GetObjectItem(state_dict, "p"); if (!port_obj || !cJSON_IsNumber(port_obj)) { @@ -166,8 +186,6 @@ int get_running_server_port_(const struct Context_* ctx, int port = cJSON_GetNumberValue(port_obj); cJSON_Delete(state_dict); return port; - - // return val; } int path_exists_(const char* path) { @@ -221,9 +239,9 @@ int establish_connection_(const struct Context_* ctx) { snprintf(buf, sizeof(buf), "%s run_pcommandbatch_server --timeout %d --project-dir %s" " --state-dir %s --instance %s_%d %s", - ctx->pcommandpath, ctx->server_idle_seconds, ctx->project_dir_path, - ctx->state_dir_path, ctx->instance_prefix, ctx->instance_num, - endbuf); + ctx->pcommandpath, ctx->server_idle_seconds, + ctx->project_dir_path, ctx->state_dir_path, ctx->instance_prefix, + ctx->instance_num, endbuf); system(buf); // Spin and wait up to a few seconds for the file to appear. @@ -260,12 +278,12 @@ int establish_connection_(const struct Context_* ctx) { // Ok we got a port; now try to connect to it. if (port != -1) { if (ctx->verbose) { - fprintf( - stderr, - "pcommandbatch client %s_%d (pid %d) will use server on port %d at " - "time %ld.\n", - ctx->instance_prefix, ctx->instance_num, ctx->pid, port, - time(NULL)); + fprintf(stderr, + "pcommandbatch client %s_%d (pid %d) will use server on port " + "%d at " + "time %ld.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid, port, + time(NULL)); } struct sockaddr_in serv_addr; @@ -288,11 +306,11 @@ int establish_connection_(const struct Context_* ctx) { } } else { // Currently not retrying on other errors. - fprintf( - stderr, - "Error: pcommandbatch client %s_%d (pid %d): connect failed (errno " - "%d).\n", - ctx->instance_prefix, ctx->instance_num, ctx->pid, errno); + fprintf(stderr, + "Error: pcommandbatch client %s_%d (pid %d): connect failed " + "(errno " + "%d).\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid, errno); close(sockfd); return -1; } @@ -346,11 +364,11 @@ int calc_paths_(struct Context_* ctx) { ctx->instance_prefix, ctx->pid); return -1; } - fprintf( - stderr, - "Error: pcommandbatch client %s (pid %d): pcommandbatch from cwd '%s' " - "is not supported.\n", - ctx->instance_prefix, ctx->pid, cwdbuf); + fprintf(stderr, + "Error: pcommandbatch client %s (pid %d): pcommandbatch from cwd " + "'%s' " + "is not supported.\n", + ctx->instance_prefix, ctx->pid, cwdbuf); return -1; } assert(ctx->pcommandpath != NULL); @@ -358,10 +376,11 @@ int calc_paths_(struct Context_* ctx) { // Spread requests for each location out randomly across a few instances. // This greatly increases scalability though is probably wasteful when - // running just a few commands. Maybe there's some way to smartly scale - // this. The best setup might be to have a single 'controller' server - // instance that spins up worker instances as needed. Though such a fancy - // setup might be overkill. + // running just a few commands since we likely spin up a new server for + // each. Maybe there's some way to smartly scale this. The best setup + // might be to have a single 'controller' server instance that spins up + // worker instances as needed. Though such a fancy setup might be + // overkill. ctx->instance_num = rand() % 6; // I was wondering if using pid would lead to a more even distribution, @@ -396,10 +415,10 @@ int send_command_(struct Context_* ctx, int argc, char** argv) { // Issue a write shutdown so they get EOF on the other end. if (shutdown(ctx->sockfd, SHUT_WR) < 0) { - fprintf( - stderr, - "Error: pcommandbatch client %s_%d (pid %d): write shutdown failed.\n", - ctx->instance_prefix, ctx->instance_num, ctx->pid); + fprintf(stderr, + "Error: pcommandbatch client %s_%d (pid %d): write shutdown " + "failed.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid); return -1; } @@ -411,26 +430,18 @@ int send_command_(struct Context_* ctx, int argc, char** argv) { } int handle_response_(const struct Context_* ctx) { - // Read the response. Currently expecting short-ish responses only; will - // have to revisit this if/when they get long. - char inbuf[512]; - ssize_t result = read(ctx->sockfd, inbuf, sizeof(inbuf) - 1); - if (result < 0 || result == sizeof(inbuf) - 1) { + char* inbuf = read_string_from_socket_(ctx); + if (!inbuf) { fprintf(stderr, "Error: pcommandbatch client %s_%d (pid %d): failed to read result " "(errno %d).\n", ctx->instance_prefix, ctx->instance_num, ctx->pid, errno); - close(ctx->sockfd); return -1; } - if (ctx->verbose) { - fprintf(stderr, - "pcommandbatch client %s_%d (pid %d) read %zd byte response.\n", - ctx->instance_prefix, ctx->instance_num, ctx->pid, result); - } - inbuf[result] = 0; // null terminate result str. cJSON* result_dict = cJSON_Parse(inbuf); + free(inbuf); + if (!result_dict) { fprintf( stderr, @@ -440,7 +451,7 @@ int handle_response_(const struct Context_* ctx) { return -1; } - // If results included output, print it. + // If results included stdout output, print it. cJSON* result_output = cJSON_GetObjectItem(result_dict, "o"); if (!result_output || !cJSON_IsString(result_output)) { fprintf( @@ -448,6 +459,7 @@ int handle_response_(const struct Context_* ctx) { "Error: pcommandbatch client %s_%d (pid %d): failed to parse result " "output value.\n", ctx->instance_prefix, ctx->instance_num, ctx->pid); + cJSON_Delete(result_dict); return -1; } char* output_str = cJSON_GetStringValue(result_output); @@ -456,6 +468,23 @@ int handle_response_(const struct Context_* ctx) { printf("%s", output_str); } + // If results included stderr output, print it. + result_output = cJSON_GetObjectItem(result_dict, "e"); + if (!result_output || !cJSON_IsString(result_output)) { + fprintf( + stderr, + "Error: pcommandbatch client %s_%d (pid %d): failed to parse result " + "output value.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid); + cJSON_Delete(result_dict); + return -1; + } + output_str = cJSON_GetStringValue(result_output); + assert(output_str); + if (output_str[0] != 0) { + fprintf(stderr, "%s", output_str); + } + cJSON* result_code = cJSON_GetObjectItem(result_dict, "r"); if (!result_code || !cJSON_IsNumber(result_code)) { fprintf( @@ -463,6 +492,7 @@ int handle_response_(const struct Context_* ctx) { "Error: pcommandbatch client %s_%d (pid %d): failed to parse result " "code value.\n", ctx->instance_prefix, ctx->instance_num, ctx->pid); + cJSON_Delete(result_dict); return -1; } int result_val = cJSON_GetNumberValue(result_code); @@ -474,3 +504,57 @@ int handle_response_(const struct Context_* ctx) { return result_val; } + +char* read_string_from_socket_(const struct Context_* ctx) { + const size_t initial_buffer_size = 1024 * 10; + char* buffer = NULL; + size_t buffer_size = 0; + size_t data_received = 0; + + // Allocate initial memory for the buffer + buffer = malloc(initial_buffer_size); + if (!buffer) { + perror("Failed to allocate memory for buffer"); + return NULL; + } + buffer_size = initial_buffer_size; + + while (1) { + // Read data from the socket. + ssize_t bytes_read = recv(ctx->sockfd, buffer + data_received, + buffer_size - data_received - 1, 0); + if (bytes_read == -1) { + perror("Error reading socket data"); + free(buffer); + return NULL; + } else if (bytes_read == 0) { + // Connection closed. + break; + } + + data_received += bytes_read; + + // Check if buffer is full (reserving space for term char); resize if + // necessary. + if (data_received + 1 >= buffer_size) { + buffer_size *= 2; + char* rbuffer = (char*)realloc(buffer, buffer_size); + if (rbuffer) { + buffer = rbuffer; + } else { + perror("Failed to resize buffer"); + free(buffer); + return NULL; + } + } + } + assert(data_received + 1 < buffer_size); + if (ctx->verbose) { + fprintf(stderr, + "pcommandbatch client %s_%d (pid %d) read %zu byte response.\n", + ctx->instance_prefix, ctx->instance_num, ctx->pid, data_received); + } + + buffer[data_received] = 0; // Null terminator. + return buffer; +} diff --git a/tools/batools/pcommands.py b/tools/batools/pcommands.py index dc2d8a13..32e9bb24 100644 --- a/tools/batools/pcommands.py +++ b/tools/batools/pcommands.py @@ -456,9 +456,9 @@ def capitalize() -> None: def upper() -> None: """Print args uppercased.""" - pcommand.disallow_in_batch() - - print(' '.join(w.upper() for w in sys.argv[2:]), end='') + pcommand.clientprint( + ' '.join(w.upper() for w in pcommand.get_args()), end='' + ) def efrocache_update() -> None: @@ -481,7 +481,7 @@ def efrocache_get() -> None: output = get_target(args[0], batch=pcommand.is_batch(), clr=pcommand.clr()) if pcommand.is_batch(): - pcommand.set_output(output) + pcommand.clientprint(output) def get_modern_make() -> None: @@ -618,13 +618,12 @@ def prefab_run_var() -> None: """Print the current platform prefab run target var.""" import batools.build - pcommand.disallow_in_batch() - - if len(sys.argv) != 3: + args = pcommand.get_args() + if len(args) != 1: raise RuntimeError('Expected 1 arg.') - base = sys.argv[2].replace('-', '_').upper() + base = args[0].replace('-', '_').upper() platform = batools.build.get_current_prefab_platform().upper() - print(f'RUN_PREFAB_{platform}_{base}', end='') + pcommand.clientprint(f'RUN_PREFAB_{platform}_{base}', end='') def prefab_binary_path() -> None: @@ -679,16 +678,19 @@ def lazybuild() -> None: import batools.build from efro.error import CleanError + # This command is not a good candidate for batch since it can be + # long running and prints various stuff throughout the process. pcommand.disallow_in_batch() + args = pcommand.get_args() - if len(sys.argv) < 5: + if len(args) < 3: raise CleanError('Expected at least 3 args') try: - category = batools.build.LazyBuildCategory(sys.argv[2]) + category = batools.build.LazyBuildCategory(args[0]) except ValueError as exc: raise CleanError(exc) from exc - target = sys.argv[3] - command = ' '.join(sys.argv[4:]) + target = args[1] + command = ' '.join(args[2:]) try: batools.build.lazybuild(target, category, command) except subprocess.CalledProcessError as exc: diff --git a/tools/efro/error.py b/tools/efro/error.py index 8bd4c8c7..f2045ce9 100644 --- a/tools/efro/error.py +++ b/tools/efro/error.py @@ -7,7 +7,9 @@ from typing import TYPE_CHECKING import errno if TYPE_CHECKING: - pass + from typing import Any + + from efro.terminal import ClrBase class CleanError(Exception): @@ -25,18 +27,29 @@ class CleanError(Exception): more descriptive exception types. """ - def pretty_print(self, flush: bool = True, prefix: str = 'Error') -> None: + def pretty_print( + self, + flush: bool = True, + prefix: str = 'Error', + file: Any = None, + clr: type[ClrBase] | None = None, + ) -> None: """Print the error to stdout, using red colored output if available. If the error has an empty message, prints nothing (not even a newline). """ from efro.terminal import Clr + if clr is None: + clr = Clr + if prefix: prefix = f'{prefix}: ' errstr = str(self) if errstr: - print(f'{Clr.SRED}{prefix}{errstr}{Clr.RST}', flush=flush) + print( + f'{clr.SRED}{prefix}{errstr}{clr.RST}', flush=flush, file=file + ) class CommunicationError(Exception): diff --git a/tools/efrotools/pcommand.py b/tools/efrotools/pcommand.py index 078e1ceb..43fb1be8 100644 --- a/tools/efrotools/pcommand.py +++ b/tools/efrotools/pcommand.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: + import io import threading from typing import Any @@ -28,7 +29,7 @@ PROJROOT = Path(__file__).resolve().parents[2] # the name of the pcommand; only the arguments *to* the command. _g_thread_local_storage: threading.local | None = None -# Discovered functions for the currently running pcommand instance. +# Discovered functions for the current project. _g_funcs: dict | None = None # Are we running as a server? @@ -43,6 +44,9 @@ def pcommand_main(globs: dict[str, Any]) -> None: """ import types + from efro.terminal import Clr + from efro.error import CleanError + global _g_funcs # pylint: disable=global-statement assert _g_funcs is None @@ -57,8 +61,14 @@ def pcommand_main(globs: dict[str, Any]) -> None: ) ) - # Call the one based on sys args. - sys.exit(_run_pcommand(sys.argv)) + try: + _run_pcommand(sys.argv) + except KeyboardInterrupt as exc: + print(f'{Clr.RED}{exc}{Clr.RST}') + sys.exit(1) + except CleanError as exc: + exc.pretty_print() + sys.exit(1) def get_args() -> list[str]: @@ -79,104 +89,99 @@ def get_args() -> list[str]: def clr() -> type[ClrBase]: - """Like efro.terminal.Clr but works correctly under pcommandbatch.""" + """Like efro.terminal.Clr but for use with pcommand.clientprint(). + + This properly colorizes or doesn't colorize based on whether the + *client* where output will be displayed is running on a terminal. + Regular print() output should still use efro.terminal.Clr for this + purpose. + """ import efro.terminal - # Note: currently just using the 'isatty' value from the client. - # ideally should expand the client-side logic to exactly match what - # efro.terminal.Clr does locally. if _g_batch_server_mode: assert _g_thread_local_storage is not None - isatty = _g_thread_local_storage.isatty - assert isinstance(isatty, bool) - return efro.terminal.ClrAlways if isatty else efro.terminal.ClrNever + clrtp: type[ClrBase] = _g_thread_local_storage.clr + assert issubclass(clrtp, efro.terminal.ClrBase) + return clrtp return efro.terminal.Clr -def set_output(output: str, newline: bool = True) -> None: - """Set an output string for the current pcommand. +def clientprint( + *args: Any, stderr: bool = False, end: str | None = None +) -> None: + """Print to client stdout. - This will be printed to stdout on the client even in batch mode. + Note that, in batch mode, the results of all clientprints will show + up only after the command completes. In regular mode, clientprint() + simply passes through to regular print(). """ - if newline: - output = f'{output}\n' - - if not _g_batch_server_mode: - print(output, end='') - return - - # Ok, we're in batch mode. Stuff this into thread-local storage to - # be returned once we're done. - assert _g_thread_local_storage is not None - if hasattr(_g_thread_local_storage, 'output'): - raise RuntimeError('Output is already set for this pcommand.') - _g_thread_local_storage.output = output + if _g_batch_server_mode: + assert _g_thread_local_storage is not None + print( + *args, + file=_g_thread_local_storage.stderr + if stderr + else _g_thread_local_storage.stdout, + end=end, + ) + else: + print(*args, end=end) -def _run_pcommand(sysargv: list[str]) -> int: - """Do the thing.""" +def _run_pcommand(sysargv: list[str]) -> None: + """Run a pcommand given raw sys args.""" from efro.error import CleanError - from efro.terminal import Clr assert _g_funcs is not None - retval = 0 + clrtp = clr() + error = False show_help = False if len(sysargv) < 2: - print(f'{Clr.RED}ERROR: command expected.{Clr.RST}') + clientprint(f'{clrtp.SRED}Error: Command expected.{clrtp.RST}') show_help = True - retval = 1 + error = True else: if sysargv[1] == 'help': if len(sysargv) == 2: show_help = True elif sysargv[2] not in _g_funcs: - print('Invalid help command.') - retval = 1 + raise CleanError('Invalid help command.') else: docs = _trim_docstring( getattr(_g_funcs[sysargv[2]], '__doc__', '') ) - print( - f'\n{Clr.MAG}{Clr.BLD}pcommand {sysargv[2]}:{Clr.RST}\n' - f'{Clr.MAG}{docs}{Clr.RST}\n' + clientprint( + f'\n{clrtp.MAG}{clrtp.BLD}' + f'pcommand {sysargv[2]}:{clrtp.RST}\n' + f'{clrtp.MAG}{docs}{clrtp.RST}\n' ) elif sysargv[1] in _g_funcs: - try: - _g_funcs[sysargv[1]]() - except KeyboardInterrupt as exc: - print(f'{Clr.RED}{exc}{Clr.RST}') - retval = 1 - except CleanError as exc: - exc.pretty_print() - retval = 1 + _g_funcs[sysargv[1]]() else: - print( - f'{Clr.RED}Unknown pcommand: "{sysargv[1]}"{Clr.RST}', - file=sys.stderr, - ) - retval = 1 + raise CleanError(f"Unknown pcommand '{sysargv[1]}'.") if show_help: - print( - f'The {Clr.MAG}{Clr.BLD}pcommand{Clr.RST} script encapsulates' + clientprint( + f'The {clrtp.MAG}{clrtp.BLD}pcommand{clrtp.RST} script encapsulates' f' a collection of project-related commands.' ) - print( - f"Run {Clr.MAG}{Clr.BLD}'pcommand [COMMAND] ...'" - f'{Clr.RST} to run a command.' + clientprint( + f"Run {clrtp.MAG}{clrtp.BLD}'pcommand [COMMAND] ...'" + f'{clrtp.RST} to run a command.' ) - print( - f"Run {Clr.MAG}{Clr.BLD}'pcommand help [COMMAND]'" - f'{Clr.RST} for full documentation for a command.' + clientprint( + f"Run {clrtp.MAG}{clrtp.BLD}'pcommand help [COMMAND]'" + f'{clrtp.RST} for full documentation for a command.' ) - print('Available commands:') + clientprint('Available commands:') for func, obj in sorted(_g_funcs.items()): doc = getattr(obj, '__doc__', '').splitlines()[0].strip() - print(f'{Clr.MAG}{func}{Clr.BLU} - {doc}{Clr.RST}') + clientprint(f'{clrtp.MAG}{func}{clrtp.BLU} - {doc}{clrtp.RST}') - return retval + if error: + raise CleanError() def enter_batch_server_mode() -> None: @@ -204,32 +209,49 @@ def is_batch() -> bool: return _g_batch_server_mode -def run_client_pcommand(args: list[str], isatty: bool) -> tuple[int, str]: - """Call a pcommand function when running as a batch server.""" +def run_client_pcommand( + args: list[str], clrtp: type[ClrBase], logpath: str +) -> tuple[int, str, str]: + """Call a pcommand function as a server. + + Returns a result code and stdout output. + """ + import io + import traceback + + from efro.error import CleanError + assert _g_batch_server_mode assert _g_thread_local_storage is not None - # Clear any output from previous commands on this thread. - if hasattr(_g_thread_local_storage, 'output'): - delattr(_g_thread_local_storage, 'output') + with io.StringIO() as stdout, io.StringIO() as stderr: + # Stuff some state into thread-local storage for the handler thread + # to access. + _g_thread_local_storage.stdout = stdout + _g_thread_local_storage.stderr = stderr + _g_thread_local_storage.argv = args + _g_thread_local_storage.clr = clrtp - # Stuff args into our thread-local storage so the user can get at - # them. - _g_thread_local_storage.argv = args - _g_thread_local_storage.isatty = isatty + try: + _run_pcommand(args) + resultcode = 0 + except KeyboardInterrupt as exc: + clientprint(f'{clrtp.RED}{exc}{clrtp.RST}') + resultcode = 1 + except CleanError as exc: + exc.pretty_print(file=stderr, clr=clrtp) + resultcode = 1 + except Exception: + traceback.print_exc(file=stderr) + print( + f'More error output may be available at {logpath}', file=stderr + ) + resultcode = 1 - # Run the command. This may return an explicit code or may throw an - # exception. - resultcode: int = _run_pcommand(args) + stdout_str = stdout.getvalue() + stderr_str = stderr.getvalue() - # Handle error result-codes consistently with exceptions. - if resultcode != 0: - raise RuntimeError(f'client pcommand returned error code {resultcode}.') - - output = getattr(_g_thread_local_storage, 'output', '') - assert isinstance(output, str) - - return (resultcode, output) + return resultcode, stdout_str, stderr_str def disallow_in_batch() -> None: diff --git a/tools/efrotools/pcommandbatch.py b/tools/efrotools/pcommandbatch.py index ad2e5d51..b716a561 100644 --- a/tools/efrotools/pcommandbatch.py +++ b/tools/efrotools/pcommandbatch.py @@ -11,42 +11,48 @@ a minimal set of modules. This can add up for large builds where hundreds or thousands of pcommands are being run. To help fight that problem, pcommandbatch introduces a way to run -pcommands by submitting requests to a temporary local server daemon. +pcommands by submitting requests to temporary local server daemons. This allows individual pcommand calls to go through a very lightweight client binary that simply forwards the command to a running server. -This cuts minimal client runtime down to nearly zero. Building and -managing the server and client are handled automatically, and systems -which are unable to compile a client binary can fall back to using -vanilla pcommand in those cases. +This cuts minimal client runtime down greatly. Building and managing +the server and client are handled automatically, and systems which are +unable to compile a client binary can fall back to using vanilla +pcommand in those cases. -A few considerations must be made when writing pcommands that work -in batch mode. By default, all existing pcommands have been fitted with -a disallow_in_batch() call which triggers an error under batch mode. +A few considerations must be made when using pcommands with batch mode. +By default, all existing pcommands have been fitted with a +disallow_in_batch() call which triggers an error under batch mode. These calls should be removed if/when each call is updated to work cleanly in batch mode. Requirements for batch-friendly pcommands follow: -- Batch mode runs parallel pcommands in different background threads. - Commands must be ok with that. +- Batch mode runs parallel pcommands in different background threads + and may run thousands of commands throughout the duration of the + server process. Batch-friendly pcommands must behave reasonably in + such an environment. -- Batch-enabled pcommands must not look at sys.argv. They should instead - use pcommand.get_args(). Be aware that this value does not include - the first two values from sys.argv (executable path and pcommand name) - so is generally cleaner to use anyway. Also be aware that args are - thread-local, so only call get_args() from the thread your pcommand - was called in. - -- Batch-enabled pcommands should not call os.chdir() or sys.exit() or +- Batch-enabled pcommands must not call os.chdir() or sys.exit() or anything else having global effects. This should be self-explanatory considering the shared server model in use. -- Standard print and log calls will wind up in the pcommandbatch server - log and will not be seen by the user or capturable by the calling - process. By default, only a return code is passed back to the client, - where an error messages instructs the user to refer to the log or run - again with batch mode disabled. Commands that should print some output - even in batch mode can use pcommand.set_output() to do so. Note that - this is currently limited to small bits of output (but that can be - changed). +- Batch-enabled pcommands should not look at sys.argv. They should + instead use pcommand.get_args(). Be aware that this value does not + include the first two values from sys.argv (executable path and + pcommand name) so is generally cleaner to use anyway. Also be aware + that args are thread-local, so only call get_args() from the thread + your pcommand was called in. + +- Batch-enabled pcommands should not use efro.terminal.Clr for coloring + terminal output; instead they should use pcommand.clr() which properly + takes into account whether the *client* is running on a tty/etc. + +- Standard print and log calls (as well as those of child processes) + will wind up in the pcommandbatch server log and will not be seen by + the user or capturable by the calling process. For batch-friendly + printing, use pcommand.clientprint(). Note that, in batch mode, all + output will be printed on the client after the command completes and + stderr and stdout will be printed separately instead of intermingled. + If a pcommand is long-running and prints at multiple times while doing + its thing, it is probably not a good fit for batch-mode. """ from __future__ import annotations @@ -302,6 +308,7 @@ class Server: self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: """Handle a client.""" + import efro.terminal from efrotools.pcommand import run_client_pcommand request_id = self._next_request_id @@ -309,6 +316,9 @@ class Server: self._client_count_since_last_check += 1 self._running_client_count += 1 try: + logpath = self._worker_log_file_path.removeprefix( + f'{self._project_dir}/' + ) reqdata: dict = json.loads((await reader.read()).decode()) assert isinstance(reqdata, dict) argv: list[str] = reqdata['a'] @@ -324,17 +334,26 @@ class Server: file=sys.stderr, ) + # Note: currently just using the 'isatty' value from the + # client. ideally should expand the client-side logic to + # exactly match what efro.terminal.Clr does locally. + clr: type[efro.terminal.ClrBase] = ( + efro.terminal.ClrAlways if isatty else efro.terminal.ClrNever + ) + try: if self._server_error is not None: resultcode = 1 - output = self._server_error + stdout_output = '' + stderr_output = self._server_error else: ( resultcode, - output, + stdout_output, + stderr_output, ) = await asyncio.get_running_loop().run_in_executor( None, - lambda: run_client_pcommand(argv, isatty), + lambda: run_client_pcommand(argv, clr, logpath), ) if VERBOSE: print( @@ -352,18 +371,17 @@ class Server: file=sys.stderr, ) traceback.print_exc() + stdout_output = '' + stderr_output = ( + f"internal pcommandbatch error; see log at '{logpath}'." + ) resultcode = 1 - logpath = self._worker_log_file_path.removeprefix( - f'{self._project_dir}/' - ) - output = ( - f'Error: pcommandbatch command failed: {argv}.' - f" For more info, see '{logpath}', or rerun with" - ' env var BA_PCOMMANDBATCH_DISABLE=1 to see' - ' all output here.\n' - ) - writer.write(json.dumps({'r': resultcode, 'o': output}).encode()) + writer.write( + json.dumps( + {'r': resultcode, 'o': stdout_output, 'e': stderr_output} + ).encode() + ) writer.close() await writer.wait_closed() diff --git a/tools/efrotools/pcommands.py b/tools/efrotools/pcommands.py index f7e0d3a6..ae59b150 100644 --- a/tools/efrotools/pcommands.py +++ b/tools/efrotools/pcommands.py @@ -503,7 +503,7 @@ def compile_mesh_file() -> None: # Show project-relative paths when possible. relpath = os.path.abspath(dst).removeprefix(f'{pcommand.PROJROOT}/') - pcommand.set_output(f'Compiling mesh: {relpath}') + pcommand.clientprint(f'Compiling mesh: {relpath}') os.makedirs(os.path.dirname(dst), exist_ok=True) subprocess.run([makebob, src, dst], check=True) @@ -525,7 +525,7 @@ def compile_collision_mesh_file() -> None: # Show project-relative paths when possible. relpath = os.path.abspath(dst).removeprefix(f'{pcommand.PROJROOT}/') - pcommand.set_output(f'Compiling collision mesh: {relpath}') + pcommand.clientprint(f'Compiling collision mesh: {relpath}') os.makedirs(os.path.dirname(dst), exist_ok=True) subprocess.run([makebob, src, dst], check=True) @@ -556,7 +556,7 @@ def compile_python_file() -> None: # Show project-relative path when possible. relpath = os.path.abspath(fname).removeprefix(f'{pcommand.PROJROOT}/') - pcommand.set_output(f'Compiling script: {relpath}') + pcommand.clientprint(f'Compiling script: {relpath}') py_compile.compile( fname, @@ -583,7 +583,7 @@ def _simple_file_copy(msg: str, make_unwritable: bool = False) -> None: src, dst = args relpath = os.path.abspath(dst).removeprefix(f'{pcommand.PROJROOT}/') - pcommand.set_output(f'{msg}: {relpath}') + pcommand.clientprint(f'{msg}: {relpath}') # If we're making built files unwritable, we need to blow # away exiting ones to allow this to succeed. @@ -723,23 +723,21 @@ def echo() -> None: Prints a Clr.RST at the end so that can be omitted. """ - from efro.terminal import Clr + clr = pcommand.clr() - pcommand.disallow_in_batch() - - clrnames = {n for n in dir(Clr) if n.isupper() and not n.startswith('_')} + clrnames = {n for n in dir(clr) if n.isupper() and not n.startswith('_')} first = True out: list[str] = [] - for arg in sys.argv[2:]: + for arg in pcommand.get_args(): if arg in clrnames: - out.append(getattr(Clr, arg)) + out.append(getattr(clr, arg)) else: if not first: out.append(' ') first = False out.append(arg) - out.append(Clr.RST) - print(''.join(out)) + out.append(clr.RST) + pcommand.clientprint(''.join(out)) def urandom_pretty() -> None: diff --git a/tools/pcommand b/tools/pcommand index f5b19bfe..0100cf6f 100755 --- a/tools/pcommand +++ b/tools/pcommand @@ -138,5 +138,12 @@ from batools.pcommands2 import ( # pylint: enable=unused-import +def foof() -> None: + """test.""" + from efro.error import CleanError + + raise CleanError('foo') + + if __name__ == '__main__': pcommand.pcommand_main(globals())