diff --git a/.efrocachemap b/.efrocachemap index 8c6c568a..856e00b0 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4046,18 +4046,18 @@ "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "c6e86f716993e4db6d40b387c1887bf9", "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "fdb8c01b631d5475740f0101b301ff03", "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "8c58b1dff5c711bf8dfe6bcc0c033e02", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "2a38697e7097b33740af6cd8da8b1ae6", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "68b0c339ca50a9d498f044d1b9c25665", "build/prefab/full/mac_arm64_gui/release/ballisticakit": "4bd338604cf6288636422ef3b62e5d44", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "105b144a26c33ee82548ba7f66f8caf7", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "d585e0b8f0a2a1edefd52b9d9fb2fd40", "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "316e826a593499760060c1c2c09fe278", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "b28622b94e3193e73347df19619f88fe", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "7070657c31c247dac4d1b14607a8435f", "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "606f7ed5a63fad5acbdba62b8f2e953f", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "febd223d98d2a7dd66588024d9726a84", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "ccfa7d16a67a8b5e28b832861433b536", "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "f98f149307ee2096aa5c82fb666d96b2", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "c8ff03b720c5c07a46ad33d9b6de3a40", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "7e3c487f558a48828e574f96df894079", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "6214da7d6186d98512d848c29d4396af", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "8725657f87d3654febe0d4176711c2cf", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "b3d826dae3e8105e5507a50d53142b56", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "ffc3963828ad90d6da6237aa374909b9", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "21b69244af53ac854f513dbe574e9281", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "92edda56e8cc74c3552f87d766a83a8c", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "9162b53189b80d4e5938cd142acd2a1a", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "2226d0e2d34051d962320c86464f3a2d", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "9162b53189b80d4e5938cd142acd2a1a", @@ -4074,14 +4074,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "299b6c4b83966040618b27563b700c2a", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "08c2f91d086e6f3fa73ebc299112358a", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "299b6c4b83966040618b27563b700c2a", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "2507494bbe98a16131895f110448c8ea", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "61d67e343abf7925dac810e2811b1e10", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "b19937612b1f9db29d7e758af7915b76", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "b73edce181aaffd3d2b5c4af87303bde", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "cbbd994d39c7297d7779e0a3a1ef0435", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "661811a9424219211429d618d8c86d01", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "e84c108ccd7a9f7d1f0c840d2cc3c108", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "ea607325f919e5b375ee4ac41262fa45", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "50245294e53c993dd3a25db2868f22e0", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "84d98c5f0dbab599afd67a0fcf8e28d8", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "08f5b3b61056a65d7f39009426d08c48", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "3e4073e334438027320f3fcc2268357f", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "51ac5f179b08a8892580ba915452b40a", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "a61545d3d93c89c5298536c133745c5d", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "1a0c96f28a75527e975af1c023716f0f", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "0725628caf7f5baa9a59d1022f62a441", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "5548f407d97e380069f6c596c4e36cd7", "src/ballistica/base/mgen/pyembed/binding_base.inc": "efa61468cf098f77cc6a234461d8b86d", diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf7d4f6..b48f8f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,10 +74,14 @@ need to happen to keep these builds behaving, so I'd like to enforce as limited a set of conditions as possible to give us the best chance at succeeding there. -- Added a workaround for WSL builds giving permission errors when staging asset +- Added a workaround for WSL Windows builds giving permission errors when staging asset files that already exist. Please holler if you are building with WSL and still running into any sort of errors, as I would love to make that path as reliable as possible. +- Fixed an issue where WSL Windows builds would re-extract everything from + efrocache when anything in the cache-map changed (which is the case for most + commits). Please holler if you are still seeing lots more 'Extracting:' lines + when running builds after pulling small updates from git. ### 1.7.34 (build 21823, api 8, 2024-04-26) - Bumped Python version from 3.11 to 3.12 for all builds and project tools. One diff --git a/tools/batools/build.py b/tools/batools/build.py index 7b7dfb2c..940d8d19 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -700,24 +700,3 @@ def docker_build() -> None: version_num, build_num, ) - - -def wsl_windows_build_path_description() -> str: - """Describe where wsl windows builds need to live.""" - return 'anywhere under /mnt/c/' - - -def is_wsl_windows_build_path(path: str) -> bool: - """Return whether a path is used for wsl windows builds. - - Building Windows Visual Studio builds through WSL is currently only - supported in specific locations; namely anywhere under /mnt/c/. This - is enforced because building on the Linux filesystem errors due to - case-sensitivity issues, and also because a number of workarounds - need to be employed to deal with filesystem/permission quirks, so - we want to keep things as consistent as possible. - - Note that said quirk workarounds WILL be applied if this returns - true, so this check should be as specific as possible. - """ - return path.startswith('/mnt/c/') diff --git a/tools/batools/pcommands2.py b/tools/batools/pcommands2.py index 41361ee3..1607319c 100644 --- a/tools/batools/pcommands2.py +++ b/tools/batools/pcommands2.py @@ -423,7 +423,7 @@ def wsl_build_check_win_drive() -> None: import subprocess import textwrap from efro.error import CleanError - from batools.build import ( + from efrotools.util import ( is_wsl_windows_build_path, wsl_windows_build_path_description, ) diff --git a/tools/batools/staging.py b/tools/batools/staging.py index fc6ad046..e8af7421 100755 --- a/tools/batools/staging.py +++ b/tools/batools/staging.py @@ -12,11 +12,9 @@ from functools import partial from typing import TYPE_CHECKING from efro.terminal import Clr -from efrotools.util import extract_arg, extract_flag +from efrotools.util import extract_arg, extract_flag, is_wsl_windows_build_path from efrotools.pyver import PYVER -from batools.build import is_wsl_windows_build_path - if TYPE_CHECKING: pass diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index d2589bdf..c25cbf04 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -28,6 +28,9 @@ from efro.dataclassio import ( ) from efro.terminal import Clr +from efrotools.util import is_wsl_windows_build_path + + if TYPE_CHECKING: import efro.terminal @@ -664,7 +667,24 @@ def _cache_prefix_for_file(fname: str) -> bytes: # We'll be calling this a lot when checking existing files, so we # want it to be efficient. Let's cache the two options there are at # the moment. + executable = os.access(fname, os.X_OK) + + if is_wsl_windows_build_path(os.getcwd()): + # Currently the filesystem during wsl windows builds tells us + # everything is executable. Normally this causes us to + # re-extract most everything which is all non-executable in the + # cache. So as a band-aid let's just hard-code everything to + # give a non-executable result here instead so we only have to + # redundantly extract the few things that ARE executable instead + # of all the things that aren't. + + # Make ourself aware if this situation ever changes. + if not executable: + print('GOT WSL PATH NON-EXECUTABLE; NOT EXPECTED') + + executable = False + if executable: if g_cache_prefix_exec is None: metadata = dataclass_to_json( @@ -771,10 +791,16 @@ def warm_start_cache(cachetype: str) -> None: # Python process for each and every file we need to touch. In that # case, this optimization would probably be unnecessary. # - # UPDATE - we now have that lightweight build system (pcommandbatch) + # UPDATE - We now have that lightweight build system (pcommandbatch) # which means individual refreshes are now much less expensive than # before, so disabling this for now. - if bool(False): + # + # UPDATE 2 - I've disabled pcommandbatch by default so flipping this + # back on for now since it really helps in some cases such as WSL + # Windows builds which are painfully slow otherwise. Can consider + # turning the back off again once asset builds have migrated to + # the cloud asset-package system. + if bool(True): cachemap: dict[str, str] with open(CACHE_MAP_NAME, encoding='utf-8') as infile: cachemap = json.loads(infile.read()) diff --git a/tools/efrotools/util.py b/tools/efrotools/util.py index b0da6206..6694ce6f 100644 --- a/tools/efrotools/util.py +++ b/tools/efrotools/util.py @@ -189,3 +189,24 @@ def get_string_hash( return str(int.from_bytes(hashobj.digest(), byteorder='big')) return hashobj.hexdigest() + + +def wsl_windows_build_path_description() -> str: + """Describe where wsl windows builds need to live.""" + return 'anywhere under /mnt/c/' + + +def is_wsl_windows_build_path(path: str) -> bool: + """Return whether a path is used for wsl windows builds. + + Building Windows Visual Studio builds through WSL is currently only + supported in specific locations; namely anywhere under /mnt/c/. This + is enforced because building on the Linux filesystem errors due to + case-sensitivity issues, and also because a number of workarounds + need to be employed to deal with filesystem/permission quirks, so + we want to keep things as consistent as possible. + + Note that said quirk workarounds WILL be applied if this returns + true, so this check should be as specific as possible. + """ + return path.startswith('/mnt/c/')