From 2b6df3f981a5e830bf7b3a0d72f4cc24ca39a388 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 2 Jul 2024 21:35:23 -0700 Subject: [PATCH] wired asset-packages into local asset builds --- .efrocachemap | 56 +++--- CHANGELOG.md | 2 +- config/projectconfig.json | 2 +- src/assets/Makefile | 3 + src/assets/ba_data/python/baenv.py | 2 +- src/ballistica/base/assets/assets.cc | 47 +++-- src/ballistica/base/graphics/texture/dds.cc | 4 +- src/ballistica/shared/ballistica.cc | 2 +- tools/batools/bacloud.py | 5 +- tools/batools/build.py | 8 + tools/batools/pcommands2.py | 25 +-- tools/batools/staging.py | 186 ++++++++++++++++++-- tools/efrotools/lazybuild.py | 11 ++ 13 files changed, 270 insertions(+), 83 deletions(-) diff --git a/.efrocachemap b/.efrocachemap index deed474f..7f15e759 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4038,26 +4038,26 @@ "build/assets/windows/Win32/ucrtbased.dll": "2def5335207d41b21b9823f6805997f1", "build/assets/windows/Win32/vc_redist.x86.exe": "b08a55e2e77623fe657bea24f223a3ae", "build/assets/windows/Win32/vcruntime140d.dll": "865b2af4d1e26a1a8073c89acb06e599", - "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "a39e78f94f2257360a0b5891ccb1e6f3", - "build/prefab/full/linux_arm64_gui/release/ballisticakit": "3fdd6d2e093dbe55c66f863790fe0712", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "ba679c627647597ce1ae0ee3c79bc628", - "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "85e1e9d4f28402201a8fba3fdc444a94", - "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "f96218be024fb0f54d875b3bb0d6695d", - "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "0e192113022123f6535d451d04bae7f6", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "bfbcc1a92c5e52aa44db0484d8b1e7e5", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "f52ff47413dc64627347a1ada68b2e1b", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "32b17d80c59ea676f3a548b2ec8799a5", - "build/prefab/full/mac_arm64_gui/release/ballisticakit": "92f821766087c12f7ecf54a5f1505a1a", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "f0c77f8eb0c5978ed6d1ff11ffc22a3f", - "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "538e04846ec2e3b4c33f1a211ac4e11a", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "373b7d863584936cb507dbb7da674949", - "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "8ba2796b27a6596fb67c27360b7e7239", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "432074c79906f984b1df221398139cf9", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "c246914f53747c2c94d3b3b32ab3fab7", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "6e2490e0a43a69c7983a39817117d77c", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "2d8bef376bf4ab160089f477609d5421", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "7d78f5d122e58c4e5a3264ad6aae1289", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "ee97d4b95248d68681eda85b658b7531", + "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "fafef337dc36981a73a7c0fb4793e1aa", + "build/prefab/full/linux_arm64_gui/release/ballisticakit": "b0aa1c5f84dc14b5eec5416e0c42c2dd", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "fd27f2e7e072311db6e76a310e5b3248", + "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "7ab1218f73a6bc9c1f785ffd7cceba24", + "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "4ba1f3ee76aa94ea10d9106b2882f477", + "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "4f5bcafa8fcea60c202f57392b790749", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "5a204e6e6fc09c7601f82ee5dbf4ac5b", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "0f922df6490548a53bb6a871a018f140", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "8830cb31c8cf6fb67691bdf697d67b2d", + "build/prefab/full/mac_arm64_gui/release/ballisticakit": "b9aa5cea9726fcdd6e6468d065b7a5f0", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "9d4a199b6ec1dc775ef8835bf185b576", + "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "7f51dec440a47bf9468c73795f6f6faf", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "7de19401a7bd5180fccca69ca3deabce", + "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "8ee6a1cd92f095e73756798c228fabb9", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "4fdadf7c85bc4025b1e89a78d136f163", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "4cabc0639011c73a7957aeae844df6fe", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "66bb1553c81ba8ee21adbb2cb954530d", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "c577ee1c5c4f08530fad64e8fefe4cf8", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "30212d030049d0fb89e3aa0cc97d0e0a", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "0a2b976285806c698f01e641f9074aca", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "f231b10895bdcb542de87b887ca181fd", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "ae936a119668ede7b36f38c8672f4bf8", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "f231b10895bdcb542de87b887ca181fd", @@ -4074,14 +4074,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "efffc4f330e77530accd9a9f82840a6c", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "c20363fe2af3d54e666b1c8ee67f6b76", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "efffc4f330e77530accd9a9f82840a6c", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "f012fac2242e90a5df21ee4b50004248", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "c2590ba0ed0426a8a8aae627d00d5447", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "846e75a2a07ec8c17f2e6eb622c456ee", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "d70c4ed329cc01aecb41eb2fa8f95130", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "de6384ade5417d115d5b6a3ff89d3bb8", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "b564e1ebdb709343ce66bf63cd1b0568", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "ab388c54bdcc895bc99414aec7d40273", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "f1bab1ce15ef218c191d0e85444f265e", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "52b68c71bcb167a388b89352c09dbbe8", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "36f3e21b79ff41c8bbe9fa797833c0d8", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "bc1f287cdcee0ad7839f2cdc5f01a6e4", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "22c3b0ec1cabf246c7dee9be3e0ef48b", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "fa12a11770061d9a34e6b4bd35930842", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "6770c990a7a562c796e74b13247f62d0", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "cd6b6dcc26978740733af1dc99d92810", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "c3d76e035872d841beb85a558bfca107", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "5548f407d97e380069f6c596c4e36cd7", "src/ballistica/base/mgen/pyembed/binding_base.inc": "efa61468cf098f77cc6a234461d8b86d", diff --git a/CHANGELOG.md b/CHANGELOG.md index 90ba8302..ac1fb89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -### 1.7.36 (build 21899, api 8, 2024-07-01) +### 1.7.36 (build 21902, api 8, 2024-07-02) - bacloud workspace commands are now a bit smarter; you can now do things like `bacloud workspace put .` or even just `bacloud workspace put` and it will work. Previously such cases required explicitly passing the workspace name diff --git a/config/projectconfig.json b/config/projectconfig.json index e32dc36f..85dc6c6e 100644 --- a/config/projectconfig.json +++ b/config/projectconfig.json @@ -1,5 +1,5 @@ { - "assets": "a-0.assets1.dev", + "assets": "a-0.bastdassets.5", "code_source_dirs": [ "src/ballistica" ], diff --git a/src/assets/Makefile b/src/assets/Makefile index c0f5b20b..f74e19ea 100644 --- a/src/assets/Makefile +++ b/src/assets/Makefile @@ -149,6 +149,9 @@ endif ASSET_TARGETS_WIN_WIN32 += $(EXTRAS_TARGETS_WIN_WIN32) ASSET_TARGETS_WIN_X64 += $(EXTRAS_TARGETS_WIN_X64) +# Asset Packages +ASSET_TARGETS_CMAKE += $(PROJ_DIR)/.cache/assetmanifests/gui_desktop_v1 + # Note: Code below needs updating when Python version changes (currently 3.11) define make-opt-pyc-target $1: $$(subst /__pycache__,,$$(subst .cpython-312.opt-1.pyc,.py,$1)) diff --git a/src/assets/ba_data/python/baenv.py b/src/assets/ba_data/python/baenv.py index 1e5c738a..551e1ce6 100644 --- a/src/assets/ba_data/python/baenv.py +++ b/src/assets/ba_data/python/baenv.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: # Build number and version of the ballistica binary we expect to be # using. -TARGET_BALLISTICA_BUILD = 21899 +TARGET_BALLISTICA_BUILD = 21902 TARGET_BALLISTICA_VERSION = '1.7.36' diff --git a/src/ballistica/base/assets/assets.cc b/src/ballistica/base/assets/assets.cc index 0223e911..3b0a4db3 100644 --- a/src/ballistica/base/assets/assets.cc +++ b/src/ballistica/base/assets/assets.cc @@ -1064,14 +1064,16 @@ auto Assets::FindAssetFile(FileType type, assert(g_base->InLogicThread()); const char* ext = ""; - const char* prefix = ""; + const char* prefix1 = ""; + const char* prefix2 = ""; switch (type) { case FileType::kSound: if (g_core->HeadlessMode()) { return "headless_dummy_path.sound"; } - prefix = "audio/"; + prefix1 = "audio/"; + prefix2 = "audio2/"; ext = ".ogg"; break; @@ -1079,17 +1081,20 @@ auto Assets::FindAssetFile(FileType type, if (g_core->HeadlessMode()) { return "headless_dummy_path.mesh"; } - prefix = "meshes/"; + prefix1 = "meshes/"; + prefix2 = "meshes2/"; ext = ".bob"; break; case FileType::kCollisionMesh: - prefix = "meshes/"; + prefix1 = "meshes/"; + prefix2 = "meshes2/"; ext = ".cob"; break; case FileType::kData: - prefix = "data/"; + prefix1 = "data/"; + prefix2 = "data2/"; ext = ".json"; break; @@ -1109,7 +1114,8 @@ auto Assets::FindAssetFile(FileType type, // g_base->graphics_server->texture_compression_types_are_set()); // assert(g_base->graphics_server // && g_base->graphics_server->texture_quality_set()); - prefix = "textures/"; + prefix1 = "textures/"; + prefix2 = "textures2/"; #if BA_OSTYPE_ANDROID && !BA_ANDROID_DDS_BUILD // On most android builds we go for .ktx, which contains etc2 and etc1. @@ -1130,20 +1136,23 @@ auto Assets::FindAssetFile(FileType type, const std::vector& asset_paths_used = asset_paths_; for (auto&& i : asset_paths_used) { - file_out = i + "/" + prefix + name + ext; // NOLINT - bool exists; + // TEMP - try our '2' stuff first. + for (auto&& prefix : {prefix2, prefix1}) { + file_out = i + "/" + prefix + name + ext; // NOLINT + bool exists; - // '#' denotes a cube map texture, which is actually 6 files. - if (strchr(file_out.c_str(), '#')) { - // Just look for one of them i guess. - std::string tmp_name = file_out; - tmp_name.replace(tmp_name.find('#'), 1, "_+x"); - exists = g_core->platform->FilePathExists(tmp_name); - } else { - exists = g_core->platform->FilePathExists(file_out); - } - if (exists) { - return file_out; + // '#' denotes a cube map texture, which is actually 6 files. + if (strchr(file_out.c_str(), '#')) { + // Just look for one of them i guess. + std::string tmp_name = file_out; + tmp_name.replace(tmp_name.find('#'), 1, "_+x"); + exists = g_core->platform->FilePathExists(tmp_name); + } else { + exists = g_core->platform->FilePathExists(file_out); + } + if (exists) { + return file_out; + } } } diff --git a/src/ballistica/base/graphics/texture/dds.cc b/src/ballistica/base/graphics/texture/dds.cc index 80a80921..b47aebd1 100644 --- a/src/ballistica/base/graphics/texture/dds.cc +++ b/src/ballistica/base/graphics/texture/dds.cc @@ -34,7 +34,9 @@ void LoadDDS(const std::string& file_name, unsigned char** buffers, int* widths, (*base_level) = 0; FILE* f = g_core->platform->FOpen(file_name.c_str(), "rb"); - if (!f) throw Exception("can't open file: \"" + file_name + "\""); + if (!f) { + throw Exception("can't open file: \"" + file_name + "\""); + } DDS_header hdr{}; diff --git a/src/ballistica/shared/ballistica.cc b/src/ballistica/shared/ballistica.cc index 3addeeeb..b0448c57 100644 --- a/src/ballistica/shared/ballistica.cc +++ b/src/ballistica/shared/ballistica.cc @@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int { namespace ballistica { // These are set automatically via script; don't modify them here. -const int kEngineBuildNumber = 21899; +const int kEngineBuildNumber = 21902; const char* kEngineVersion = "1.7.36"; const int kEngineApiVersion = 8; diff --git a/tools/batools/bacloud.py b/tools/batools/bacloud.py index 90ef4329..4238f95f 100755 --- a/tools/batools/bacloud.py +++ b/tools/batools/bacloud.py @@ -88,9 +88,10 @@ class App: # Make sure we can locate the project bacloud is being run from. self._project_root = Path(sys.argv[0]).parents[1] + # Look for a few things we expect to have in a project. if not all( - Path(self._project_root, name).is_dir() - for name in ('tools', 'config', 'tests') + Path(self._project_root, name).exists() + for name in ['config/projectconfig.json', 'tools/batools'] ): raise CleanError('Unable to locate project directory.') diff --git a/tools/batools/build.py b/tools/batools/build.py index 2dac969e..52ec13ee 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -253,6 +253,14 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'tools', 'src/assets', '.efrocachemap', + # Needed to rebuild on asset-package changes. + 'config/projectconfig.json', + ], + # This file won't exist if we are using a dev asset-package, + # in which case we want to always run so we can ask the + # server for package updates each time. + srcpaths_exist=[ + '.cache/asset_package_resolved', ], command=command, filefilter=_filefilter, diff --git a/tools/batools/pcommands2.py b/tools/batools/pcommands2.py index be493892..96bb56cf 100644 --- a/tools/batools/pcommands2.py +++ b/tools/batools/pcommands2.py @@ -670,13 +670,18 @@ def asset_package_assemble() -> None: f'Expected a string asset-package-version; got {type(apversion)}.' ) - subprocess.run( - [ - f'{pcommand.PROJROOT}/tools/bacloud', - 'assetpackage', - '_assemble', - apversion, - flavor, - ], - check=True, - ) + try: + subprocess.run( + [ + f'{pcommand.PROJROOT}/tools/bacloud', + 'assetpackage', + '_assemble', + apversion, + flavor, + ], + check=True, + ) + except Exception as exc: + raise CleanError( + f'Failed to assemble {apversion}' f' ({flavor} flavor).' + ) from exc diff --git a/tools/batools/staging.py b/tools/batools/staging.py index f2d4155b..ead574b9 100755 --- a/tools/batools/staging.py +++ b/tools/batools/staging.py @@ -17,7 +17,7 @@ from efrotools.util import is_wsl_windows_build_path from efrotools.pyver import PYVER if TYPE_CHECKING: - pass + from concurrent.futures import Future # Suffix for the pyc files we include in stagings. We're using # deterministic opt pyc files; see PEP 552. @@ -47,6 +47,7 @@ class AssetStager: self.src = f'{self.projroot}/build/assets' self.dst: str | None = None self.serverdst: str | None = None + self.asset_package_flavor: str | None = None self.win_extras_src: str | None = None self.win_platform: str | None = None self.win_type: str | None = None @@ -115,8 +116,21 @@ class AssetStager: if self.win_extras_src is not None: self._sync_windows_extras() - # Standard stuff in ba_data. - self._sync_ba_data() + # Legacy assets going into ba_data. + self._sync_ba_data_legacy() + + # New asset-package stuff going into ba_data. + if self.asset_package_flavor is not None: + self._sync_ba_data_new() + + if self.include_binary_executable: + self._sync_binary_executable() + + if self.include_python_dylib: + self._sync_python_dylib() + + if self.include_shell_executable: + self._sync_shell_executable() # On Android we need to build a payload file so it knows what to # pull out of the apk. @@ -170,6 +184,7 @@ class AssetStager: self.desc = 'cmake' self.dst = args[-1] self.tex_suffix = '.dds' + self.asset_package_flavor = 'gui_desktop_v1' # Link/copy in a binary *if* builddir is provided. self.include_binary_executable = self.builddir is not None self.executable_name = 'ballisticakit' @@ -449,8 +464,7 @@ class AssetStager: ] subprocess.run(cmd, check=True) - def _sync_ba_data(self) -> None: - # pylint: disable=too-many-branches + def _sync_ba_data_legacy(self) -> None: assert self.dst is not None os.makedirs(f'{self.dst}/ba_data', exist_ok=True) cmd: list[str] = [ @@ -461,15 +475,16 @@ class AssetStager: '--prune-empty-dirs', ] - # Normally we use --delete-excluded so that we can do sparse - # syncs for quick iteration on android apks/etc. However for our - # modular builds we need to avoid that flag because we do a - # second pass after to sync in our python-dylib stuff and with - # that flag it all gets blown on the first pass. - if not self.include_python_dylib: + # Traditionally we used --delete-excluded so that we could do + # sparse syncs for quick iteration on android apks/etc. However + # for our modular builds (and now for asset-package assets) we + # need to avoid that flag because we do further passes after to + # sync in python-dylib stuff or asset-package stuff and with that + # flag it all gets blown away on the first pass. + if not self.include_python_dylib and self.asset_package_flavor is None: cmd.append('--delete-excluded') else: - # Shouldn't be trying to do sparse stuff. + # Shouldn't be trying to do sparse stuff in server builds. if self.serverdst is not None: assert self.include_json and self.include_collision_meshes else: @@ -481,8 +496,9 @@ class AssetStager: and self.include_meshes and self.include_collision_meshes ) - # Keep rsync from trying to prune this as an 'empty' dir. + # Keep rsync from deleting the other stuff we're overlaying. cmd += ['--exclude', '/python-dylib'] + cmd += ['--exclude', '/textures2'] if self.include_scripts: cmd += [ @@ -524,14 +540,146 @@ class AssetStager: ] subprocess.run(cmd, check=True) - if self.include_binary_executable: - self._sync_binary_executable() + def _sync_ba_data_new(self) -> None: + # pylint: disable=too-many-locals + import json + import stat + import shutil + from threading import Lock + from concurrent.futures import ThreadPoolExecutor - if self.include_python_dylib: - self._sync_python_dylib() + from bacommon.bacloud import asset_file_cache_path - if self.include_shell_executable: - self._sync_shell_executable() + assert self.asset_package_flavor is not None + assert self.dst is not None + + # Just going with raw json here instead of dataclassio to + # maximize speed; we'll be going over lots of files here. + with open( + f'.cache/assetmanifests/{self.asset_package_flavor}', + encoding='utf-8', + ) as infile: + manifest = json.loads(infile.read()) + + filehashes: dict[str, str] = manifest['h'] + + mkdirlock = Lock() + + def _prep_syncdir(syncdir: str) -> None: + # First, take a pass through and delete all files not found in + # our manifest. + assert self.dst is not None + dstdir = os.path.join(self.dst, syncdir) + os.makedirs(dstdir, exist_ok=True) + for entry in os.scandir(dstdir): + if entry.is_file(): + path = os.path.join(syncdir, entry.name) + if path not in filehashes: + os.unlink(os.path.join(self.dst, path)) + + def _sync_path(src: str, dst: str) -> None: + # Quick-out: if there's a file already at dst and its + # modtime and size *exactly* match src, we're done. Note + # that this is a bit different than Makefile logic where + # things update when src is newer than dst. In our case, + # a manifest change could cause src to point to a cache + # file with an *older* modtime than the previous one + # (cache file modtimes are static and arbitrary) so such + # logic doesn't work. However if we look for an *exact* + # modtime match as well as size match we can be + # reasonably sure that the file is still the same. We'll + # see how this goes... + srcstat = os.stat(src) + try: + dststat = os.stat(dst) + except FileNotFoundError: + dststat = None + if ( + dststat is not None + and srcstat.st_size == dststat.st_size + and srcstat.st_mtime == dststat.st_mtime + ): + return + + # If dst is a directory, blow it away (use the stat we + # already fetched to save a bit of time). + if dststat is not None and stat.S_ISDIR(dststat.st_mode): + shutil.rmtree(dst) + + # Hold a lock while creating any parent directories just in + # case multiple files are trying to create the same + # directory simultaneously (not sure if that could cause + # problems but might as well be extra safe). + with mkdirlock: + os.makedirs(os.path.dirname(dst), exist_ok=True) + + # Ok, dst doesn't exist or modtimes don't line up. Copy it + # and try to copy its modtime. + shutil.copyfile(src, dst) + shutil.copystat(src, dst) + + def _cleanup_syncdir(syncdir: str) -> None: + """Handle pruning empty directories.""" + # Walk the tree bottom-up so we can properly kill recursive + # empty dirs. + assert self.dst is not None + dstdir = os.path.join(self.dst, syncdir) + for basename, dirnames, filenames in os.walk(dstdir, topdown=False): + # It seems that child dirs we kill during the walk are still + # listed when the parent dir is visited, so lets make sure + # to only acknowledge still-existing ones. + dirnames = [ + d + for d in dirnames + if os.path.exists(os.path.join(basename, d)) + ] + if not dirnames and not filenames and basename != dstdir: + os.rmdir(basename) + + syncdirs: list[str] = ['ba_data/textures2'] + + futures: list[Future] + with ThreadPoolExecutor(max_workers=4) as executor: + + # First, prep each of our sync dirs (make sure they exist, + # blow away files not in our manifest, etc.) + futures = [] + for syncdir in syncdirs: + futures.append(executor.submit(_prep_syncdir, syncdir=syncdir)) + # Await all results to get any exceptions. + for future in futures: + _result = future.result() + + # Now go through all our manifest paths, syncing any files + # destined for any of our syncdirs. + futures = [] + for path, hashval in filehashes.items(): + for syncdir in syncdirs: + if path.startswith(syncdir): + futures.append( + executor.submit( + _sync_path, + src=( + f'.cache/assetdata/' + f'{asset_file_cache_path(hashval)}' + ), + dst=os.path.join(self.dst, path), + ) + ) + # Await all results to get any exceptions. + for future in futures: + _result = future.result() + + # Lastly, run a cleanup pass on all our sync dirs (blow away + # empty dirs, etc.) + futures = [] + for syncdir in syncdirs: + futures.append( + executor.submit(_cleanup_syncdir, syncdir=syncdir) + ) + # Await all results to get any exceptions. + for future in futures: + _result = future.result() def _sync_shell_executable(self) -> None: if self.executable_name is None: diff --git a/tools/efrotools/lazybuild.py b/tools/efrotools/lazybuild.py index f2a2b7c4..f61581dc 100644 --- a/tools/efrotools/lazybuild.py +++ b/tools/efrotools/lazybuild.py @@ -52,11 +52,13 @@ class LazyBuildContext: dirfilter: Callable[[str, str], bool] | None = None, filefilter: Callable[[str, str], bool] | None = None, srcpaths_fullclean: list[str] | None = None, + srcpaths_exist: list[str] | None = None, manifest_file: str | None = None, command_fullclean: str | None = None, ) -> None: self.target = target self.srcpaths = srcpaths + self.srcpaths_exist = srcpaths_exist self.command = command self.dirfilter = dirfilter self.filefilter = filefilter @@ -158,6 +160,7 @@ class LazyBuildContext: Path(self.target).touch() def _check_for_changes(self) -> None: + # pylint: disable=too-many-branches manfile = self.manifest_file # If we're watching for file adds/removes/renames in addition # to just modtimes, build a set of all files we come across. @@ -186,6 +189,7 @@ class LazyBuildContext: src_did_change, src_unchanged_count = self._check_path( srcpath, man_input_paths ) + if src_did_change: self.have_changes = True # If we're *not* building a manifest @@ -194,6 +198,13 @@ class LazyBuildContext: return self.total_unchanged_count += src_unchanged_count + # Check our exist-only paths; we simply look to see if these exist + # and build if not. + if self.srcpaths_exist is not None: + for srcpath in self.srcpaths_exist: + if not os.path.exists(srcpath): + self.have_changes = True + # If we built a manifest, check/write it and kick off # a full-clean if anything differed. if manfile is not None: