From 66bf931b46b4a0b006d482dc5056b29a00363e47 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 1 Sep 2023 12:11:41 -0700 Subject: [PATCH] more starter-cache inclusion tweaks --- .efrocachemap | 32 +++++++++++++++---------------- tools/efrotools/efrocache.py | 37 ++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/.efrocachemap b/.efrocachemap index b406d953..530fddc6 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4072,18 +4072,18 @@ "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "ab0cb59c874beb50e554d3d7c552565e", "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "dec2c09bd136aa413b9b94729f7d6d2e", "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "a5c43eb8befc810d693ec9c9f83fcba4", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "091ee6cb6bb2cd6ff6c28a1a487488df", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "92bef80f4f6a5feda383035470ea2131", "build/prefab/full/mac_arm64_gui/release/ballisticakit": "d6a10e3a6bc1a07609598f962439c6c8", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "c3f9f9b30a84a3d93cefdac1efc6bb4c", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "0150dcf48cb6a34adecaaa27d8940d29", "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "231bd6e18145b06028e4f90fd7de106c", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "7af10f955a2a010332fa67daf9971e3b", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "27c074268b5d1665ad45af2b5d62d03c", "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "4586629e8f114c1b5a22444ae26b5917", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "b6bc12d306e0a7c40f0f8d676dff1736", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "2a776f145931be151cfc2f90907a1ddd", "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "518c529a7133d5ea41d730d12ae156c4", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "bc2734437901fc1e2f236a10000dadbf", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "91e559906c51edd86def448b36341f4c", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "ccb3e7da1cfe95ed2038635e65d493c0", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "35adf9afb9a8e2e28ac73e2480328078", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "dc1424b526bd15a53a1f26346416b09e", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "8ee060e63821fc3f0775ffaca476d3b5", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "33ca30ccfb51b1b047f5564edb505e79", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "174581fbf846e3f14c3ef6f139ceefa1", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "a3607fd941915ab11503f82acfc392b5", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "b5a129d83796c9e7015ab5e319d2c22f", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "a3607fd941915ab11503f82acfc392b5", @@ -4100,14 +4100,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "c5c40967e63471c9c4abd6dfbef892df", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "d34c0a142e7d391a109a33ea3cc77c08", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "c5c40967e63471c9c4abd6dfbef892df", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "f2e326710ed9b4d60b776cc466dcd6ef", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "cf3eda1d6dde5e2940d3c95c189b81c7", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "34c74a370208dcb7c04895faf5663f44", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "a2e89d0e98d234e55eb0cbe3aa3fdeb2", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "07437d8692539b74468b6d5603bc6a84", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "c85405193970b0c2ce78b1ec06016cbf", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "534b291ceec828449a9fabbefe131b43", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "4ede5778fde3aba2392d89226aaad032", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "88b6da386b4044afa482da6ad85f704d", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "d2cbb43b7a5e4f7ddb7d70703f49ddf1", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "997d010eeae0181b249f5936a891e9b1", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "0b8e42d5d61d953ea4e0df9b435a795a", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "dd5e4e4b58e59054330a905b35337c67", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "8036025f45cb9c536dfe9fec83c6ef21", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "ef0330100297333917d1b68b98c002b2", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "f3ca2b3dd2fd16bfe3e5253b087a6c28", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "f8cd3af311ac63147882590123b78318", "src/ballistica/base/mgen/pyembed/binding_base.inc": "ad347097a38e0d7ede9eb6dec6a80ee9", diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 42899a2f..a716bd24 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -298,6 +298,7 @@ def filter_makefile(makefile_dir: str, contents: str) -> str: def update_cache(makefile_dirs: list[str]) -> None: """Given a list of directories containing Makefiles, update caches.""" # pylint: disable=too-many-locals + # pylint: disable=too-many-branches import multiprocessing @@ -309,9 +310,9 @@ def update_cache(makefile_dirs: list[str]) -> None: fnames_starter_server: list[str] = [] fnames_all: list[str] = [] - # If a path contains any of these substrings it will be included in - # the server starter cache. - add_to_server_starter_paths = { + # If a path contains any of these substrings it will always be included + # in starter caches. + starter_cache_always_include_paths = { 'build/assets/ba_data/fonts', 'build/assets/ba_data/data', 'build/assets/ba_data/python', @@ -373,6 +374,11 @@ def update_cache(makefile_dirs: list[str]) -> None: # Now selectively add to starter cache lists. + always_include = False + + if any(p in fullpath for p in starter_cache_always_include_paths): + always_include = True + # Always keep certain file types out of starter caches. if any( fullpath.endswith(ending) @@ -380,30 +386,33 @@ def update_cache(makefile_dirs: list[str]) -> None: ): continue - # Keep big files out of starter caches. The main benefits of - # the cache is that we can reduce the overhead for - # downloading individual tiny files by grabbing them all at - # once, but that advantage diminishes as the files get - # bigger. And not all platforms will use all files, so it - # generally more efficient to grab bigger ones as needed. - if os.path.getsize(fullpath) > 300_000: + # Keep big files out of starter caches (unless flagged as + # always-include). The main benefits of starter-caches is + # that we can reduce the overhead for downloading individual + # tiny files by grabbing them all at once, but that + # advantage diminishes as the files get bigger. And not all + # platforms will use all files, so it generally more + # efficient to grab bigger ones as needed. + if os.path.getsize(fullpath) > 50_000 and not always_include: continue # Gui starter gets everything that made it this far. fnames_starter_gui.append(fullpath) - # For server starter, limit to a few key dirs. - if not any(p in fullpath for p in add_to_server_starter_paths): + # Server starter cuts out everything not explicitly + # always-included. + if not always_include: continue - # Also exclude some endings from server starters. + # Server starter also exclude some things from within + # always-included dirs. if any( fullpath.endswith(ending) for ending in never_add_to_starter_endings_server ): continue - # Ok this one qualifies for the server starter too. + # If it made it this far, add it to the server cache. fnames_starter_server.append(fullpath) # Ok, we've got a big list of filenames we need to cache in the