From 46dda011a4f4940d361dc1d2288d655201579733 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 1 Sep 2023 10:32:40 -0700 Subject: [PATCH] adding server starter cache archive --- .efrocachemap | 56 ++++----- .idea/dictionaries/ericf.xml | 2 + CHANGELOG.md | 2 +- .../.idea/dictionaries/ericf.xml | 2 + src/assets/ba_data/python/baenv.py | 2 +- src/ballistica/shared/ballistica.cc | 2 +- tools/efrotools/efrocache.py | 112 +++++++++++++----- 7 files changed, 118 insertions(+), 60 deletions(-) diff --git a/.efrocachemap b/.efrocachemap index e44a3fcb..7576f237 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4064,26 +4064,26 @@ "build/assets/windows/Win32/ucrtbased.dll": "2def5335207d41b21b9823f6805997f1", "build/assets/windows/Win32/vc_redist.x86.exe": "b08a55e2e77623fe657bea24f223a3ae", "build/assets/windows/Win32/vcruntime140d.dll": "865b2af4d1e26a1a8073c89acb06e599", - "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "a44a6d1692466ce3ef645e287df9d40c", - "build/prefab/full/linux_arm64_gui/release/ballisticakit": "d10895a3d5db3eb87444c977d3cc7d5b", - "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "7c8673d3c53823f7337d9cc06363c52b", - "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "e0912c7058d289c72ececf1f82af05e9", - "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "08d0270c1ff62138d90cfd8f03c2a1f4", - "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "43fc3db1e9c0faea8583d728ba62c9cf", - "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "f0ce38fea6c063d526363df303a3eaff", - "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "ffa4b4d37c326be0f5fc90209aa826c4", - "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "0a73555124eb90070bced01136dea1f2", - "build/prefab/full/mac_arm64_gui/release/ballisticakit": "8f3b965f338dea1db2a87e761f178926", - "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "46526e8ca5d33be7828fc39c0c483058", - "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "c47bcafdc7a6ac65cd7e30878a68b56c", - "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "2b41adbceeb183e7024ca0f8afa0770e", - "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "837c943294f33316645071acf649d66c", - "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "09eb1243aaf52998ad182cff4efb7375", - "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "476f9d1a9ae840cb458d862460ba1c59", - "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "a4674b228aa7e966fabf4822594ea7dd", - "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "326cb9f1e606f0a106e9a346f23a4930", - "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "fd000a93901f067421d4c337ef788fa5", - "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "3c317e611a66b8f307c20b2893af19e6", + "build/prefab/full/linux_arm64_gui/debug/ballisticakit": "71d6137c5b76b2b3303843f75b133309", + "build/prefab/full/linux_arm64_gui/release/ballisticakit": "4a8365954ce522fbf4b88806556c05c6", + "build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "34f6f72380e8ddd78381720a3318cdfb", + "build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "b0018db87173b93897a4b0784080617f", + "build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "1f64b93ef8c96c1a08c8dd9df0ce59b6", + "build/prefab/full/linux_x86_64_gui/release/ballisticakit": "9215dd53e512b8b72cec42be17f7ad76", + "build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "156e9fb1070dbeb23b7ef2b23a42696f", + "build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "d577b91ba57cfc3f5853174c086891d0", + "build/prefab/full/mac_arm64_gui/debug/ballisticakit": "49d71ee8bed1860d8d13adaa846569ff", + "build/prefab/full/mac_arm64_gui/release/ballisticakit": "d59d87a16bb17e8a82a4046a5d58bce9", + "build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "5765f63cab7da22557612bd8f35c38c0", + "build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "b1d683392bccd43eaad266e89fd074ab", + "build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "81ed1fdf2e1f92503cf128ee5c7c4ff9", + "build/prefab/full/mac_x86_64_gui/release/ballisticakit": "9316816cd78ad2ee634b7fb169aee793", + "build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "18a1cf10489bb2a4b872018a68915326", + "build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "c8de3a1c97598e3c9bc5c44049d9a867", + "build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "1c74ebe6bc6436d10a7356ad72b0744a", + "build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "dfad3a32837a532b14798988cbd88114", + "build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "2af7e3281e4ea3e71e10b574a7d9cdf3", + "build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "e7560b950d01566bb3f744751c55a060", "build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "a3607fd941915ab11503f82acfc392b5", "build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "b5a129d83796c9e7015ab5e319d2c22f", "build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "a3607fd941915ab11503f82acfc392b5", @@ -4100,14 +4100,14 @@ "build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "c5c40967e63471c9c4abd6dfbef892df", "build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "d34c0a142e7d391a109a33ea3cc77c08", "build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "c5c40967e63471c9c4abd6dfbef892df", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "3539e7c953250aea0ce2b17444bf2416", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "70b578f8b50222347aeb7fdc4e232eae", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "8d04b891a8edc9f36789fadd18f8b44f", - "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "5963b640147b3a762f09860eefded894", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "e55e703e9920612601ada6011450b8e0", - "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "d983ab4d7130d3945c5a71e8b762e7f1", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "5cb3581574e84116b76333c89ca5550f", - "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "477579f388832b2d296c89c467b24094", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "508dcdd4f35c2c211e6431380ada9a6d", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "47de9ad2862b86b856ec65b6bbc41c86", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "01e15b11d87dbedd73739c64be81338e", + "build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "99d71b31725a4da1c82e85cadda43781", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "1a0d22823c2e31c87cd265c4ee3b0d34", + "build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "9a96d073c06835c2992a898cd81c2d0d", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "c4262788c2946b6274186590a03bdbd2", + "build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "c230e4d4727f8eb09104f1c7aed2f2ca", "src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c", "src/assets/ba_data/python/babase/_mgen/enums.py": "f8cd3af311ac63147882590123b78318", "src/ballistica/base/mgen/pyembed/binding_base.inc": "ad347097a38e0d7ede9eb6dec6a80ee9", diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 77c090b9..077d07ec 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -1028,6 +1028,7 @@ fflush fhash fhashes + fhashpaths fhdr fieldattr fieldname @@ -1326,6 +1327,7 @@ hashobj hashopenssl hashpath + hashpaths hashstr hashtype hashval diff --git a/CHANGELOG.md b/CHANGELOG.md index 65aa9a8d..4636e832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -### 1.7.28 (build 21295, api 8, 2023-09-01) +### 1.7.28 (build 21298, api 8, 2023-09-01) - Added some high level functionality for copying and deleting feature-sets to the `tools/spinoff` tool. For example, to create your own `poo` feature-set, diff --git a/ballisticakit-cmake/.idea/dictionaries/ericf.xml b/ballisticakit-cmake/.idea/dictionaries/ericf.xml index f1cbefcf..f68e5937 100644 --- a/ballisticakit-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticakit-cmake/.idea/dictionaries/ericf.xml @@ -639,6 +639,7 @@ fffffffffifff fflush fgets + fhashpaths fieldname fieldpath fifteenbits @@ -803,6 +804,7 @@ handfull handlemessage hashfiles + hashpaths hastouchscreen hatmotion haveint diff --git a/src/assets/ba_data/python/baenv.py b/src/assets/ba_data/python/baenv.py index 57297554..4c5c12d7 100644 --- a/src/assets/ba_data/python/baenv.py +++ b/src/assets/ba_data/python/baenv.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: # Build number and version of the ballistica binary we expect to be # using. -TARGET_BALLISTICA_BUILD = 21295 +TARGET_BALLISTICA_BUILD = 21298 TARGET_BALLISTICA_VERSION = '1.7.28' diff --git a/src/ballistica/shared/ballistica.cc b/src/ballistica/shared/ballistica.cc index 05883e42..3540e08f 100644 --- a/src/ballistica/shared/ballistica.cc +++ b/src/ballistica/shared/ballistica.cc @@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int { namespace ballistica { // These are set automatically via script; don't modify them here. -const int kEngineBuildNumber = 21295; +const int kEngineBuildNumber = 21298; const char* kEngineVersion = "1.7.28"; const int kEngineApiVersion = 8; diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 20296b09..664c533f 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -297,6 +297,7 @@ def filter_makefile(makefile_dir: str, contents: str) -> str: def update_cache(makefile_dirs: list[str]) -> None: """Given a list of directories containing Makefiles, update caches.""" + # pylint: disable=too-many-locals import multiprocessing @@ -304,8 +305,20 @@ def update_cache(makefile_dirs: list[str]) -> None: # Build a list of files going into our starter cache, files going # into our headless starter cache, and all files. - fnames_starter: list[str] = [] + fnames_starter_gui: list[str] = [] + fnames_starter_server: list[str] = [] fnames_all: list[str] = [] + + # If a path contains any of these substrings it will be included in + # the server starter cache. + server_starter_paths = { + 'build/assets/ba_data/fonts', + 'build/assets/ba_data/data', + 'build/assets/ba_data/python', + 'build/assets/ba_data/python-site-packages', + 'build/assets/ba_data/meshes', + } + for path in makefile_dirs: cdp = f'cd {path} && ' if path else '' @@ -340,12 +353,24 @@ def update_cache(makefile_dirs: list[str]) -> None: for rawpath in rawpaths: fullpath = _project_centric_path(os.path.join(path, rawpath)) - # The main reason for this cache is to reduce round trips to - # the staging server for tiny files, so let's include small - # files only here. For larger stuff its ok to have a request - # per file.. + # The main reason for starter caches is to reduce overhead + # for downloading individual tiny files, so let's include + # small files only in starter lists. For larger stuff, a + # request per file shouldn't be too inefficient. Also, + # prebuilt binaries tend to be larger and we don't want to + # include a bunch of binaries for other platforms that we + # won't use. if os.path.getsize(fullpath) < 100000: - fnames_starter.append(fullpath) + # Gui starter gets everything. + fnames_starter_gui.append(fullpath) + + # For server starter, limit to a few key dirs. + if any(p in fullpath for p in server_starter_paths): + # We include the meshes dir but we only want + # collision meshes; not display ones. + if not fullpath.endswith('.bob'): + fnames_starter_server.append(fullpath) + fnames_all.append(fullpath) # Ok, we've got a big list of filenames we need to cache in the @@ -364,7 +389,13 @@ def update_cache(makefile_dirs: list[str]) -> None: flush=True, ) else: - _update_cloud_cache(fnames_starter, fnames_all, hashes, hashes_existing) + _update_cloud_cache( + fnames_starter_gui, + fnames_starter_server, + fnames_all, + hashes, + hashes_existing, + ) print(f'{Clr.SBLU}Efrocache update successful!{Clr.RST}') @@ -392,7 +423,8 @@ def _gen_complete_state_hashes(fnames: list[str]) -> str: def _update_cloud_cache( - fnames_starter: list[str], + fnames_starter_gui: list[str], + fnames_starter_server: list[str], fnames_all: list[str], hashes_str: str, hashes_existing_str: str, @@ -423,11 +455,22 @@ def _update_cloud_cache( subprocess.run(['rm', '-rf', staging_dir], check=True) subprocess.run(['mkdir', '-p', staging_dir], check=True) - _gather_cache_files(fnames_starter, fnames_all, staging_dir, mapping_file) + _gather_cache_files( + fnames_starter_gui, + fnames_starter_server, + fnames_all, + staging_dir, + mapping_file, + ) print( - f'{Clr.SBLU}Starter cache includes {len(fnames_starter)} items;' - f' excludes {len(fnames_all) - len(fnames_starter)}{Clr.RST}' + f'{Clr.SBLU}Starter gui cache includes {len(fnames_starter_gui)} items;' + f' excludes {len(fnames_all) - len(fnames_starter_gui)}{Clr.RST}' + ) + print( + f'{Clr.SBLU}Starter server cache includes' + f' {len(fnames_starter_server)} items;' + f' excludes {len(fnames_all) - len(fnames_starter_server)}{Clr.RST}' ) # Sync all individual cache files to the staging server. @@ -458,7 +501,8 @@ def _update_cloud_cache( def _gather_cache_files( - fnames_starter: list[str], + fnames_starter_gui: list[str], + fnames_starter_server: list[str], fnames_all: list[str], staging_dir: str, mapping_file: str, @@ -478,10 +522,13 @@ def _gather_cache_files( names_to_hashpaths[fname] = fhashpath fhashpaths_all.add(fhashpath) - # Now calc hashpaths for our starter file set. - fhashpaths_starter: set[str] = set() - for fname in fnames_starter: - fhashpaths_starter.add(names_to_hashpaths[fname]) + # Now calc hashpaths for our starter file sets. + fhashpaths_starter_gui: set[str] = set() + for fname in fnames_starter_gui: + fhashpaths_starter_gui.add(names_to_hashpaths[fname]) + fhashpaths_starter_server: set[str] = set() + for fname in fnames_starter_server: + fhashpaths_starter_server.add(names_to_hashpaths[fname]) # We want the server to have a startercache(server).tar.xz files # which contain the entire subsets we were passed. It is much more @@ -497,26 +544,33 @@ def _gather_cache_files( 'import os\n' 'import pathlib\n' 'import subprocess\n' - f'fnames_starter = {repr(fhashpaths_starter)}\n' + f'fnames_starter_gui = {repr(fhashpaths_starter_gui)}\n' + f'fnames_starter_server = {repr(fhashpaths_starter_server)}\n' f'fnames_all = {repr(fhashpaths_all)}\n' - 'subprocess.run(["rm", "-rf", "efrocache"], check=True)\n' - 'print("Gathering starter cache files...", flush=True)\n' - 'for fname in fnames_starter:\n' - ' dst = os.path.join("efrocache", fname)\n' - ' os.makedirs(os.path.dirname(dst), exist_ok=True)\n' - ' subprocess.run(["cp", fname, dst], check=True)\n' - 'print("Touching full file set...", flush=True)\n' + 'print("Updating modtimes on all current cache files...", flush=True)\n' 'for fname in fnames_all:\n' ' fpath = pathlib.Path(fname)\n' ' assert fpath.exists()\n' ' fpath.touch()\n' - 'print("Compressing starter cache archive...", flush=True)\n' - 'subprocess.run(["tar", "-Jcf", "tmp.tar.xz", "efrocache"],' + 'for scname, scarchivename, fnames_starter in [\n' + ' ("gui", "startercache", fnames_starter_gui),\n' + ' ("server", "startercacheserver", fnames_starter_server)]:\n' + ' print(f"Gathering {scname} starter-cache files...", flush=True)\n' + ' subprocess.run(["rm", "-rf", "efrocache"], check=True)\n' + ' for fname in fnames_starter:\n' + ' dst = os.path.join("efrocache", fname)\n' + ' os.makedirs(os.path.dirname(dst), exist_ok=True)\n' + ' subprocess.run(["cp", fname, dst], check=True)\n' + ' print(f"Compressing {scname} starter-cache archive...",' + ' flush=True)\n' + ' subprocess.run(["tar", "-Jcf", "tmp.tar.xz", "efrocache"],' ' check=True)\n' - 'subprocess.run(["mv", "tmp.tar.xz", "startercache.tar.xz"],' + ' subprocess.run(["mv", "tmp.tar.xz", f"{scarchivename}.tar.xz"],' ' check=True)\n' - 'subprocess.run(["rm", "-rf", "efrocache", "genstartercache.py"])\n' - 'print("Starter cache generation complete!", flush=True)\n' + ' subprocess.run(["rm", "-rf", "efrocache"], check=True)\n' + ' print(scname.capitalize() + "starter-cache generation complete!",' + ' flush=True)\n' + 'subprocess.run(["rm", "-rf", "genstartercache.py"])\n' ) with open(