From 1bb99635a731612d85f8c1e5b35d36777cc438d3 Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Fri, 1 Sep 2023 11:09:57 -0700 Subject: [PATCH] enabled efrocache starter archives for server builds --- .idea/dictionaries/ericf.xml | 7 +++++ CHANGELOG.md | 5 ++++ .../.idea/dictionaries/ericf.xml | 7 +++++ src/assets/Makefile | 16 +++++----- tools/batools/pcommands.py | 9 +++++- tools/efrotools/efrocache.py | 29 ++++++++++++------- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 077d07ec..23ee3503 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -249,6 +249,7 @@ bametainternal baplus baplusmeta + bapoo barcolor barebones bargs @@ -406,8 +407,10 @@ cachedir cacheentry cachefile + cachefname cachemap cachepath + cachetype cadata cafile calced @@ -1028,6 +1031,7 @@ fflush fhash fhashes + fhashpath fhashpaths fhdr fieldattr @@ -2577,6 +2581,7 @@ sbylw sbytes scanresults + scarchivename scenefile scenefiles scenename @@ -2584,6 +2589,7 @@ sched sclx scly + scname scoreconfig scorescreen scoreteam @@ -2793,6 +2799,7 @@ standin starscale startercache + startercacheserver startms startscan startsplits diff --git a/CHANGELOG.md b/CHANGELOG.md index 4636e832..f4390f8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ it. - Public builds now properly reconstruct the CMakeLists.txt file for project changes. +- Efrocache now supports a starter-archive when building server builds. This + means that if you do something like `make clean; make + prefab-server-release-build` you should only see a few file downloads + happening instead of hundreds or thousands which would happen before, which + should be significantly faster & more efficient. ### 1.7.27 (build 21282, api 8, 2023-08-30) diff --git a/ballisticakit-cmake/.idea/dictionaries/ericf.xml b/ballisticakit-cmake/.idea/dictionaries/ericf.xml index f68e5937..22391a37 100644 --- a/ballisticakit-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticakit-cmake/.idea/dictionaries/ericf.xml @@ -152,6 +152,7 @@ bametainternal baplus baplusmeta + bapoo barebones bargs bascenev @@ -268,6 +269,8 @@ bwst cacert cachedir + cachefname + cachetype cadata cafile calced @@ -639,6 +642,7 @@ fffffffffifff fflush fgets + fhashpath fhashpaths fieldname fieldpath @@ -1532,8 +1536,10 @@ sbytes scancode scanresults + scarchivename scenepacket scenetime + scname screenmessage scriptdst scrollwidget @@ -1656,6 +1662,7 @@ stager standin startedptr + startercacheserver startms startpos startsplits diff --git a/src/assets/Makefile b/src/assets/Makefile index 01856715..be1741a5 100644 --- a/src/assets/Makefile +++ b/src/assets/Makefile @@ -40,49 +40,49 @@ endif # Build everything needed for all platforms. all: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets @$(PCOMMAND) clean_orphaned_assets # Build everything needed for our cmake builds (linux, mac). cmake: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-cmake @$(PCOMMAND) clean_orphaned_assets # Build everything needed for our server builds. server: - @echo Note - skipping warm_start_asset_build for server target. + @$(PCOMMAND) warm_start_asset_build server @$(MAKE) assets-server @$(PCOMMAND) clean_orphaned_assets # Build everything needed for x86 windows builds. win-Win32: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-win-Win32 @$(PCOMMAND) clean_orphaned_assets # Build everything needed for x86-64 windows builds. win-x64: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-win-x64 @$(PCOMMAND) clean_orphaned_assets # Build everything needed for our mac xcode builds. mac: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-mac @$(PCOMMAND) clean_orphaned_assets # Build everything needed for our ios/tvos builds. ios: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-ios @$(PCOMMAND) clean_orphaned_assets # Build everything needed for android. android: - @$(PCOMMAND) warm_start_asset_build + @$(PCOMMAND) warm_start_asset_build gui @$(MAKE) assets-android @$(PCOMMAND) clean_orphaned_assets diff --git a/tools/batools/pcommands.py b/tools/batools/pcommands.py index adc59626..3e5031f8 100644 --- a/tools/batools/pcommands.py +++ b/tools/batools/pcommands.py @@ -520,17 +520,24 @@ def warm_start_asset_build() -> None: import os import subprocess from pathlib import Path + from efrotools import getprojectconfig + from efro.error import CleanError pcommand.disallow_in_batch() + args = pcommand.get_args() + if len(args) != 1: + raise CleanError('Expected a single "gui" or "server" arg.') + cachetype = args[0] + public: bool = getprojectconfig(pcommand.PROJROOT)['public'] if public: from efrotools.efrocache import warm_start_cache os.chdir(pcommand.PROJROOT) - warm_start_cache() + warm_start_cache(cachetype) else: # For internal builds we don't use efrocache but we do use an # internal build cache. Download an initial cache/etc. if need be. diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 664c533f..84601a86 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -360,7 +360,7 @@ def update_cache(makefile_dirs: list[str]) -> None: # prebuilt binaries tend to be larger and we don't want to # include a bunch of binaries for other platforms that we # won't use. - if os.path.getsize(fullpath) < 100000: + if os.path.getsize(fullpath) < 200_000: # Gui starter gets everything. fnames_starter_gui.append(fullpath) @@ -659,7 +659,7 @@ def _check_warm_start_entries(entries: list[tuple[str, str]]) -> None: list(executor.map(_check_warm_start_entry, entries)) -def warm_start_cache() -> None: +def warm_start_cache(cachetype: str) -> None: """Run a pre-pass on the efrocache to improve efficiency. This may fetch an initial cache archive, batch update mod times @@ -667,16 +667,23 @@ def warm_start_cache() -> None: """ import tempfile + if cachetype not in {'gui', 'server'}: + raise ValueError(f"Invalid cachetype '{cachetype}'.") + base_url = get_repository_base_url() local_cache_dir = get_local_cache_dir() - # We maintain a starter archive on the staging server, which is simply - # a set of commonly used recent cache entries compressed into a - # single archive. If we have no local cache yet we can download and - # expand this to give us a nice head start and greatly reduce the - # initial set of individual files we have to fetch. (downloading a - # single compressed archive is much more efficient than downloading - # thousands) + cachefname = ( + 'startercacheserver' if cachetype == 'server' else 'startercache' + ) + + # We maintain starter-cache archives on the staging server, which + # are simply sets of commonly used recent cache entries compressed + # into a single archive. If we have no local cache yet we can + # download and expand this to give us a nice head start and greatly + # reduce the initial set of individual files we have to fetch + # (downloading a single compressed archive is much more efficient + # than downloading thousands). if not os.path.exists(local_cache_dir): print('Downloading efrocache starter-archive...', flush=True) @@ -684,13 +691,13 @@ def warm_start_cache() -> None: # and then move it into place as our shiny new cache dir. with tempfile.TemporaryDirectory() as tmpdir: starter_cache_file_path = os.path.join( - tmpdir, 'startercache.tar.xz' + tmpdir, f'{cachefname}.tar.xz' ) subprocess.run( [ 'curl', '--fail', - f'{base_url}/startercache.tar.xz', + f'{base_url}/{cachefname}.tar.xz', '--output', starter_cache_file_path, ],