enabled efrocache starter archives for server builds

.idea/dictionaries/ericf.xml

@@ -249,6 +249,7 @@
       <w>bametainternal</w>
       <w>baplus</w>
       <w>baplusmeta</w>
+      <w>bapoo</w>
       <w>barcolor</w>
       <w>barebones</w>
       <w>bargs</w>
@@ -406,8 +407,10 @@
       <w>cachedir</w>
       <w>cacheentry</w>
       <w>cachefile</w>
+      <w>cachefname</w>
       <w>cachemap</w>
       <w>cachepath</w>
+      <w>cachetype</w>
       <w>cadata</w>
       <w>cafile</w>
       <w>calced</w>
@@ -1028,6 +1031,7 @@
       <w>fflush</w>
       <w>fhash</w>
       <w>fhashes</w>
+      <w>fhashpath</w>
       <w>fhashpaths</w>
       <w>fhdr</w>
       <w>fieldattr</w>
@@ -2577,6 +2581,7 @@
       <w>sbylw</w>
       <w>sbytes</w>
       <w>scanresults</w>
+      <w>scarchivename</w>
       <w>scenefile</w>
       <w>scenefiles</w>
       <w>scenename</w>
@@ -2584,6 +2589,7 @@
       <w>sched</w>
       <w>sclx</w>
       <w>scly</w>
+      <w>scname</w>
       <w>scoreconfig</w>
       <w>scorescreen</w>
       <w>scoreteam</w>
@@ -2793,6 +2799,7 @@
       <w>standin</w>
       <w>starscale</w>
       <w>startercache</w>
+      <w>startercacheserver</w>
       <w>startms</w>
       <w>startscan</w>
       <w>startsplits</w>

CHANGELOG.md

@@ -9,6 +9,11 @@
   it.
 - Public builds now properly reconstruct the CMakeLists.txt file for project
   changes.
+- Efrocache now supports a starter-archive when building server builds. This
+  means that if you do something like `make clean; make
+  prefab-server-release-build` you should only see a few file downloads
+  happening instead of hundreds or thousands which would happen before, which
+  should be significantly faster & more efficient.
   
 ### 1.7.27 (build 21282, api 8, 2023-08-30)
 

ballisticakit-cmake/.idea/dictionaries/ericf.xml

@@ -152,6 +152,7 @@
       <w>bametainternal</w>
       <w>baplus</w>
       <w>baplusmeta</w>
+      <w>bapoo</w>
       <w>barebones</w>
       <w>bargs</w>
       <w>bascenev</w>
@@ -268,6 +269,8 @@
       <w>bwst</w>
       <w>cacert</w>
       <w>cachedir</w>
+      <w>cachefname</w>
+      <w>cachetype</w>
       <w>cadata</w>
       <w>cafile</w>
       <w>calced</w>
@@ -639,6 +642,7 @@
       <w>fffffffffifff</w>
       <w>fflush</w>
       <w>fgets</w>
+      <w>fhashpath</w>
       <w>fhashpaths</w>
       <w>fieldname</w>
       <w>fieldpath</w>
@@ -1532,8 +1536,10 @@
       <w>sbytes</w>
       <w>scancode</w>
       <w>scanresults</w>
+      <w>scarchivename</w>
       <w>scenepacket</w>
       <w>scenetime</w>
+      <w>scname</w>
       <w>screenmessage</w>
       <w>scriptdst</w>
       <w>scrollwidget</w>
@@ -1656,6 +1662,7 @@
       <w>stager</w>
       <w>standin</w>
       <w>startedptr</w>
+      <w>startercacheserver</w>
       <w>startms</w>
       <w>startpos</w>
       <w>startsplits</w>

src/assets/Makefile

@@ -40,49 +40,49 @@ endif
 
 # Build everything needed for all platforms.
 all:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for our cmake builds (linux, mac).
 cmake:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-cmake
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for our server builds.
 server:
-	@echo Note - skipping warm_start_asset_build for server target.
+	@$(PCOMMAND) warm_start_asset_build server
 	@$(MAKE) assets-server
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for x86 windows builds.
 win-Win32:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-win-Win32
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for x86-64 windows builds.
 win-x64:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-win-x64
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for our mac xcode builds.
 mac:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-mac
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for our ios/tvos builds.
 ios:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-ios
 	@$(PCOMMAND) clean_orphaned_assets
 
 # Build everything needed for android.
 android:
-	@$(PCOMMAND) warm_start_asset_build
+	@$(PCOMMAND) warm_start_asset_build gui
 	@$(MAKE) assets-android
 	@$(PCOMMAND) clean_orphaned_assets
 

tools/batools/pcommands.py

@@ -520,17 +520,24 @@ def warm_start_asset_build() -> None:
     import os
     import subprocess
     from pathlib import Path
+
     from efrotools import getprojectconfig
+    from efro.error import CleanError
 
     pcommand.disallow_in_batch()
 
+    args = pcommand.get_args()
+    if len(args) != 1:
+        raise CleanError('Expected a single "gui" or "server" arg.')
+    cachetype = args[0]
+
     public: bool = getprojectconfig(pcommand.PROJROOT)['public']
 
     if public:
         from efrotools.efrocache import warm_start_cache
 
         os.chdir(pcommand.PROJROOT)
-        warm_start_cache()
+        warm_start_cache(cachetype)
     else:
         # For internal builds we don't use efrocache but we do use an
         # internal build cache. Download an initial cache/etc. if need be.

tools/efrotools/efrocache.py

@@ -360,7 +360,7 @@ def update_cache(makefile_dirs: list[str]) -> None:
             # prebuilt binaries tend to be larger and we don't want to
             # include a bunch of binaries for other platforms that we
             # won't use.
-            if os.path.getsize(fullpath) < 100000:
+            if os.path.getsize(fullpath) < 200_000:
                 # Gui starter gets everything.
                 fnames_starter_gui.append(fullpath)
 
@@ -659,7 +659,7 @@ def _check_warm_start_entries(entries: list[tuple[str, str]]) -> None:
         list(executor.map(_check_warm_start_entry, entries))
 
 
-def warm_start_cache() -> None:
+def warm_start_cache(cachetype: str) -> None:
     """Run a pre-pass on the efrocache to improve efficiency.
 
     This may fetch an initial cache archive, batch update mod times
@@ -667,16 +667,23 @@ def warm_start_cache() -> None:
     """
     import tempfile
 
+    if cachetype not in {'gui', 'server'}:
+        raise ValueError(f"Invalid cachetype '{cachetype}'.")
+
     base_url = get_repository_base_url()
     local_cache_dir = get_local_cache_dir()
 
-    # We maintain a starter archive on the staging server, which is simply
+    cachefname = (
-    # a set of commonly used recent cache entries compressed into a
+        'startercacheserver' if cachetype == 'server' else 'startercache'
-    # single archive. If we have no local cache yet we can download and
+    )
-    # expand this to give us a nice head start and greatly reduce the
+
-    # initial set of individual files we have to fetch. (downloading a
+    # We maintain starter-cache archives on the staging server, which
-    # single compressed archive is much more efficient than downloading
+    # are simply sets of commonly used recent cache entries compressed
-    # thousands)
+    # into a single archive. If we have no local cache yet we can
+    # download and expand this to give us a nice head start and greatly
+    # reduce the initial set of individual files we have to fetch
+    # (downloading a single compressed archive is much more efficient
+    # than downloading thousands).
     if not os.path.exists(local_cache_dir):
         print('Downloading efrocache starter-archive...', flush=True)
 
@@ -684,13 +691,13 @@ def warm_start_cache() -> None:
         # and then move it into place as our shiny new cache dir.
         with tempfile.TemporaryDirectory() as tmpdir:
             starter_cache_file_path = os.path.join(
-                tmpdir, 'startercache.tar.xz'
+                tmpdir, f'{cachefname}.tar.xz'
             )
             subprocess.run(
                 [
                     'curl',
                     '--fail',
-                    f'{base_url}/startercache.tar.xz',
+                    f'{base_url}/{cachefname}.tar.xz',
                     '--output',
                     starter_cache_file_path,
                 ],