mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-19 21:37:57 +08:00
enabled efrocache starter archives for server builds
This commit is contained in:
Eric Froemling
parent
46dda011a4
commit
1bb99635a7
7
.idea/dictionaries/ericf.xml
generated
7
.idea/dictionaries/ericf.xml
generated
@ -249,6 +249,7 @@
|
||||
<w>bametainternal</w>
|
||||
<w>baplus</w>
|
||||
<w>baplusmeta</w>
|
||||
<w>bapoo</w>
|
||||
<w>barcolor</w>
|
||||
<w>barebones</w>
|
||||
<w>bargs</w>
|
||||
@ -406,8 +407,10 @@
|
||||
<w>cachedir</w>
|
||||
<w>cacheentry</w>
|
||||
<w>cachefile</w>
|
||||
<w>cachefname</w>
|
||||
<w>cachemap</w>
|
||||
<w>cachepath</w>
|
||||
<w>cachetype</w>
|
||||
<w>cadata</w>
|
||||
<w>cafile</w>
|
||||
<w>calced</w>
|
||||
@ -1028,6 +1031,7 @@
|
||||
<w>fflush</w>
|
||||
<w>fhash</w>
|
||||
<w>fhashes</w>
|
||||
<w>fhashpath</w>
|
||||
<w>fhashpaths</w>
|
||||
<w>fhdr</w>
|
||||
<w>fieldattr</w>
|
||||
@ -2577,6 +2581,7 @@
|
||||
<w>sbylw</w>
|
||||
<w>sbytes</w>
|
||||
<w>scanresults</w>
|
||||
<w>scarchivename</w>
|
||||
<w>scenefile</w>
|
||||
<w>scenefiles</w>
|
||||
<w>scenename</w>
|
||||
@ -2584,6 +2589,7 @@
|
||||
<w>sched</w>
|
||||
<w>sclx</w>
|
||||
<w>scly</w>
|
||||
<w>scname</w>
|
||||
<w>scoreconfig</w>
|
||||
<w>scorescreen</w>
|
||||
<w>scoreteam</w>
|
||||
@ -2793,6 +2799,7 @@
|
||||
<w>standin</w>
|
||||
<w>starscale</w>
|
||||
<w>startercache</w>
|
||||
<w>startercacheserver</w>
|
||||
<w>startms</w>
|
||||
<w>startscan</w>
|
||||
<w>startsplits</w>
|
||||
|
||||
@ -9,6 +9,11 @@
|
||||
it.
|
||||
- Public builds now properly reconstruct the CMakeLists.txt file for project
|
||||
changes.
|
||||
- Efrocache now supports a starter-archive when building server builds. This
|
||||
means that if you do something like `make clean; make
|
||||
prefab-server-release-build` you should only see a few file downloads
|
||||
happening instead of hundreds or thousands which would happen before, which
|
||||
should be significantly faster & more efficient.
|
||||
|
||||
### 1.7.27 (build 21282, api 8, 2023-08-30)
|
||||
|
||||
|
||||
7
ballisticakit-cmake/.idea/dictionaries/ericf.xml
generated
7
ballisticakit-cmake/.idea/dictionaries/ericf.xml
generated
@ -152,6 +152,7 @@
|
||||
<w>bametainternal</w>
|
||||
<w>baplus</w>
|
||||
<w>baplusmeta</w>
|
||||
<w>bapoo</w>
|
||||
<w>barebones</w>
|
||||
<w>bargs</w>
|
||||
<w>bascenev</w>
|
||||
@ -268,6 +269,8 @@
|
||||
<w>bwst</w>
|
||||
<w>cacert</w>
|
||||
<w>cachedir</w>
|
||||
<w>cachefname</w>
|
||||
<w>cachetype</w>
|
||||
<w>cadata</w>
|
||||
<w>cafile</w>
|
||||
<w>calced</w>
|
||||
@ -639,6 +642,7 @@
|
||||
<w>fffffffffifff</w>
|
||||
<w>fflush</w>
|
||||
<w>fgets</w>
|
||||
<w>fhashpath</w>
|
||||
<w>fhashpaths</w>
|
||||
<w>fieldname</w>
|
||||
<w>fieldpath</w>
|
||||
@ -1532,8 +1536,10 @@
|
||||
<w>sbytes</w>
|
||||
<w>scancode</w>
|
||||
<w>scanresults</w>
|
||||
<w>scarchivename</w>
|
||||
<w>scenepacket</w>
|
||||
<w>scenetime</w>
|
||||
<w>scname</w>
|
||||
<w>screenmessage</w>
|
||||
<w>scriptdst</w>
|
||||
<w>scrollwidget</w>
|
||||
@ -1656,6 +1662,7 @@
|
||||
<w>stager</w>
|
||||
<w>standin</w>
|
||||
<w>startedptr</w>
|
||||
<w>startercacheserver</w>
|
||||
<w>startms</w>
|
||||
<w>startpos</w>
|
||||
<w>startsplits</w>
|
||||
|
||||
@ -40,49 +40,49 @@ endif
|
||||
|
||||
# Build everything needed for all platforms.
|
||||
all:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for our cmake builds (linux, mac).
|
||||
cmake:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-cmake
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for our server builds.
|
||||
server:
|
||||
@echo Note - skipping warm_start_asset_build for server target.
|
||||
@$(PCOMMAND) warm_start_asset_build server
|
||||
@$(MAKE) assets-server
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for x86 windows builds.
|
||||
win-Win32:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-win-Win32
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for x86-64 windows builds.
|
||||
win-x64:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-win-x64
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for our mac xcode builds.
|
||||
mac:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-mac
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for our ios/tvos builds.
|
||||
ios:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-ios
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
# Build everything needed for android.
|
||||
android:
|
||||
@$(PCOMMAND) warm_start_asset_build
|
||||
@$(PCOMMAND) warm_start_asset_build gui
|
||||
@$(MAKE) assets-android
|
||||
@$(PCOMMAND) clean_orphaned_assets
|
||||
|
||||
|
||||
@ -520,17 +520,24 @@ def warm_start_asset_build() -> None:
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from efrotools import getprojectconfig
|
||||
from efro.error import CleanError
|
||||
|
||||
pcommand.disallow_in_batch()
|
||||
|
||||
args = pcommand.get_args()
|
||||
if len(args) != 1:
|
||||
raise CleanError('Expected a single "gui" or "server" arg.')
|
||||
cachetype = args[0]
|
||||
|
||||
public: bool = getprojectconfig(pcommand.PROJROOT)['public']
|
||||
|
||||
if public:
|
||||
from efrotools.efrocache import warm_start_cache
|
||||
|
||||
os.chdir(pcommand.PROJROOT)
|
||||
warm_start_cache()
|
||||
warm_start_cache(cachetype)
|
||||
else:
|
||||
# For internal builds we don't use efrocache but we do use an
|
||||
# internal build cache. Download an initial cache/etc. if need be.
|
||||
|
||||
@ -360,7 +360,7 @@ def update_cache(makefile_dirs: list[str]) -> None:
|
||||
# prebuilt binaries tend to be larger and we don't want to
|
||||
# include a bunch of binaries for other platforms that we
|
||||
# won't use.
|
||||
if os.path.getsize(fullpath) < 100000:
|
||||
if os.path.getsize(fullpath) < 200_000:
|
||||
# Gui starter gets everything.
|
||||
fnames_starter_gui.append(fullpath)
|
||||
|
||||
@ -659,7 +659,7 @@ def _check_warm_start_entries(entries: list[tuple[str, str]]) -> None:
|
||||
list(executor.map(_check_warm_start_entry, entries))
|
||||
|
||||
|
||||
def warm_start_cache() -> None:
|
||||
def warm_start_cache(cachetype: str) -> None:
|
||||
"""Run a pre-pass on the efrocache to improve efficiency.
|
||||
|
||||
This may fetch an initial cache archive, batch update mod times
|
||||
@ -667,16 +667,23 @@ def warm_start_cache() -> None:
|
||||
"""
|
||||
import tempfile
|
||||
|
||||
if cachetype not in {'gui', 'server'}:
|
||||
raise ValueError(f"Invalid cachetype '{cachetype}'.")
|
||||
|
||||
base_url = get_repository_base_url()
|
||||
local_cache_dir = get_local_cache_dir()
|
||||
|
||||
# We maintain a starter archive on the staging server, which is simply
|
||||
# a set of commonly used recent cache entries compressed into a
|
||||
# single archive. If we have no local cache yet we can download and
|
||||
# expand this to give us a nice head start and greatly reduce the
|
||||
# initial set of individual files we have to fetch. (downloading a
|
||||
# single compressed archive is much more efficient than downloading
|
||||
# thousands)
|
||||
cachefname = (
|
||||
'startercacheserver' if cachetype == 'server' else 'startercache'
|
||||
)
|
||||
|
||||
# We maintain starter-cache archives on the staging server, which
|
||||
# are simply sets of commonly used recent cache entries compressed
|
||||
# into a single archive. If we have no local cache yet we can
|
||||
# download and expand this to give us a nice head start and greatly
|
||||
# reduce the initial set of individual files we have to fetch
|
||||
# (downloading a single compressed archive is much more efficient
|
||||
# than downloading thousands).
|
||||
if not os.path.exists(local_cache_dir):
|
||||
print('Downloading efrocache starter-archive...', flush=True)
|
||||
|
||||
@ -684,13 +691,13 @@ def warm_start_cache() -> None:
|
||||
# and then move it into place as our shiny new cache dir.
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
starter_cache_file_path = os.path.join(
|
||||
tmpdir, 'startercache.tar.xz'
|
||||
tmpdir, f'{cachefname}.tar.xz'
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
'curl',
|
||||
'--fail',
|
||||
f'{base_url}/startercache.tar.xz',
|
||||
f'{base_url}/{cachefname}.tar.xz',
|
||||
'--output',
|
||||
starter_cache_file_path,
|
||||
],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user