RYDE-WORK/ballistica
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ballistica.git synced 2026-01-19 21:37:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed some efrocache logic my recent changes broke.

Browse Source 
warm_start_cache functionality should now work again which means a lot less time spent in "Refreshing from cache"
when the .efrocache map changes.
This commit is contained in:
Eric Froemling 2023-08-03 13:47:34 -07:00
parent c93d3f9359
commit ebf6ca5be1
No known key found for this signature in database
GPG Key ID: 89C93F0F8D6D5A98
2 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tools/batools/build.py
View File

@ -98,6 +98,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'Makefile',
'src/meta',
'src/ballistica/shared/foundation/types.h',
'.efrocachemap',
],
# Our meta Makefile targets generally don't list tools
# scripts that can affect their creation as sources, so
@ -105,8 +106,8 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
# changes we'll blow away all existing meta builds.
#
# Update: also including featureset-defs here; any time
# we're mucking with those it's good to start fresh to be
# sure.
# we're mucking with those it's good to start things fresh
# to be safe.
srcpaths_fullclean=[
'tools/efrotools',
'tools/efrotoolsinternal',
@ -135,6 +136,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'Makefile',
'src',
'ballisticakit-cmake/CMakeLists.txt',
'.efrocachemap',
],
dirfilter=(
lambda root, dirname: not (
@ -166,6 +168,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'Makefile',
'src',
'ballisticakit-windows',
'.efrocachemap',
],
dirfilter=_win_dirfilter,
command=command,
@ -183,6 +186,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'Makefile',
'tools/pcommand',
'src/resources',
'.efrocachemap',
],
command=command,
).run()
@ -208,6 +212,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'Makefile',
'tools',
'src/assets',
'.efrocachemap',
],
command=command,
filefilter=_filefilter,
@ -234,6 +239,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
'config/featuresets',
'tools/batools/dummymodule.py',
'src/ballistica',
'.efrocachemap',
],
command=command,
filefilter=_filefilter,

27
tools/efrotools/efrocache.py
View File

@ -143,9 +143,9 @@ def get_target(path: str) -> None:
local_cache_path = os.path.join(local_cache_dir, subpath)
# First off: if there's already a cache file in place, check its
# hash. If its calced hash matches its path, we can just update its
# timestamp and call it a day.
# First off: if there's already a file in place, check its hash. If
# its calced hash matches the hash-map's value for it, we can just
# update its timestamp and call it a day.
if os.path.isfile(path):
existing_hash = get_existing_file_hash(path)
if existing_hash == hashval:
@ -492,6 +492,10 @@ def _write_cache_files(
outfile.write(json.dumps(mapping, indent=2, sort_keys=True))
def _path_from_hash(hashstr: str) -> str:
return os.path.join(hashstr[:2], hashstr[2:4], hashstr[4:])
def _write_cache_file(staging_dir: str, fname: str) -> tuple[str, str, str]:
import hashlib
@ -511,7 +515,7 @@ def _write_cache_file(staging_dir: str, fname: str) -> tuple[str, str, str]:
md5 = hashlib.md5()
md5.update(prefix + fdataraw)
finalhash = md5.hexdigest()
hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
hashpath = _path_from_hash(finalhash)
path = os.path.join(staging_dir, hashpath)
os.makedirs(os.path.dirname(path), exist_ok=True)
@ -623,13 +627,19 @@ def warm_start_cache() -> None:
# per file, it adds up when done for thousands of files each time
# the cache map changes. It is much more efficient to do it all in
# one go here.
#
# Note to self: it could be nice to put together a lightweight build
# server system of some sort so we don't have to spin up a full
# Python process for each and every file we need to touch. In that
# case, this optimization would probably be unnecessary.
cachemap: dict[str, str]
with open(CACHE_MAP_NAME, encoding='utf-8') as infile:
cachemap = json.loads(infile.read())
assert isinstance(cachemap, dict)
cachemap_mtime = os.path.getmtime(CACHE_MAP_NAME)
entries: list[tuple[str, str]] = []
for fname, url in cachemap.items():
for fname, filehash in cachemap.items():
# File hasn't been pulled from cache yet = ignore.
if not os.path.exists(fname):
continue
@ -638,14 +648,15 @@ def warm_start_cache() -> None:
if cachemap_mtime < os.path.getmtime(fname):
continue
# Don't have the cache source file for this guy = ignore.
cachefile = local_cache_dir + '/' + '/'.join(url.split('/')[-3:])
# Don't have the cache source file for this guy = ignore. This
# can happen if cache files have been blown away since the last
# time this was built.
cachefile = os.path.join(local_cache_dir, _path_from_hash(filehash))
if not os.path.exists(cachefile):
continue
# Ok, add it to the list of files we can potentially update
# timestamps on once we check its hash.
filehash = ''.join(url.split('/')[-3:])
entries.append((fname, filehash))
if entries: