diff --git a/tools/batools/build.py b/tools/batools/build.py index b9c38e42..bb09f159 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -98,6 +98,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'Makefile', 'src/meta', 'src/ballistica/shared/foundation/types.h', + '.efrocachemap', ], # Our meta Makefile targets generally don't list tools # scripts that can affect their creation as sources, so @@ -105,8 +106,8 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: # changes we'll blow away all existing meta builds. # # Update: also including featureset-defs here; any time - # we're mucking with those it's good to start fresh to be - # sure. + # we're mucking with those it's good to start things fresh + # to be safe. srcpaths_fullclean=[ 'tools/efrotools', 'tools/efrotoolsinternal', @@ -135,6 +136,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'Makefile', 'src', 'ballisticakit-cmake/CMakeLists.txt', + '.efrocachemap', ], dirfilter=( lambda root, dirname: not ( @@ -166,6 +168,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'Makefile', 'src', 'ballisticakit-windows', + '.efrocachemap', ], dirfilter=_win_dirfilter, command=command, @@ -183,6 +186,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'Makefile', 'tools/pcommand', 'src/resources', + '.efrocachemap', ], command=command, ).run() @@ -208,6 +212,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'Makefile', 'tools', 'src/assets', + '.efrocachemap', ], command=command, filefilter=_filefilter, @@ -234,6 +239,7 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None: 'config/featuresets', 'tools/batools/dummymodule.py', 'src/ballistica', + '.efrocachemap', ], command=command, filefilter=_filefilter, diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 86e4d6ce..27599df0 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -143,9 +143,9 @@ def get_target(path: str) -> None: local_cache_path = os.path.join(local_cache_dir, subpath) - # First off: if there's already a cache file in place, check its - # hash. If its calced hash matches its path, we can just update its - # timestamp and call it a day. + # First off: if there's already a file in place, check its hash. If + # its calced hash matches the hash-map's value for it, we can just + # update its timestamp and call it a day. if os.path.isfile(path): existing_hash = get_existing_file_hash(path) if existing_hash == hashval: @@ -492,6 +492,10 @@ def _write_cache_files( outfile.write(json.dumps(mapping, indent=2, sort_keys=True)) +def _path_from_hash(hashstr: str) -> str: + return os.path.join(hashstr[:2], hashstr[2:4], hashstr[4:]) + + def _write_cache_file(staging_dir: str, fname: str) -> tuple[str, str, str]: import hashlib @@ -511,7 +515,7 @@ def _write_cache_file(staging_dir: str, fname: str) -> tuple[str, str, str]: md5 = hashlib.md5() md5.update(prefix + fdataraw) finalhash = md5.hexdigest() - hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:]) + hashpath = _path_from_hash(finalhash) path = os.path.join(staging_dir, hashpath) os.makedirs(os.path.dirname(path), exist_ok=True) @@ -623,13 +627,19 @@ def warm_start_cache() -> None: # per file, it adds up when done for thousands of files each time # the cache map changes. It is much more efficient to do it all in # one go here. + # + # Note to self: it could be nice to put together a lightweight build + # server system of some sort so we don't have to spin up a full + # Python process for each and every file we need to touch. In that + # case, this optimization would probably be unnecessary. cachemap: dict[str, str] with open(CACHE_MAP_NAME, encoding='utf-8') as infile: cachemap = json.loads(infile.read()) assert isinstance(cachemap, dict) cachemap_mtime = os.path.getmtime(CACHE_MAP_NAME) entries: list[tuple[str, str]] = [] - for fname, url in cachemap.items(): + for fname, filehash in cachemap.items(): + # File hasn't been pulled from cache yet = ignore. if not os.path.exists(fname): continue @@ -638,14 +648,15 @@ def warm_start_cache() -> None: if cachemap_mtime < os.path.getmtime(fname): continue - # Don't have the cache source file for this guy = ignore. - cachefile = local_cache_dir + '/' + '/'.join(url.split('/')[-3:]) + # Don't have the cache source file for this guy = ignore. This + # can happen if cache files have been blown away since the last + # time this was built. + cachefile = os.path.join(local_cache_dir, _path_from_hash(filehash)) if not os.path.exists(cachefile): continue # Ok, add it to the list of files we can potentially update # timestamps on once we check its hash. - filehash = ''.join(url.split('/')[-3:]) entries.append((fname, filehash)) if entries: