mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-19 13:25:31 +08:00
wired asset-packages into local asset builds
This commit is contained in:
Eric
parent
7e24e6ee45
commit
2b6df3f981
56
.efrocachemap
generated
56
.efrocachemap
generated
@ -4038,26 +4038,26 @@
|
||||
"build/assets/windows/Win32/ucrtbased.dll": "2def5335207d41b21b9823f6805997f1",
|
||||
"build/assets/windows/Win32/vc_redist.x86.exe": "b08a55e2e77623fe657bea24f223a3ae",
|
||||
"build/assets/windows/Win32/vcruntime140d.dll": "865b2af4d1e26a1a8073c89acb06e599",
|
||||
"build/prefab/full/linux_arm64_gui/debug/ballisticakit": "a39e78f94f2257360a0b5891ccb1e6f3",
|
||||
"build/prefab/full/linux_arm64_gui/release/ballisticakit": "3fdd6d2e093dbe55c66f863790fe0712",
|
||||
"build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "ba679c627647597ce1ae0ee3c79bc628",
|
||||
"build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "85e1e9d4f28402201a8fba3fdc444a94",
|
||||
"build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "f96218be024fb0f54d875b3bb0d6695d",
|
||||
"build/prefab/full/linux_x86_64_gui/release/ballisticakit": "0e192113022123f6535d451d04bae7f6",
|
||||
"build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "bfbcc1a92c5e52aa44db0484d8b1e7e5",
|
||||
"build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "f52ff47413dc64627347a1ada68b2e1b",
|
||||
"build/prefab/full/mac_arm64_gui/debug/ballisticakit": "32b17d80c59ea676f3a548b2ec8799a5",
|
||||
"build/prefab/full/mac_arm64_gui/release/ballisticakit": "92f821766087c12f7ecf54a5f1505a1a",
|
||||
"build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "f0c77f8eb0c5978ed6d1ff11ffc22a3f",
|
||||
"build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "538e04846ec2e3b4c33f1a211ac4e11a",
|
||||
"build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "373b7d863584936cb507dbb7da674949",
|
||||
"build/prefab/full/mac_x86_64_gui/release/ballisticakit": "8ba2796b27a6596fb67c27360b7e7239",
|
||||
"build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "432074c79906f984b1df221398139cf9",
|
||||
"build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "c246914f53747c2c94d3b3b32ab3fab7",
|
||||
"build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "6e2490e0a43a69c7983a39817117d77c",
|
||||
"build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "2d8bef376bf4ab160089f477609d5421",
|
||||
"build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "7d78f5d122e58c4e5a3264ad6aae1289",
|
||||
"build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "ee97d4b95248d68681eda85b658b7531",
|
||||
"build/prefab/full/linux_arm64_gui/debug/ballisticakit": "fafef337dc36981a73a7c0fb4793e1aa",
|
||||
"build/prefab/full/linux_arm64_gui/release/ballisticakit": "b0aa1c5f84dc14b5eec5416e0c42c2dd",
|
||||
"build/prefab/full/linux_arm64_server/debug/dist/ballisticakit_headless": "fd27f2e7e072311db6e76a310e5b3248",
|
||||
"build/prefab/full/linux_arm64_server/release/dist/ballisticakit_headless": "7ab1218f73a6bc9c1f785ffd7cceba24",
|
||||
"build/prefab/full/linux_x86_64_gui/debug/ballisticakit": "4ba1f3ee76aa94ea10d9106b2882f477",
|
||||
"build/prefab/full/linux_x86_64_gui/release/ballisticakit": "4f5bcafa8fcea60c202f57392b790749",
|
||||
"build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "5a204e6e6fc09c7601f82ee5dbf4ac5b",
|
||||
"build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "0f922df6490548a53bb6a871a018f140",
|
||||
"build/prefab/full/mac_arm64_gui/debug/ballisticakit": "8830cb31c8cf6fb67691bdf697d67b2d",
|
||||
"build/prefab/full/mac_arm64_gui/release/ballisticakit": "b9aa5cea9726fcdd6e6468d065b7a5f0",
|
||||
"build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "9d4a199b6ec1dc775ef8835bf185b576",
|
||||
"build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "7f51dec440a47bf9468c73795f6f6faf",
|
||||
"build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "7de19401a7bd5180fccca69ca3deabce",
|
||||
"build/prefab/full/mac_x86_64_gui/release/ballisticakit": "8ee6a1cd92f095e73756798c228fabb9",
|
||||
"build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "4fdadf7c85bc4025b1e89a78d136f163",
|
||||
"build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "4cabc0639011c73a7957aeae844df6fe",
|
||||
"build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "66bb1553c81ba8ee21adbb2cb954530d",
|
||||
"build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "c577ee1c5c4f08530fad64e8fefe4cf8",
|
||||
"build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "30212d030049d0fb89e3aa0cc97d0e0a",
|
||||
"build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "0a2b976285806c698f01e641f9074aca",
|
||||
"build/prefab/lib/linux_arm64_gui/debug/libballisticaplus.a": "f231b10895bdcb542de87b887ca181fd",
|
||||
"build/prefab/lib/linux_arm64_gui/release/libballisticaplus.a": "ae936a119668ede7b36f38c8672f4bf8",
|
||||
"build/prefab/lib/linux_arm64_server/debug/libballisticaplus.a": "f231b10895bdcb542de87b887ca181fd",
|
||||
@ -4074,14 +4074,14 @@
|
||||
"build/prefab/lib/mac_x86_64_gui/release/libballisticaplus.a": "efffc4f330e77530accd9a9f82840a6c",
|
||||
"build/prefab/lib/mac_x86_64_server/debug/libballisticaplus.a": "c20363fe2af3d54e666b1c8ee67f6b76",
|
||||
"build/prefab/lib/mac_x86_64_server/release/libballisticaplus.a": "efffc4f330e77530accd9a9f82840a6c",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "f012fac2242e90a5df21ee4b50004248",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "c2590ba0ed0426a8a8aae627d00d5447",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "846e75a2a07ec8c17f2e6eb622c456ee",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "d70c4ed329cc01aecb41eb2fa8f95130",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "de6384ade5417d115d5b6a3ff89d3bb8",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "b564e1ebdb709343ce66bf63cd1b0568",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "ab388c54bdcc895bc99414aec7d40273",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "f1bab1ce15ef218c191d0e85444f265e",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "52b68c71bcb167a388b89352c09dbbe8",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "36f3e21b79ff41c8bbe9fa797833c0d8",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "bc1f287cdcee0ad7839f2cdc5f01a6e4",
|
||||
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "22c3b0ec1cabf246c7dee9be3e0ef48b",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "fa12a11770061d9a34e6b4bd35930842",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "6770c990a7a562c796e74b13247f62d0",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "cd6b6dcc26978740733af1dc99d92810",
|
||||
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "c3d76e035872d841beb85a558bfca107",
|
||||
"src/assets/ba_data/python/babase/_mgen/__init__.py": "f885fed7f2ed98ff2ba271f9dbe3391c",
|
||||
"src/assets/ba_data/python/babase/_mgen/enums.py": "5548f407d97e380069f6c596c4e36cd7",
|
||||
"src/ballistica/base/mgen/pyembed/binding_base.inc": "efa61468cf098f77cc6a234461d8b86d",
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
### 1.7.36 (build 21899, api 8, 2024-07-01)
|
||||
### 1.7.36 (build 21902, api 8, 2024-07-02)
|
||||
- bacloud workspace commands are now a bit smarter; you can now do things like
|
||||
`bacloud workspace put .` or even just `bacloud workspace put` and it will
|
||||
work. Previously such cases required explicitly passing the workspace name
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"assets": "a-0.assets1.dev",
|
||||
"assets": "a-0.bastdassets.5",
|
||||
"code_source_dirs": [
|
||||
"src/ballistica"
|
||||
],
|
||||
|
||||
@ -149,6 +149,9 @@ endif
|
||||
ASSET_TARGETS_WIN_WIN32 += $(EXTRAS_TARGETS_WIN_WIN32)
|
||||
ASSET_TARGETS_WIN_X64 += $(EXTRAS_TARGETS_WIN_X64)
|
||||
|
||||
# Asset Packages
|
||||
ASSET_TARGETS_CMAKE += $(PROJ_DIR)/.cache/assetmanifests/gui_desktop_v1
|
||||
|
||||
# Note: Code below needs updating when Python version changes (currently 3.11)
|
||||
define make-opt-pyc-target
|
||||
$1: $$(subst /__pycache__,,$$(subst .cpython-312.opt-1.pyc,.py,$1))
|
||||
|
||||
@ -52,7 +52,7 @@ if TYPE_CHECKING:
|
||||
|
||||
# Build number and version of the ballistica binary we expect to be
|
||||
# using.
|
||||
TARGET_BALLISTICA_BUILD = 21899
|
||||
TARGET_BALLISTICA_BUILD = 21902
|
||||
TARGET_BALLISTICA_VERSION = '1.7.36'
|
||||
|
||||
|
||||
|
||||
@ -1064,14 +1064,16 @@ auto Assets::FindAssetFile(FileType type,
|
||||
assert(g_base->InLogicThread());
|
||||
|
||||
const char* ext = "";
|
||||
const char* prefix = "";
|
||||
const char* prefix1 = "";
|
||||
const char* prefix2 = "";
|
||||
|
||||
switch (type) {
|
||||
case FileType::kSound:
|
||||
if (g_core->HeadlessMode()) {
|
||||
return "headless_dummy_path.sound";
|
||||
}
|
||||
prefix = "audio/";
|
||||
prefix1 = "audio/";
|
||||
prefix2 = "audio2/";
|
||||
ext = ".ogg";
|
||||
break;
|
||||
|
||||
@ -1079,17 +1081,20 @@ auto Assets::FindAssetFile(FileType type,
|
||||
if (g_core->HeadlessMode()) {
|
||||
return "headless_dummy_path.mesh";
|
||||
}
|
||||
prefix = "meshes/";
|
||||
prefix1 = "meshes/";
|
||||
prefix2 = "meshes2/";
|
||||
ext = ".bob";
|
||||
break;
|
||||
|
||||
case FileType::kCollisionMesh:
|
||||
prefix = "meshes/";
|
||||
prefix1 = "meshes/";
|
||||
prefix2 = "meshes2/";
|
||||
ext = ".cob";
|
||||
break;
|
||||
|
||||
case FileType::kData:
|
||||
prefix = "data/";
|
||||
prefix1 = "data/";
|
||||
prefix2 = "data2/";
|
||||
ext = ".json";
|
||||
break;
|
||||
|
||||
@ -1109,7 +1114,8 @@ auto Assets::FindAssetFile(FileType type,
|
||||
// g_base->graphics_server->texture_compression_types_are_set());
|
||||
// assert(g_base->graphics_server
|
||||
// && g_base->graphics_server->texture_quality_set());
|
||||
prefix = "textures/";
|
||||
prefix1 = "textures/";
|
||||
prefix2 = "textures2/";
|
||||
|
||||
#if BA_OSTYPE_ANDROID && !BA_ANDROID_DDS_BUILD
|
||||
// On most android builds we go for .ktx, which contains etc2 and etc1.
|
||||
@ -1130,20 +1136,23 @@ auto Assets::FindAssetFile(FileType type,
|
||||
const std::vector<std::string>& asset_paths_used = asset_paths_;
|
||||
|
||||
for (auto&& i : asset_paths_used) {
|
||||
file_out = i + "/" + prefix + name + ext; // NOLINT
|
||||
bool exists;
|
||||
// TEMP - try our '2' stuff first.
|
||||
for (auto&& prefix : {prefix2, prefix1}) {
|
||||
file_out = i + "/" + prefix + name + ext; // NOLINT
|
||||
bool exists;
|
||||
|
||||
// '#' denotes a cube map texture, which is actually 6 files.
|
||||
if (strchr(file_out.c_str(), '#')) {
|
||||
// Just look for one of them i guess.
|
||||
std::string tmp_name = file_out;
|
||||
tmp_name.replace(tmp_name.find('#'), 1, "_+x");
|
||||
exists = g_core->platform->FilePathExists(tmp_name);
|
||||
} else {
|
||||
exists = g_core->platform->FilePathExists(file_out);
|
||||
}
|
||||
if (exists) {
|
||||
return file_out;
|
||||
// '#' denotes a cube map texture, which is actually 6 files.
|
||||
if (strchr(file_out.c_str(), '#')) {
|
||||
// Just look for one of them i guess.
|
||||
std::string tmp_name = file_out;
|
||||
tmp_name.replace(tmp_name.find('#'), 1, "_+x");
|
||||
exists = g_core->platform->FilePathExists(tmp_name);
|
||||
} else {
|
||||
exists = g_core->platform->FilePathExists(file_out);
|
||||
}
|
||||
if (exists) {
|
||||
return file_out;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -34,7 +34,9 @@ void LoadDDS(const std::string& file_name, unsigned char** buffers, int* widths,
|
||||
(*base_level) = 0;
|
||||
|
||||
FILE* f = g_core->platform->FOpen(file_name.c_str(), "rb");
|
||||
if (!f) throw Exception("can't open file: \"" + file_name + "\"");
|
||||
if (!f) {
|
||||
throw Exception("can't open file: \"" + file_name + "\"");
|
||||
}
|
||||
|
||||
DDS_header hdr{};
|
||||
|
||||
|
||||
@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int {
|
||||
namespace ballistica {
|
||||
|
||||
// These are set automatically via script; don't modify them here.
|
||||
const int kEngineBuildNumber = 21899;
|
||||
const int kEngineBuildNumber = 21902;
|
||||
const char* kEngineVersion = "1.7.36";
|
||||
const int kEngineApiVersion = 8;
|
||||
|
||||
|
||||
@ -88,9 +88,10 @@ class App:
|
||||
|
||||
# Make sure we can locate the project bacloud is being run from.
|
||||
self._project_root = Path(sys.argv[0]).parents[1]
|
||||
# Look for a few things we expect to have in a project.
|
||||
if not all(
|
||||
Path(self._project_root, name).is_dir()
|
||||
for name in ('tools', 'config', 'tests')
|
||||
Path(self._project_root, name).exists()
|
||||
for name in ['config/projectconfig.json', 'tools/batools']
|
||||
):
|
||||
raise CleanError('Unable to locate project directory.')
|
||||
|
||||
|
||||
@ -253,6 +253,14 @@ def lazybuild(target: str, category: LazyBuildCategory, command: str) -> None:
|
||||
'tools',
|
||||
'src/assets',
|
||||
'.efrocachemap',
|
||||
# Needed to rebuild on asset-package changes.
|
||||
'config/projectconfig.json',
|
||||
],
|
||||
# This file won't exist if we are using a dev asset-package,
|
||||
# in which case we want to always run so we can ask the
|
||||
# server for package updates each time.
|
||||
srcpaths_exist=[
|
||||
'.cache/asset_package_resolved',
|
||||
],
|
||||
command=command,
|
||||
filefilter=_filefilter,
|
||||
|
||||
@ -670,13 +670,18 @@ def asset_package_assemble() -> None:
|
||||
f'Expected a string asset-package-version; got {type(apversion)}.'
|
||||
)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
f'{pcommand.PROJROOT}/tools/bacloud',
|
||||
'assetpackage',
|
||||
'_assemble',
|
||||
apversion,
|
||||
flavor,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
f'{pcommand.PROJROOT}/tools/bacloud',
|
||||
'assetpackage',
|
||||
'_assemble',
|
||||
apversion,
|
||||
flavor,
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise CleanError(
|
||||
f'Failed to assemble {apversion}' f' ({flavor} flavor).'
|
||||
) from exc
|
||||
|
||||
@ -17,7 +17,7 @@ from efrotools.util import is_wsl_windows_build_path
|
||||
from efrotools.pyver import PYVER
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
from concurrent.futures import Future
|
||||
|
||||
# Suffix for the pyc files we include in stagings. We're using
|
||||
# deterministic opt pyc files; see PEP 552.
|
||||
@ -47,6 +47,7 @@ class AssetStager:
|
||||
self.src = f'{self.projroot}/build/assets'
|
||||
self.dst: str | None = None
|
||||
self.serverdst: str | None = None
|
||||
self.asset_package_flavor: str | None = None
|
||||
self.win_extras_src: str | None = None
|
||||
self.win_platform: str | None = None
|
||||
self.win_type: str | None = None
|
||||
@ -115,8 +116,21 @@ class AssetStager:
|
||||
if self.win_extras_src is not None:
|
||||
self._sync_windows_extras()
|
||||
|
||||
# Standard stuff in ba_data.
|
||||
self._sync_ba_data()
|
||||
# Legacy assets going into ba_data.
|
||||
self._sync_ba_data_legacy()
|
||||
|
||||
# New asset-package stuff going into ba_data.
|
||||
if self.asset_package_flavor is not None:
|
||||
self._sync_ba_data_new()
|
||||
|
||||
if self.include_binary_executable:
|
||||
self._sync_binary_executable()
|
||||
|
||||
if self.include_python_dylib:
|
||||
self._sync_python_dylib()
|
||||
|
||||
if self.include_shell_executable:
|
||||
self._sync_shell_executable()
|
||||
|
||||
# On Android we need to build a payload file so it knows what to
|
||||
# pull out of the apk.
|
||||
@ -170,6 +184,7 @@ class AssetStager:
|
||||
self.desc = 'cmake'
|
||||
self.dst = args[-1]
|
||||
self.tex_suffix = '.dds'
|
||||
self.asset_package_flavor = 'gui_desktop_v1'
|
||||
# Link/copy in a binary *if* builddir is provided.
|
||||
self.include_binary_executable = self.builddir is not None
|
||||
self.executable_name = 'ballisticakit'
|
||||
@ -449,8 +464,7 @@ class AssetStager:
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
def _sync_ba_data(self) -> None:
|
||||
# pylint: disable=too-many-branches
|
||||
def _sync_ba_data_legacy(self) -> None:
|
||||
assert self.dst is not None
|
||||
os.makedirs(f'{self.dst}/ba_data', exist_ok=True)
|
||||
cmd: list[str] = [
|
||||
@ -461,15 +475,16 @@ class AssetStager:
|
||||
'--prune-empty-dirs',
|
||||
]
|
||||
|
||||
# Normally we use --delete-excluded so that we can do sparse
|
||||
# syncs for quick iteration on android apks/etc. However for our
|
||||
# modular builds we need to avoid that flag because we do a
|
||||
# second pass after to sync in our python-dylib stuff and with
|
||||
# that flag it all gets blown on the first pass.
|
||||
if not self.include_python_dylib:
|
||||
# Traditionally we used --delete-excluded so that we could do
|
||||
# sparse syncs for quick iteration on android apks/etc. However
|
||||
# for our modular builds (and now for asset-package assets) we
|
||||
# need to avoid that flag because we do further passes after to
|
||||
# sync in python-dylib stuff or asset-package stuff and with that
|
||||
# flag it all gets blown away on the first pass.
|
||||
if not self.include_python_dylib and self.asset_package_flavor is None:
|
||||
cmd.append('--delete-excluded')
|
||||
else:
|
||||
# Shouldn't be trying to do sparse stuff.
|
||||
# Shouldn't be trying to do sparse stuff in server builds.
|
||||
if self.serverdst is not None:
|
||||
assert self.include_json and self.include_collision_meshes
|
||||
else:
|
||||
@ -481,8 +496,9 @@ class AssetStager:
|
||||
and self.include_meshes
|
||||
and self.include_collision_meshes
|
||||
)
|
||||
# Keep rsync from trying to prune this as an 'empty' dir.
|
||||
# Keep rsync from deleting the other stuff we're overlaying.
|
||||
cmd += ['--exclude', '/python-dylib']
|
||||
cmd += ['--exclude', '/textures2']
|
||||
|
||||
if self.include_scripts:
|
||||
cmd += [
|
||||
@ -524,14 +540,146 @@ class AssetStager:
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
if self.include_binary_executable:
|
||||
self._sync_binary_executable()
|
||||
def _sync_ba_data_new(self) -> None:
|
||||
# pylint: disable=too-many-locals
|
||||
import json
|
||||
import stat
|
||||
import shutil
|
||||
from threading import Lock
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
if self.include_python_dylib:
|
||||
self._sync_python_dylib()
|
||||
from bacommon.bacloud import asset_file_cache_path
|
||||
|
||||
if self.include_shell_executable:
|
||||
self._sync_shell_executable()
|
||||
assert self.asset_package_flavor is not None
|
||||
assert self.dst is not None
|
||||
|
||||
# Just going with raw json here instead of dataclassio to
|
||||
# maximize speed; we'll be going over lots of files here.
|
||||
with open(
|
||||
f'.cache/assetmanifests/{self.asset_package_flavor}',
|
||||
encoding='utf-8',
|
||||
) as infile:
|
||||
manifest = json.loads(infile.read())
|
||||
|
||||
filehashes: dict[str, str] = manifest['h']
|
||||
|
||||
mkdirlock = Lock()
|
||||
|
||||
def _prep_syncdir(syncdir: str) -> None:
|
||||
# First, take a pass through and delete all files not found in
|
||||
# our manifest.
|
||||
assert self.dst is not None
|
||||
dstdir = os.path.join(self.dst, syncdir)
|
||||
os.makedirs(dstdir, exist_ok=True)
|
||||
for entry in os.scandir(dstdir):
|
||||
if entry.is_file():
|
||||
path = os.path.join(syncdir, entry.name)
|
||||
if path not in filehashes:
|
||||
os.unlink(os.path.join(self.dst, path))
|
||||
|
||||
def _sync_path(src: str, dst: str) -> None:
|
||||
# Quick-out: if there's a file already at dst and its
|
||||
# modtime and size *exactly* match src, we're done. Note
|
||||
# that this is a bit different than Makefile logic where
|
||||
# things update when src is newer than dst. In our case,
|
||||
# a manifest change could cause src to point to a cache
|
||||
# file with an *older* modtime than the previous one
|
||||
# (cache file modtimes are static and arbitrary) so such
|
||||
# logic doesn't work. However if we look for an *exact*
|
||||
# modtime match as well as size match we can be
|
||||
# reasonably sure that the file is still the same. We'll
|
||||
# see how this goes...
|
||||
srcstat = os.stat(src)
|
||||
try:
|
||||
dststat = os.stat(dst)
|
||||
except FileNotFoundError:
|
||||
dststat = None
|
||||
if (
|
||||
dststat is not None
|
||||
and srcstat.st_size == dststat.st_size
|
||||
and srcstat.st_mtime == dststat.st_mtime
|
||||
):
|
||||
return
|
||||
|
||||
# If dst is a directory, blow it away (use the stat we
|
||||
# already fetched to save a bit of time).
|
||||
if dststat is not None and stat.S_ISDIR(dststat.st_mode):
|
||||
shutil.rmtree(dst)
|
||||
|
||||
# Hold a lock while creating any parent directories just in
|
||||
# case multiple files are trying to create the same
|
||||
# directory simultaneously (not sure if that could cause
|
||||
# problems but might as well be extra safe).
|
||||
with mkdirlock:
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
|
||||
# Ok, dst doesn't exist or modtimes don't line up. Copy it
|
||||
# and try to copy its modtime.
|
||||
shutil.copyfile(src, dst)
|
||||
shutil.copystat(src, dst)
|
||||
|
||||
def _cleanup_syncdir(syncdir: str) -> None:
|
||||
"""Handle pruning empty directories."""
|
||||
# Walk the tree bottom-up so we can properly kill recursive
|
||||
# empty dirs.
|
||||
assert self.dst is not None
|
||||
dstdir = os.path.join(self.dst, syncdir)
|
||||
for basename, dirnames, filenames in os.walk(dstdir, topdown=False):
|
||||
# It seems that child dirs we kill during the walk are still
|
||||
# listed when the parent dir is visited, so lets make sure
|
||||
# to only acknowledge still-existing ones.
|
||||
dirnames = [
|
||||
d
|
||||
for d in dirnames
|
||||
if os.path.exists(os.path.join(basename, d))
|
||||
]
|
||||
if not dirnames and not filenames and basename != dstdir:
|
||||
os.rmdir(basename)
|
||||
|
||||
syncdirs: list[str] = ['ba_data/textures2']
|
||||
|
||||
futures: list[Future]
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
|
||||
# First, prep each of our sync dirs (make sure they exist,
|
||||
# blow away files not in our manifest, etc.)
|
||||
futures = []
|
||||
for syncdir in syncdirs:
|
||||
futures.append(executor.submit(_prep_syncdir, syncdir=syncdir))
|
||||
# Await all results to get any exceptions.
|
||||
for future in futures:
|
||||
_result = future.result()
|
||||
|
||||
# Now go through all our manifest paths, syncing any files
|
||||
# destined for any of our syncdirs.
|
||||
futures = []
|
||||
for path, hashval in filehashes.items():
|
||||
for syncdir in syncdirs:
|
||||
if path.startswith(syncdir):
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_sync_path,
|
||||
src=(
|
||||
f'.cache/assetdata/'
|
||||
f'{asset_file_cache_path(hashval)}'
|
||||
),
|
||||
dst=os.path.join(self.dst, path),
|
||||
)
|
||||
)
|
||||
# Await all results to get any exceptions.
|
||||
for future in futures:
|
||||
_result = future.result()
|
||||
|
||||
# Lastly, run a cleanup pass on all our sync dirs (blow away
|
||||
# empty dirs, etc.)
|
||||
futures = []
|
||||
for syncdir in syncdirs:
|
||||
futures.append(
|
||||
executor.submit(_cleanup_syncdir, syncdir=syncdir)
|
||||
)
|
||||
# Await all results to get any exceptions.
|
||||
for future in futures:
|
||||
_result = future.result()
|
||||
|
||||
def _sync_shell_executable(self) -> None:
|
||||
if self.executable_name is None:
|
||||
|
||||
@ -52,11 +52,13 @@ class LazyBuildContext:
|
||||
dirfilter: Callable[[str, str], bool] | None = None,
|
||||
filefilter: Callable[[str, str], bool] | None = None,
|
||||
srcpaths_fullclean: list[str] | None = None,
|
||||
srcpaths_exist: list[str] | None = None,
|
||||
manifest_file: str | None = None,
|
||||
command_fullclean: str | None = None,
|
||||
) -> None:
|
||||
self.target = target
|
||||
self.srcpaths = srcpaths
|
||||
self.srcpaths_exist = srcpaths_exist
|
||||
self.command = command
|
||||
self.dirfilter = dirfilter
|
||||
self.filefilter = filefilter
|
||||
@ -158,6 +160,7 @@ class LazyBuildContext:
|
||||
Path(self.target).touch()
|
||||
|
||||
def _check_for_changes(self) -> None:
|
||||
# pylint: disable=too-many-branches
|
||||
manfile = self.manifest_file
|
||||
# If we're watching for file adds/removes/renames in addition
|
||||
# to just modtimes, build a set of all files we come across.
|
||||
@ -186,6 +189,7 @@ class LazyBuildContext:
|
||||
src_did_change, src_unchanged_count = self._check_path(
|
||||
srcpath, man_input_paths
|
||||
)
|
||||
|
||||
if src_did_change:
|
||||
self.have_changes = True
|
||||
# If we're *not* building a manifest
|
||||
@ -194,6 +198,13 @@ class LazyBuildContext:
|
||||
return
|
||||
self.total_unchanged_count += src_unchanged_count
|
||||
|
||||
# Check our exist-only paths; we simply look to see if these exist
|
||||
# and build if not.
|
||||
if self.srcpaths_exist is not None:
|
||||
for srcpath in self.srcpaths_exist:
|
||||
if not os.path.exists(srcpath):
|
||||
self.have_changes = True
|
||||
|
||||
# If we built a manifest, check/write it and kick off
|
||||
# a full-clean if anything differed.
|
||||
if manfile is not None:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user