mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-19 21:37:57 +08:00
609 lines
20 KiB
Python
Executable File
609 lines
20 KiB
Python
Executable File
# Released under the MIT License. See LICENSE for details.
|
|
#
|
|
"""Updates src/assets/Makefile based on source assets present."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from typing import TYPE_CHECKING
|
|
|
|
from efrotools.pyver import PYVERNODOT
|
|
|
|
if TYPE_CHECKING:
|
|
pass
|
|
|
|
PYC_SUFFIX = f'.cpython-{PYVERNODOT}.opt-1.pyc'
|
|
|
|
ASSETS_SRC = 'src/assets'
|
|
BUILD_DIR = 'build/assets'
|
|
|
|
|
|
def _get_targets(
|
|
projroot: str,
|
|
varname: str,
|
|
inext: str,
|
|
outext: str,
|
|
all_targets: set,
|
|
limit_to_prefix: str | None = None,
|
|
) -> str:
|
|
"""Generic function to map source extension to dst files."""
|
|
# pylint: disable=too-many-locals
|
|
# pylint: disable=too-many-positional-arguments
|
|
|
|
src = ASSETS_SRC
|
|
dst = BUILD_DIR
|
|
targets = []
|
|
|
|
# Create outext targets for all inext files we find.
|
|
for root, _dname, fnames in os.walk(os.path.join(projroot, src)):
|
|
src_abs = os.path.join(projroot, src)
|
|
if limit_to_prefix is not None and not root.startswith(
|
|
os.path.join(src_abs, limit_to_prefix)
|
|
):
|
|
continue
|
|
|
|
# Write the target to make sense from within src/assets/
|
|
assert root.startswith(src_abs)
|
|
dstrootvar = '$(BUILD_DIR)' + root.removeprefix(src_abs)
|
|
dstfin = dst + root.removeprefix(src_abs)
|
|
for fname in fnames:
|
|
outname = fname[: -len(inext)] + outext
|
|
if fname.endswith(inext):
|
|
all_targets.add(os.path.join(dstfin, outname))
|
|
targets.append(os.path.join(dstrootvar, outname))
|
|
|
|
return '\n' + varname + ' = \\\n ' + ' \\\n '.join(sorted(targets))
|
|
|
|
|
|
def _get_py_targets(
|
|
projroot: str,
|
|
meta_manifests: dict[str, str],
|
|
explicit_sources: set[str],
|
|
src: str,
|
|
dst: str,
|
|
py_targets: list[str],
|
|
pyc_targets: list[str],
|
|
all_targets: set[str],
|
|
subset: str,
|
|
) -> None:
|
|
# pylint: disable=too-many-positional-arguments
|
|
# pylint: disable=too-many-branches
|
|
# pylint: disable=too-many-locals
|
|
# pylint: disable=too-many-statements
|
|
|
|
py_generated_root = f'{ASSETS_SRC}/ba_data/python/babase/_mgen'
|
|
|
|
def _do_get_targets(
|
|
proot: str, fnames: list[str], is_explicit: bool = False
|
|
) -> None:
|
|
# Special case: don't make targets for stuff in specific dirs.
|
|
if proot in {
|
|
f'{ASSETS_SRC}/ba_data/data/maps',
|
|
f'{ASSETS_SRC}/mac_disk_image',
|
|
f'{ASSETS_SRC}/workspace',
|
|
}:
|
|
return
|
|
assert proot.startswith(src), f'{proot} does not start with {src}'
|
|
assert dst.startswith(BUILD_DIR)
|
|
dstrootvar = (
|
|
'$(BUILD_DIR)'
|
|
+ dst.removeprefix(BUILD_DIR)
|
|
+ proot.removeprefix(src)
|
|
)
|
|
dstfin = dst + proot[len(src) :]
|
|
for fname in fnames:
|
|
# Ignore non-python files and flycheck/emacs temp files.
|
|
if (
|
|
not fname.endswith('.py')
|
|
or fname.startswith('flycheck_')
|
|
or fname.startswith('.#')
|
|
):
|
|
continue
|
|
|
|
# Ignore any files in the list of explicit sources we got;
|
|
# we explicitly add those at the end and don't want to do it
|
|
# twice (since we don't know if this one will always exist
|
|
# anyway).
|
|
if (
|
|
os.path.join(proot, fname) in explicit_sources
|
|
and not is_explicit
|
|
):
|
|
continue
|
|
|
|
if proot.startswith(f'{ASSETS_SRC}/ba_data/python-site-packages'):
|
|
in_subset = 'private-common'
|
|
elif proot.startswith(f'{ASSETS_SRC}/ba_data') or proot.startswith(
|
|
f'{ASSETS_SRC}/server'
|
|
):
|
|
in_subset = 'public'
|
|
elif proot.startswith('tools/efro') and not proot.startswith(
|
|
'tools/efrotools'
|
|
):
|
|
# We want to pull just 'efro' out of tools; not efrotools.
|
|
in_subset = 'public_tools'
|
|
elif proot.startswith('tools/bacommon'):
|
|
in_subset = 'public_tools'
|
|
elif proot.startswith(f'{ASSETS_SRC}/windows/x64'):
|
|
in_subset = 'private-windows-x64'
|
|
elif proot.startswith(f'{ASSETS_SRC}/windows/Win32'):
|
|
in_subset = 'private-windows-Win32'
|
|
elif proot.startswith(f'{ASSETS_SRC}/pylib-apple'):
|
|
in_subset = 'private-apple'
|
|
elif proot.startswith(f'{ASSETS_SRC}/pylib-android'):
|
|
in_subset = 'private-android'
|
|
else:
|
|
in_subset = 'private-common'
|
|
|
|
if subset == 'all':
|
|
pass
|
|
elif subset != in_subset:
|
|
continue
|
|
|
|
# gamedata pass includes only data; otherwise do all else
|
|
|
|
# .py:
|
|
targetpath = os.path.join(dstfin, fname)
|
|
assert targetpath not in all_targets
|
|
all_targets.add(targetpath)
|
|
py_targets.append(os.path.join(dstrootvar, fname))
|
|
|
|
# and .pyc:
|
|
fname_pyc = fname[:-3] + PYC_SUFFIX
|
|
all_targets.add(os.path.join(dstfin, '__pycache__', fname_pyc))
|
|
pyc_targets.append(
|
|
os.path.join(dstrootvar, '__pycache__', fname_pyc)
|
|
)
|
|
|
|
# Create py and pyc targets for all physical scripts in src, with
|
|
# the exception of our dynamically generated stuff.
|
|
for physical_root, _dname, physical_fnames in os.walk(
|
|
os.path.join(projroot, src)
|
|
):
|
|
# Skip any generated files; we'll add those from the meta manifest.
|
|
# (dont want our results to require a meta build beforehand)
|
|
if physical_root == os.path.join(
|
|
projroot, py_generated_root
|
|
) or physical_root.startswith(
|
|
os.path.join(projroot, py_generated_root) + '/'
|
|
):
|
|
continue
|
|
|
|
_do_get_targets(
|
|
physical_root.removeprefix(projroot + '/'), physical_fnames
|
|
)
|
|
|
|
# Now create targets for any of our dynamically generated stuff that
|
|
# lives under this dir.
|
|
meta_targets: list[str] = []
|
|
for manifest in meta_manifests.values():
|
|
# Sanity check; make sure meta system is giving actual paths;
|
|
# no accidental makefile vars.
|
|
if '$' in manifest:
|
|
raise RuntimeError(
|
|
'meta-manifest value contains a $; probably a bug.'
|
|
)
|
|
meta_targets += json.loads(manifest)
|
|
|
|
meta_targets = [
|
|
t
|
|
for t in meta_targets
|
|
if t.startswith(src + '/') and t.startswith(py_generated_root + '/')
|
|
]
|
|
|
|
for target in meta_targets:
|
|
_do_get_targets(
|
|
proot=os.path.dirname(target), fnames=[os.path.basename(target)]
|
|
)
|
|
|
|
# Now create targets for any explicitly passed paths.
|
|
for expsrc in explicit_sources:
|
|
if expsrc.startswith(f'{src}/'):
|
|
_do_get_targets(
|
|
proot=os.path.dirname(expsrc),
|
|
fnames=[os.path.basename(expsrc)],
|
|
is_explicit=True,
|
|
)
|
|
|
|
|
|
def _get_py_targets_subset(
|
|
projroot: str,
|
|
meta_manifests: dict[str, str],
|
|
explicit_sources: set[str],
|
|
all_targets: set[str],
|
|
subset: str,
|
|
suffix: str,
|
|
) -> str:
|
|
# pylint: disable=too-many-locals
|
|
# pylint: disable=too-many-positional-arguments
|
|
if subset == 'public_tools':
|
|
src = 'tools'
|
|
dst = f'{BUILD_DIR}/ba_data/python'
|
|
copyrule = '$(BUILD_DIR)/ba_data/python/%.py : $(TOOLS_DIR)/%.py'
|
|
else:
|
|
src = ASSETS_SRC
|
|
dst = BUILD_DIR
|
|
copyrule = '$(BUILD_DIR)/%.py : %.py'
|
|
|
|
# Separate these into '1' and '2'.
|
|
py_targets: list[str] = []
|
|
pyc_targets: list[str] = []
|
|
|
|
_get_py_targets(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
src,
|
|
dst,
|
|
py_targets,
|
|
pyc_targets,
|
|
all_targets,
|
|
subset=subset,
|
|
)
|
|
|
|
# Need to sort these combined to keep pairs together.
|
|
combined_targets = [
|
|
(py_targets[i], pyc_targets[i]) for i in range(len(py_targets))
|
|
]
|
|
combined_targets.sort()
|
|
py_targets = [t[0] for t in combined_targets]
|
|
pyc_targets = [t[1] for t in combined_targets]
|
|
|
|
out = (
|
|
f'\nSCRIPT_TARGETS_PY{suffix} = \\\n '
|
|
+ ' \\\n '.join(py_targets)
|
|
+ '\n'
|
|
)
|
|
|
|
out += (
|
|
f'\nSCRIPT_TARGETS_PYC{suffix} = \\\n '
|
|
+ ' \\\n '.join(pyc_targets)
|
|
+ '\n'
|
|
)
|
|
|
|
# We transform all non-public targets into efrocache-fetches in public.
|
|
efc = '' if subset.startswith('public') else '# __EFROCACHE_TARGET__\n'
|
|
|
|
out += (
|
|
'\n# Rule to copy src asset scripts to dst.\n'
|
|
'# (and make non-writable so I\'m less likely to '
|
|
'accidentally edit them there)\n'
|
|
f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
|
|
# '#\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
|
|
'\t@$(PCOMMANDBATCH) copy_python_file $^ $@\n'
|
|
)
|
|
|
|
# out += (
|
|
# '\n# Rule to copy src asset scripts to dst.\n'
|
|
# '# (and make non-writable so I\'m less likely to '
|
|
# 'accidentally edit them there)\n'
|
|
# f'{efc}$(SCRIPT_TARGETS_PY{suffix}) : {copyrule}\n'
|
|
# '\t@echo Copying script: $(subst $(BUILD_DIR)/,,$@)\n'
|
|
# '\t@mkdir -p $(dir $@)\n'
|
|
# '\t@rm -f $@\n'
|
|
# '\t@cp $^ $@\n'
|
|
# '\t@chmod 444 $@\n'
|
|
# )
|
|
|
|
# Fancy new simple loop-based target generation.
|
|
out += (
|
|
f'\n# These are too complex to define in a pattern rule;\n'
|
|
f'# Instead we generate individual targets in a loop.\n'
|
|
f'$(foreach element,$(SCRIPT_TARGETS_PYC{suffix}),\\\n'
|
|
f'$(eval $(call make-opt-pyc-target,$(element))))'
|
|
)
|
|
|
|
# Old code to explicitly emit individual targets.
|
|
if bool(False):
|
|
out += (
|
|
'\n# Looks like path mangling from py to pyc is too complex for'
|
|
' pattern rules so\n# just generating explicit targets'
|
|
' for each. Could perhaps look into using a\n# fancy for-loop'
|
|
' instead, but perhaps listing these explicitly isn\'t so bad.\n'
|
|
)
|
|
for i, target in enumerate(pyc_targets):
|
|
# Note: there's currently a bug which can cause python bytecode
|
|
# generation to be non-deterministic. This can break our blessing
|
|
# process since we bless in core but then regenerate bytecode in
|
|
# spinoffs. See https://bugs.python.org/issue34722
|
|
# For now setting PYTHONHASHSEED=1 is a workaround.
|
|
out += (
|
|
'\n'
|
|
+ target
|
|
+ ': \\\n '
|
|
+ py_targets[i]
|
|
+ '\n\t@echo Compiling script: $(subst $(BUILD_DIR),,$^)\n'
|
|
'\t@rm -rf $@ && PYTHONHASHSEED=1 $(TOOLS_DIR)/pcommand'
|
|
' compile_python_file $^'
|
|
' && chmod 444 $@\n'
|
|
)
|
|
|
|
return out
|
|
|
|
|
|
def _get_extras_targets_win(
|
|
projroot: str, all_targets: set[str], platform: str
|
|
) -> str:
|
|
targets: list[str] = []
|
|
base = f'{ASSETS_SRC}/windows'
|
|
dstbase = 'windows'
|
|
for root, _dnames, fnames in os.walk(os.path.join(projroot, base)):
|
|
for fname in fnames:
|
|
# Only include the platform we were passed.
|
|
if not root.startswith(
|
|
os.path.join(projroot, f'{ASSETS_SRC}/windows/{platform}')
|
|
):
|
|
continue
|
|
|
|
ext = os.path.splitext(fname)[-1]
|
|
|
|
# "I don't like .DS_Store files. They're coarse and rough and
|
|
# irritating and they get everywhere."
|
|
if fname == '.DS_Store':
|
|
continue
|
|
|
|
# Ignore python files as they're handled separately.
|
|
if ext in ['.py', '.pyc']:
|
|
continue
|
|
|
|
# Various stuff we expect to be there...
|
|
if ext in [
|
|
'.exe',
|
|
'.dll',
|
|
'.bat',
|
|
'.txt',
|
|
'.whl',
|
|
'.ps1',
|
|
'.css',
|
|
'.sample',
|
|
'.ico',
|
|
'.pyd',
|
|
'.ctypes',
|
|
'.rst',
|
|
'.fish',
|
|
'.csh',
|
|
'.cat',
|
|
'.pdb',
|
|
'.lib',
|
|
'.html',
|
|
] or fname in [
|
|
'activate',
|
|
'README',
|
|
'command_template',
|
|
'fetch_macholib',
|
|
]:
|
|
base_abs = os.path.join(projroot, base)
|
|
assert root.startswith(base_abs)
|
|
targetpath = os.path.join(
|
|
dstbase + root.removeprefix(base_abs), fname
|
|
)
|
|
# print(f'DSTBASE {dstbase} ROOT {root}
|
|
# TARGETPATH {targetpath}')
|
|
targets.append('$(BUILD_DIR)/' + targetpath)
|
|
all_targets.add(BUILD_DIR + '/' + targetpath)
|
|
continue
|
|
|
|
# Complain if something new shows up instead of blindly
|
|
# including it.
|
|
raise RuntimeError(f'Unexpected extras file: {fname}')
|
|
|
|
targets.sort()
|
|
p_up = platform.upper()
|
|
out = (
|
|
f'\nEXTRAS_TARGETS_WIN_{p_up} = \\\n ' + ' \\\n '.join(targets) + '\n'
|
|
)
|
|
|
|
# We transform all these targets into efrocache-fetches in public.
|
|
out += (
|
|
'\n# Rule to copy src extras to build.\n'
|
|
f'# __EFROCACHE_TARGET__\n'
|
|
f'$(EXTRAS_TARGETS_WIN_{p_up}) : $(BUILD_DIR)/% :'
|
|
' %\n'
|
|
'\t@$(PCOMMANDBATCH) copy_win_extra_file $^ $@\n'
|
|
# '\t@echo Copying file: $(subst $(BUILD_DIR)/,,$@)\n'
|
|
# '\t@mkdir -p $(dir $@)\n'
|
|
# '\t@rm -f $@\n'
|
|
# '\t@cp $^ $@\n'
|
|
)
|
|
|
|
return out
|
|
|
|
|
|
def generate_assets_makefile(
|
|
projroot: str,
|
|
fname: str,
|
|
existing_data: str,
|
|
meta_manifests: dict[str, str],
|
|
explicit_sources: set[str],
|
|
) -> dict[str, str]:
|
|
"""Main script entry point."""
|
|
# pylint: disable=too-many-locals
|
|
from efrotools.project import getprojectconfig
|
|
from pathlib import Path
|
|
|
|
public = getprojectconfig(Path(projroot))['public']
|
|
assert isinstance(public, bool)
|
|
|
|
original = existing_data
|
|
lines = original.splitlines()
|
|
|
|
auto_start_public = lines.index('# __AUTOGENERATED_PUBLIC_BEGIN__')
|
|
auto_end_public = lines.index('# __AUTOGENERATED_PUBLIC_END__')
|
|
auto_start_private = lines.index('# __AUTOGENERATED_PRIVATE_BEGIN__')
|
|
auto_end_private = lines.index('# __AUTOGENERATED_PRIVATE_END__')
|
|
|
|
all_targets_public: set[str] = set()
|
|
all_targets_private: set[str] = set()
|
|
|
|
# We always auto-generate the public section.
|
|
our_lines_public = [
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_public,
|
|
subset='public',
|
|
suffix='_PUBLIC',
|
|
),
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_public,
|
|
subset='public_tools',
|
|
suffix='_PUBLIC_TOOLS',
|
|
),
|
|
]
|
|
|
|
# Only auto-generate the private section in the private repo.
|
|
if public:
|
|
our_lines_private = lines[auto_start_private + 1 : auto_end_private]
|
|
else:
|
|
our_lines_private = [
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_private,
|
|
subset='private-apple',
|
|
suffix='_PRIVATE_APPLE',
|
|
),
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_private,
|
|
subset='private-android',
|
|
suffix='_PRIVATE_ANDROID',
|
|
),
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_private,
|
|
subset='private-common',
|
|
suffix='_PRIVATE_COMMON',
|
|
),
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_private,
|
|
subset='private-windows-Win32',
|
|
suffix='_PRIVATE_WIN_WIN32',
|
|
),
|
|
_get_py_targets_subset(
|
|
projroot,
|
|
meta_manifests,
|
|
explicit_sources,
|
|
all_targets_private,
|
|
subset='private-windows-x64',
|
|
suffix='_PRIVATE_WIN_X64',
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'COB_TARGETS',
|
|
'.collisionmesh.obj',
|
|
'.cob',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'BOB_TARGETS',
|
|
'.mesh.obj',
|
|
'.bob',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'FONT_TARGETS',
|
|
'.fdata',
|
|
'.fdata',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'PEM_TARGETS',
|
|
'.pem',
|
|
'.pem',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'DATA_TARGETS',
|
|
'.json',
|
|
'.json',
|
|
all_targets_private,
|
|
limit_to_prefix='ba_data/data',
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'AUDIO_TARGETS',
|
|
'.wav',
|
|
'.ogg',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'TEX2D_DDS_TARGETS',
|
|
'.tex2d.png',
|
|
'.dds',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'TEX2D_PVR_TARGETS',
|
|
'.tex2d.png',
|
|
'.pvr',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'TEX2D_KTX_TARGETS',
|
|
'.tex2d.png',
|
|
'.ktx',
|
|
all_targets_private,
|
|
),
|
|
_get_targets(
|
|
projroot,
|
|
'TEX2D_PREVIEW_PNG_TARGETS',
|
|
'.tex2d.png',
|
|
'_preview.png',
|
|
all_targets_private,
|
|
),
|
|
_get_extras_targets_win(projroot, all_targets_private, 'Win32'),
|
|
_get_extras_targets_win(projroot, all_targets_private, 'x64'),
|
|
]
|
|
filtered = (
|
|
lines[: auto_start_public + 1]
|
|
+ our_lines_public
|
|
+ lines[auto_end_public : auto_start_private + 1]
|
|
+ our_lines_private
|
|
+ lines[auto_end_private:]
|
|
)
|
|
out_files: dict[str, str] = {}
|
|
|
|
out = '\n'.join(filtered) + '\n'
|
|
|
|
out_files[fname] = out
|
|
|
|
# Write a simple manifest of the things we expect to have in build.
|
|
# We can use this to clear out orphaned files as part of builds.
|
|
out_files['src/assets/.asset_manifest_public.json'] = _gen_manifest(
|
|
all_targets_public
|
|
)
|
|
# Only *generate* the private manifest in the private repo. In public
|
|
# we just give what's already on disk.
|
|
manprivpath = 'src/assets/.asset_manifest_private.json'
|
|
if not public:
|
|
out_files[manprivpath] = _gen_manifest(all_targets_private)
|
|
return out_files
|
|
|
|
|
|
def _gen_manifest(all_targets: set[str]) -> str:
|
|
# Lastly, write a simple manifest of the things we expect to have
|
|
# in build. We can use this to clear out orphaned files as part of builds.
|
|
assert all(t.startswith(BUILD_DIR) for t in all_targets)
|
|
manifest = sorted(t[13:] for t in all_targets)
|
|
return json.dumps(manifest, indent=1)
|