diff --git a/Makefile b/Makefile index 3be1dd02..2acb74ce 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,27 @@ all: help .PHONY: all +################################################################################ +# # +# Prefab # +# # +################################################################################ + +# Prebuilt binaries for various platforms. + +prefab-mac: prefab-mac-build + @cd build/prefab-mac/debug && ./ballisticacore + +prefab-mac-build: assets-cmake build/prefab-mac/debug/ballisticacore + @${STAGE_ASSETS} -cmake build/prefab-mac/debug + +build/prefab-mac/debug/ballisticacore: .efrocachemap + @tools/snippets efrocache_get $@ + +# Tell make which of these targets don't represent files. +.PHONY: prefab-mac prefab-mac-build + + ################################################################################ # # # General # diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 72a79eb8..8aba17ca 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -164,11 +164,11 @@ def update_cache(makefile_dirs: List[str]) -> None: cdp = f'cd {path} && ' if path else '' mfpath = os.path.join(path, 'Makefile') print(f'Building cache targets for {mfpath}...') - subprocess.run(f'{cdp}make -j{cpus} efrocache_build', + subprocess.run(f'{cdp}make -j{cpus} efrocache-build', shell=True, check=True) - rawpaths = subprocess.run(f'{cdp}make efrocache_list', + rawpaths = subprocess.run(f'{cdp}make efrocache-list', shell=True, check=True, capture_output=True).stdout.decode().split() diff --git a/tools/stage_assets b/tools/stage_assets new file mode 100755 index 00000000..1c3f727e --- /dev/null +++ b/tools/stage_assets @@ -0,0 +1,353 @@ +#!/usr/bin/env python3.7 +# Copyright (c) 2011-2019 Eric Froemling +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ----------------------------------------------------------------------------- +"""Stage assets for a build.""" + +from __future__ import annotations + +import hashlib +import os +import sys +from functools import partial +from typing import TYPE_CHECKING + +from efrotools.pybuild import PYTHON_VERSION_MAJOR + +if TYPE_CHECKING: + from typing import Optional + +# Suffix for the pyc files we include in stagings. +# We're using deterministic opt pyc files; see PEP 552. +# Note: this means anyone wanting to modify .py files in a build +# will need to wipe out the existing .pyc files first or the changes +# will be ignored. +OPT_PYC_SUFFIX = ('cpython-' + PYTHON_VERSION_MAJOR.replace('.', '') + + '.opt-1.pyc') + + +class Config: + """Encapsulates command options.""" + + def __init__(self) -> None: + # We always calc src relative to this script. + self.src = os.path.abspath( + os.path.dirname(sys.argv[0]) + '/../assets/build') + self.dst: Optional[str] = None + self.win_extras_src: Optional[str] = None + self.include_audio = True + self.include_models = True + self.include_collide_models = True + self.include_scripts = True + self.include_python = True + self.include_textures = True + self.include_fonts = True + self.include_json = True + self.include_pylib = False + self.pylib_src_name: Optional[str] = None + self.include_payload_file = False + self.tex_suffix: Optional[str] = None + self.is_payload_full = False + + def _parse_android_args(self) -> None: + # On Android we get nitpicky with what + # we want to copy in since we can speed up + # iterations by installing stripped down + # apks. + self.dst = 'assets/ballistica_files' + self.pylib_src_name = 'pylib-android' + self.include_payload_file = True + self.tex_suffix = '.ktx' + self.include_audio = False + self.include_models = False + self.include_collide_models = False + self.include_scripts = False + self.include_python = False + self.include_textures = False + self.include_fonts = False + self.include_json = False + self.include_pylib = False + for arg in sys.argv[1:]: + if arg == '-full': + self.include_audio = True + self.include_models = True + self.include_collide_models = True + self.include_scripts = True + self.include_python = True + self.include_textures = True + self.include_fonts = True + self.include_json = True + self.is_payload_full = True + self.include_pylib = True + elif arg == '-none': + pass + elif arg == '-models': + self.include_models = True + self.include_collide_models = True + elif arg == '-python': + self.include_python = True + self.include_pylib = True + elif arg == '-textures': + self.include_textures = True + elif arg == '-fonts': + self.include_fonts = True + elif arg == '-scripts': + self.include_scripts = True + elif arg == '-audio': + self.include_audio = True + + def parse_args(self) -> None: + """Parse args and apply to the cfg.""" + if '-android' in sys.argv: + self._parse_android_args() + elif '-cmake' in sys.argv: + self.dst = sys.argv[2] + self.tex_suffix = '.dds' + elif '-win-Win32' in sys.argv: + self.dst = sys.argv[2] + self.tex_suffix = '.dds' + self.win_extras_src = os.path.abspath( + os.path.dirname(sys.argv[0]) + + '/../ballisticacore-windows/Extras/Win32') + elif '-win-x64' in sys.argv: + self.dst = sys.argv[2] + self.tex_suffix = '.dds' + self.win_extras_src = os.path.abspath( + os.path.dirname(sys.argv[0]) + + '/../ballisticacore-windows/Extras/x64') + elif '-win-server-Win32' in sys.argv: + self.dst = sys.argv[2] + self.win_extras_src = os.path.abspath( + os.path.dirname(sys.argv[0]) + + '/../ballisticacore-windows/Extras/Win32') + self.include_textures = False + self.include_audio = False + self.include_models = False + elif '-win-server-x64' in sys.argv: + self.dst = sys.argv[2] + self.win_extras_src = os.path.abspath( + os.path.dirname(sys.argv[0]) + + '/../ballisticacore-windows/Extras/x64') + self.include_textures = False + self.include_audio = False + self.include_models = False + elif '-cmake-server' in sys.argv: + self.dst = sys.argv[2] + self.include_textures = False + self.include_audio = False + self.include_models = False + + elif '-xcode-mac' in sys.argv: + self.src = os.environ['SOURCE_ROOT'] + '/assets/build' + self.dst = (os.environ['TARGET_BUILD_DIR'] + '/' + + os.environ['UNLOCALIZED_RESOURCES_FOLDER_PATH']) + self.include_pylib = True + self.pylib_src_name = 'pylib-apple' + self.tex_suffix = '.dds' + elif '-xcode-ios' in sys.argv: + self.src = os.environ['SOURCE_ROOT'] + '/assets/build' + self.dst = (os.environ['TARGET_BUILD_DIR'] + '/' + + os.environ['UNLOCALIZED_RESOURCES_FOLDER_PATH']) + self.include_pylib = True + self.pylib_src_name = 'pylib-apple' + self.tex_suffix = '.pvr' + else: + raise Exception('no valid platform set') + + +def md5sum(filename: str) -> str: + """Generate an md5sum given a filename.""" + md5 = hashlib.md5() + with open(filename, mode='rb') as infile: + for buf in iter(partial(infile.read, 128), b''): + md5.update(buf) + return md5.hexdigest() + + +def _run(cmd: str, echo: bool = False) -> None: + """Run an os command; raise Exception on non-zero return value.""" + if echo: + print(cmd) + result = os.system(cmd) + if result != 0: + raise Exception("error running cmd: '" + cmd + "'") + + +def _write_payload_file(assets_root: str, full: bool) -> None: + if not assets_root.endswith('/'): + assets_root = assets_root + '/' + + # Now construct a payload file if we have any files. + file_list = [] + payload_str = '' + for root, _subdirs, fnames in os.walk(assets_root): + for fname in fnames: + if fname.startswith('.'): + continue + if fname == 'payload_info': + continue + fpath = os.path.join(root, fname) + fpathshort = fpath.replace(assets_root, '') + if ' ' in fpathshort: + raise Exception("invalid filename found (contains spaces): '" + + fpathshort + "'") + payload_str += fpathshort + ' ' + md5sum(fpath) + '\n' + file_list.append(fpathshort) + + if file_list: + # Write the file count, whether this is a 'full' payload, and finally + # the file list. + payload_str = (str(len(file_list)) + '\n' + ('1' if full else '0') + + '\n' + payload_str) + payload_path = assets_root + '/payload_info' + with open(payload_path, 'w') as outfile: + outfile.write(payload_str) + else: + # Hmm; do we need to build an empty payload in this case? + pass + + +def _sync_windows_extras(cfg: Config) -> str: + assert cfg.win_extras_src is not None + if not os.path.isdir(cfg.win_extras_src): + raise Exception('win extras src dir not found: ' + cfg.win_extras_src) + + # Ok, lets do full syncs on each subdir we find so we + # properly delete anything in dst that disappeared from src. + # Lastly we'll sync over the remaining top level files. + # It'll technically be possible to orphaned top level + # files in dst, but we should be pulling those into dists + # individually by name anyway so it should be safe. + for dirname in ('DLLs', 'Lib'): + # We also need to be more particular about which pyc files we pull + # over. The Python stuff in Extras is also used by some scripts + # so there may be arbitrary non-opt pycs hanging around. + assert cfg.dst is not None + _run(f'mkdir -p "{cfg.dst}/{dirname}"') + cmd = ("rsync --recursive --update --delete --delete-excluded " + " --prune-empty-dirs" + " --include '*.ico' --include '*.cat'" + " --include '*.dll' --include '*.pyd'" + " --include '*.py' --include '*." + OPT_PYC_SUFFIX + "'" + " --include '*/' --exclude '*' \"" + + os.path.join(cfg.win_extras_src, dirname) + "/\" " + "\"" + cfg.dst + "/" + dirname + "/\"") + _run(cmd) + + # Now sync the top level individual files. + cmd = ("rsync --update " + cfg.win_extras_src + "/{*.dll,*.exe}" + " \"" + cfg.dst + "/\"") + _run(cmd) + + # We may want to add some site-packages on top; say where they should go. + return 'Lib' + + +def _sync_pylib(cfg: Config) -> str: + assert cfg.pylib_src_name is not None + assert cfg.dst is not None + _run(f'mkdir -p "{cfg.dst}/pylib"') + cmd = (f"rsync --recursive --update --delete --delete-excluded " + f" --prune-empty-dirs" + f" --include '*.py' --include '*.{OPT_PYC_SUFFIX}'" + f" --include '*/' --exclude '*'" + f" \"{cfg.src}/{cfg.pylib_src_name}/\" " + f"\"{cfg.dst}/pylib/\"") + _run(cmd) + + # We may want to add some site-packages on top; say where they should go. + return 'pylib' + + +def main() -> None: + """Stage assets for a build.""" + + cfg = Config() + cfg.parse_args() + + # Ok, now for every top level dir in src, come up with a nice single + # command to sync the needed subset of it to dst. + + # We can now use simple speedy timestamp based updates since + # we no longer have to try to preserve timestamps to get .pyc files + # to behave (hooray!) + + site_packages_target: Optional[str] = None + + # Do our stripped down pylib dir for platforms that use that. + if cfg.include_pylib: + site_packages_target = _sync_pylib(cfg) + + # On windows we need to pull in some dlls and this and that + # (we also include a non-stripped-down set of python libs). + if cfg.win_extras_src is not None: + site_packages_target = _sync_windows_extras(cfg) + + # Lastly, drop our site-packages into the python sys dir if desired. + # Ideally we should have a separate directory for these, but + # there's so few of them that this is simpler at the moment.. + if site_packages_target is not None: + cmd = (f"rsync --recursive " + f" --include '*.py' --include '*.{OPT_PYC_SUFFIX}'" + f" --include '*/' --exclude '*'" + f" \"{cfg.src}/pylib-site-packages/\" " + f"\"{cfg.dst}/{site_packages_target}/\"") + _run(cmd) + + # Now standard common game data. + assert cfg.dst is not None + _run('mkdir -p "' + cfg.dst + '/data"') + cmd = ("rsync --recursive --update --delete --delete-excluded" + " --prune-empty-dirs") + + if cfg.include_scripts: + cmd += " --include '*.py' --include '*." + OPT_PYC_SUFFIX + "'" + + if cfg.include_textures: + assert cfg.tex_suffix is not None + cmd += " --include '*" + cfg.tex_suffix + "'" + + if cfg.include_audio: + cmd += " --include '*.ogg'" + + if cfg.include_fonts: + cmd += " --include '*.fdata'" + + if cfg.include_json: + cmd += " --include '*.json'" + + if cfg.include_models: + cmd += " --include '*.bob'" + + if cfg.include_collide_models: + cmd += " --include '*.cob'" + + cmd += (" --include='*/' --exclude='*' \"" + cfg.src + "/data/\" \"" + + cfg.dst + "/data/\"") + _run(cmd) + + # On Android we need to build a payload file so it knows + # what to pull out of the apk. + if cfg.include_payload_file: + _write_payload_file(cfg.dst, cfg.is_payload_full) + + +if __name__ == '__main__': + main() diff --git a/tools/update_project b/tools/update_project index 30eab1e1..1da434a3 100755 --- a/tools/update_project +++ b/tools/update_project @@ -196,7 +196,8 @@ class App: print(f'{CLRRED} Found "{line}"{CLREND}') print(CLRRED + f'All {len(auto_changes)} errors are auto-fixable;' - ' pass --fix to apply corrections.' + CLREND) + ' run tools/update_project --fix to apply corrections.' + + CLREND) sys.exit(255) else: for i, change in enumerate(auto_changes):