From 52643b8d0c9d94b9fff1f0a1df3d8885fbbe9ff0 Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Wed, 22 Apr 2020 16:20:06 -0700 Subject: [PATCH] Added python-site-packages to ba_data --- .idea/ballisticacore.iml | 1 + Makefile | 8 +++---- assets/.asset_manifest_2.json | 4 ++-- assets/Makefile | 8 +++---- assets/src/ba_data/python/_ba.py | 2 +- assets/src/ba_data/python/ba/_app.py | 29 ++++++++++++++--------- assets/src/ba_data/python/ba/_meta.py | 2 +- assets/src/ba_data/python/ba/_modutils.py | 25 +++++++++---------- docs/ba_module.md | 29 +++++++++++++---------- tools/batools/build.py | 10 ++------ tools/efrotools/pybuild.py | 13 ---------- tools/snippets | 11 +++++---- tools/stage_assets | 27 ++++----------------- 13 files changed, 74 insertions(+), 95 deletions(-) diff --git a/.idea/ballisticacore.iml b/.idea/ballisticacore.iml index 0d927f73..2dd49af3 100644 --- a/.idea/ballisticacore.iml +++ b/.idea/ballisticacore.iml @@ -13,6 +13,7 @@ + diff --git a/Makefile b/Makefile index 14d90af1..78d0effe 100644 --- a/Makefile +++ b/Makefile @@ -113,11 +113,11 @@ clean-list: # Prebuilt binaries for various platforms. -# Assemble/run a debug build for this platform. +# Assemble & run a debug build for this platform. prefab-debug: prefab-debug-build ${${shell tools/snippets prefab_run_var debug}} -# Assemble/run a release build for this platform. +# Assemble & run a release build for this platform. prefab-release: prefab-release-build ${${shell tools/snippets prefab_run_var release}} @@ -129,11 +129,11 @@ prefab-debug-build: prefab-release-build: @tools/snippets make_prefab release -# Assemble/run a server debug build for this platform. +# Assemble & run a server debug build for this platform. prefab-server-debug: prefab-server-debug-build ${${shell tools/snippets prefab_run_var server-debug}} -# Assemble/run a server release build for this platform. +# Assemble & run a server release build for this platform. prefab-server-release: prefab-server-release-build ${${shell tools/snippets prefab_run_var server-release}} diff --git a/assets/.asset_manifest_2.json b/assets/.asset_manifest_2.json index 41c18f31..4c9c85e2 100644 --- a/assets/.asset_manifest_2.json +++ b/assets/.asset_manifest_2.json @@ -939,6 +939,8 @@ "ba_data/models/zoeTorso.bob", "ba_data/models/zoeUpperArm.bob", "ba_data/models/zoeUpperLeg.bob", + "ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc", + "ba_data/python-site-packages/typing_extensions.py", "ba_data/textures/achievementBoxer.dds", "ba_data/textures/achievementBoxer.ktx", "ba_data/textures/achievementBoxer.pvr", @@ -4243,8 +4245,6 @@ "pylib-apple/xmlrpc/server.py", "pylib-apple/zipapp.py", "pylib-apple/zipfile.py", - "pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc", - "pylib-site-packages/typing_extensions.py", "windows/Win32/DLLs/_asyncio.pyd", "windows/Win32/DLLs/_bz2.pyd", "windows/Win32/DLLs/_ctypes.pyd", diff --git a/assets/Makefile b/assets/Makefile index 0a6876c5..60fb41b7 100644 --- a/assets/Makefile +++ b/assets/Makefile @@ -7807,10 +7807,10 @@ build/pylib-android/email/mime/__pycache__/base.cpython-37.opt-1.pyc: \ SCRIPT_TARGETS_PY_2_COMMON = \ - build/pylib-site-packages/typing_extensions.py + build/ba_data/python-site-packages/typing_extensions.py SCRIPT_TARGETS_PYC_2_COMMON = \ - build/pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc + build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc # Rule to copy src asset scripts to dst. # (and make non-writable so I'm less likely to accidentally edit them there) @@ -7821,8 +7821,8 @@ $(SCRIPT_TARGETS_PY_2_COMMON) : ../.efrocachemap # just generating explicit targets for each. Could perhaps look into using a # fancy for-loop instead, but perhaps listing these explicitly isn't so bad. -build/pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc: \ - build/pylib-site-packages/typing_extensions.py +build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc: \ + build/ba_data/python-site-packages/typing_extensions.py @echo Compiling script: $^ @rm -rf $@ && $(TOOLS_DIR)/snippets compile_python_files $^ && chmod 444 $@ diff --git a/assets/src/ba_data/python/_ba.py b/assets/src/ba_data/python/_ba.py index e6a69c3c..a311d968 100644 --- a/assets/src/ba_data/python/_ba.py +++ b/assets/src/ba_data/python/_ba.py @@ -34,7 +34,7 @@ NOTE: This file was autogenerated by gendummymodule; do not edit by hand. """ # (hash we can use to see if this file is out of date) -# SOURCES_HASH=163268159341929947935319762789060730391 +# SOURCES_HASH=191214134064734166962017844135940898076 # I'm sorry Pylint. I know this file saddens you. Be strong. # pylint: disable=useless-suppression diff --git a/assets/src/ba_data/python/ba/_app.py b/assets/src/ba_data/python/ba/_app.py index 6fedc456..b950c4ef 100644 --- a/assets/src/ba_data/python/ba/_app.py +++ b/assets/src/ba_data/python/ba/_app.py @@ -180,14 +180,19 @@ class App: return self._test_build @property - def user_scripts_directory(self) -> str: - """Path where the game is looking for custom user scripts.""" - return self._user_scripts_directory + def python_directory_user(self) -> str: + """Path where the app looks for custom user scripts.""" + return self._python_directory_user @property - def system_scripts_directory(self) -> str: - """Path where the game is looking for its bundled scripts.""" - return self._system_scripts_directory + def python_directory_ba(self) -> str: + """Path where the app looks for its bundled scripts.""" + return self._python_directory_ba + + @property + def python_directory_site(self) -> str: + """Path containing pip packages bundled with the app.""" + return self._python_directory_site @property def config(self) -> ba.AppConfig: @@ -293,10 +298,12 @@ class App: assert isinstance(self._debug_build, bool) self._test_build: bool = env['test_build'] assert isinstance(self._test_build, bool) - self._user_scripts_directory: str = env['user_scripts_directory'] - assert isinstance(self._user_scripts_directory, str) - self._system_scripts_directory: str = env['system_scripts_directory'] - assert isinstance(self._system_scripts_directory, str) + self._python_directory_user: str = env['python_directory_user'] + assert isinstance(self._python_directory_user, str) + self._python_directory_ba: str = env['python_directory_ba'] + assert isinstance(self._python_directory_ba, str) + self._python_directory_site: str = env['python_directory_site'] + assert isinstance(self._python_directory_site, str) self._platform: str = env['platform'] assert isinstance(self._platform, str) self._subplatform: str = env['subplatform'] @@ -541,7 +548,7 @@ class App: # Notify the user if we're using custom system scripts. # FIXME: This no longer works since sys-scripts is an absolute path; # need to just add a proper call to query this. - # if env['system_scripts_directory'] != 'data/scripts': + # if env['python_directory_ba'] != 'data/scripts': # ba.screenmessage("Using custom system scripts...", # color=(0, 1, 0)) diff --git a/assets/src/ba_data/python/ba/_meta.py b/assets/src/ba_data/python/ba/_meta.py index e4f281fe..ca5588e9 100644 --- a/assets/src/ba_data/python/ba/_meta.py +++ b/assets/src/ba_data/python/ba/_meta.py @@ -56,7 +56,7 @@ def start_scan() -> None: app = _ba.app if app.metascan is not None: print('WARNING: meta scan run more than once.') - scriptdirs = [app.system_scripts_directory, app.user_scripts_directory] + scriptdirs = [app.python_directory_ba, app.python_directory_user] thread = ScanThread(scriptdirs) thread.start() diff --git a/assets/src/ba_data/python/ba/_modutils.py b/assets/src/ba_data/python/ba/_modutils.py index e24fe638..9887e5cc 100644 --- a/assets/src/ba_data/python/ba/_modutils.py +++ b/assets/src/ba_data/python/ba/_modutils.py @@ -37,7 +37,7 @@ def get_human_readable_user_scripts_path() -> str: """ from ba import _lang app = _ba.app - path: Optional[str] = app.user_scripts_directory + path: Optional[str] = app.python_directory_user if path is None: return '' @@ -48,10 +48,10 @@ def get_human_readable_user_scripts_path() -> str: ext_storage_path: Optional[str] = ( _ba.android_get_external_storage_path()) if (ext_storage_path is not None - and app.user_scripts_directory.startswith(ext_storage_path)): + and app.python_directory_user.startswith(ext_storage_path)): path = ('<' + _lang.Lstr(resource='externalStorageText').evaluate() + - '>' + app.user_scripts_directory[len(ext_storage_path):]) + '>' + app.python_directory_user[len(ext_storage_path):]) return path @@ -70,8 +70,8 @@ def show_user_scripts() -> None: return # Secondly, if the dir doesn't exist, attempt to make it. - if not os.path.exists(app.user_scripts_directory): - os.makedirs(app.user_scripts_directory) + if not os.path.exists(app.python_directory_user): + os.makedirs(app.python_directory_user) # On android, attempt to write a file in their user-scripts dir telling # them about modding. This also has the side-effect of allowing us to @@ -81,7 +81,7 @@ def show_user_scripts() -> None: # they can see it. if app.platform == 'android': try: - usd: Optional[str] = app.user_scripts_directory + usd: Optional[str] = app.python_directory_user if usd is not None and os.path.isdir(usd): file_name = usd + '/about_this_folder.txt' with open(file_name, 'w') as outfile: @@ -95,7 +95,7 @@ def show_user_scripts() -> None: # On a few platforms we try to open the dir in the UI. if app.platform in ['mac', 'windows']: - _ba.open_dir_externally(app.user_scripts_directory) + _ba.open_dir_externally(app.python_directory_user) # Otherwise we just print a pretty version of it. else: @@ -109,7 +109,7 @@ def create_user_system_scripts() -> None: """ app = _ba.app import shutil - path = (app.user_scripts_directory + '/sys/' + app.version) + path = (app.python_directory_user + '/sys/' + app.version) if os.path.exists(path): shutil.rmtree(path) if os.path.exists(path + "_tmp"): @@ -118,9 +118,10 @@ def create_user_system_scripts() -> None: # Hmm; shutil.copytree doesn't seem to work nicely on android, # so lets do it manually. - src_dir = app.system_scripts_directory + # NOTE: Should retry this now that we have 3.7 (this note was for 2.7) + src_dir = app.python_directory_ba dst_dir = path + "_tmp" - filenames = os.listdir(app.system_scripts_directory) + filenames = os.listdir(app.python_directory_ba) for fname in filenames: print('COPYING', src_dir + '/' + fname, '->', dst_dir) shutil.copyfile(src_dir + '/' + fname, dst_dir + '/' + fname) @@ -140,7 +141,7 @@ def delete_user_system_scripts() -> None: """Clean out the scripts created by create_user_system_scripts().""" import shutil app = _ba.app - path = (app.user_scripts_directory + '/sys/' + app.version) + path = (app.python_directory_user + '/sys/' + app.version) if os.path.exists(path): shutil.rmtree(path) print( @@ -150,6 +151,6 @@ def delete_user_system_scripts() -> None: print('User system scripts not found.') # If the sys path is empty, kill it. - dpath = app.user_scripts_directory + '/sys' + dpath = app.python_directory_user + '/sys' if os.path.isdir(dpath) and not os.listdir(dpath): os.rmdir(dpath) diff --git a/docs/ba_module.md b/docs/ba_module.md index 70913ba2..a9b5f382 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,5 +1,5 @@ -

last updated on 2020-04-21 for Ballistica version 1.5.0 build 20001

+

last updated on 2020-04-22 for Ballistica version 1.5.0 build 20001

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!

@@ -709,7 +709,7 @@ likely result in errors.

Attributes:

-
api_version, build_number, config, config_file_path, debug_build, interface_type, language, locale, on_tv, platform, subplatform, system_scripts_directory, test_build, ui_bounds, user_agent_string, user_scripts_directory, version, vr_mode
+
api_version, build_number, config, config_file_path, debug_build, interface_type, language, locale, on_tv, platform, python_directory_ba, python_directory_site, python_directory_user, subplatform, test_build, ui_bounds, user_agent_string, version, vr_mode

api_version

int

@@ -789,6 +789,21 @@ likely result in errors.

Examples are: 'mac', 'windows', android'.

+
+

python_directory_ba

+

str

+

Path where the app looks for its bundled scripts.

+ +
+

python_directory_site

+

str

+

Path containing pip packages bundled with the app.

+ +
+

python_directory_user

+

str

+

Path where the app looks for custom user scripts.

+

subplatform

str

@@ -797,11 +812,6 @@ likely result in errors.

Can be empty. For the 'android' platform, subplatform may be 'google', 'amazon', etc.

-
-

system_scripts_directory

-

str

-

Path where the game is looking for its bundled scripts.

-

test_build

bool

@@ -822,11 +832,6 @@ likely result in errors.

str

String containing various bits of info about OS/device/etc.

-
-

user_scripts_directory

-

str

-

Path where the game is looking for custom user scripts.

-

version

str

diff --git a/tools/batools/build.py b/tools/batools/build.py index 2ea701e1..f359eb86 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -437,7 +437,8 @@ def get_current_prefab_platform() -> str: system = platform.system() machine = platform.machine() if system == 'Darwin': - # Currently there's just x86_64 on mac; will need to revisit when arm + # Currently there's just x86_64 on mac; + # will need to revisit when arm # cpus happen. return 'mac' if system == 'Linux': @@ -455,13 +456,6 @@ def get_current_prefab_platform() -> str: f' {platform.system()}.') -def make_prefab(target: PrefabTarget) -> None: - """Make a prefab build for the current platform.""" - from efrotools import run - platform = get_current_prefab_platform() - run(f'make prefab-{platform}-{target.value}-build') - - def _vstr(nums: Sequence[int]) -> str: return '.'.join(str(i) for i in nums) diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py index 31b3be0d..6e6596df 100644 --- a/tools/efrotools/pybuild.py +++ b/tools/efrotools/pybuild.py @@ -425,19 +425,6 @@ def gather() -> None: for existing_dir in existing_dirs: efrotools.run('rm -rf "' + existing_dir + '"') - # Build our set of site-packages that we'll bundle in addition - # to the base system. - # FIXME: Should we perhaps make this part more explicit?.. - # we might get unexpected changes sneaking if we're just - # pulling from installed python. But then again, anytime we're doing - # a new python build/gather we should expect *some* changes even if - # only at the build-system level since we pull some of that directly - # from latest git stuff. - efrotools.run('mkdir -p "assets/src/pylib-site-packages"') - efrotools.run('cp "/usr/local/lib/python' + PYTHON_VERSION_MAJOR + - '/site-packages/typing_extensions.py"' - ' "assets/src/pylib-site-packages/"') - for buildtype in ['debug', 'release']: debug = buildtype == 'debug' bsuffix = '_debug' if buildtype == 'debug' else '' diff --git a/tools/snippets b/tools/snippets index 8ce5f076..0135831b 100755 --- a/tools/snippets +++ b/tools/snippets @@ -194,15 +194,15 @@ def androidaddr() -> None: sdkutils = os.path.abspath( os.path.join(os.path.dirname(sys.argv[0]), 'android_sdk_utils')) rootdir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) - ndkpath = subprocess.check_output([sdkutils, 'get-ndk-path' - ]).decode('utf-8').strip() + ndkpath = subprocess.check_output([sdkutils, + 'get-ndk-path']).decode().strip() if not os.path.isdir(ndkpath): print("ERROR: ndk-path '" + ndkpath + '" does not exist') sys.exit(255) lines = subprocess.check_output( ['find', os.path.join(ndkpath, 'toolchains'), '-name', - '*addr2line']).decode('utf-8').strip().splitlines() + '*addr2line']).decode().strip().splitlines() lines = [line for line in lines if archs[arch]['prefix'] in line] if len(lines) != 1: print("ERROR: couldn't find addr2line binary") @@ -558,12 +558,15 @@ def prefab_run_var() -> None: def make_prefab() -> None: """Run prefab builds for the current platform.""" + import subprocess import batools.build if len(sys.argv) != 3: raise RuntimeError('Expected one argument') target = batools.build.PrefabTarget(sys.argv[2]) + platform = batools.build.get_current_prefab_platform() try: - batools.build.make_prefab(target) + subprocess.run(['make', f'prefab-{platform}-{target.value}-build'], + check=True) except (Exception, KeyboardInterrupt) as exc: if str(exc): print(f'make_prefab failed with error: {exc}') diff --git a/tools/stage_assets b/tools/stage_assets index 6bad425a..213b5b67 100755 --- a/tools/stage_assets +++ b/tools/stage_assets @@ -221,7 +221,7 @@ def _write_payload_file(assets_root: str, full: bool) -> None: pass -def _sync_windows_extras(cfg: Config) -> str: +def _sync_windows_extras(cfg: Config) -> None: assert cfg.win_extras_src is not None if not os.path.isdir(cfg.win_extras_src): raise Exception('win extras src dir not found: ' + cfg.win_extras_src) @@ -258,11 +258,8 @@ def _sync_windows_extras(cfg: Config) -> str: # unless they're marked executable, so do that here. _run(f'chmod +x {cfg.dst}/*.exe') - # We may want to add some site-packages on top; say where they should go. - return 'Lib' - -def _sync_pylib(cfg: Config) -> str: +def _sync_pylib(cfg: Config) -> None: assert cfg.pylib_src_name is not None assert cfg.dst is not None _run(f'mkdir -p "{cfg.dst}/pylib"') @@ -274,9 +271,6 @@ def _sync_pylib(cfg: Config) -> str: f"\"{cfg.dst}/pylib/\"") _run(cmd) - # We may want to add some site-packages on top; say where they should go. - return 'pylib' - def main() -> None: """Stage assets for a build.""" @@ -291,27 +285,14 @@ def main() -> None: # we no longer have to try to preserve timestamps to get .pyc files # to behave (hooray!) - site_packages_target: Optional[str] = None - # Do our stripped down pylib dir for platforms that use that. if cfg.include_pylib: - site_packages_target = _sync_pylib(cfg) + _sync_pylib(cfg) # On windows we need to pull in some dlls and this and that # (we also include a non-stripped-down set of python libs). if cfg.win_extras_src is not None: - site_packages_target = _sync_windows_extras(cfg) - - # Lastly, drop our site-packages into the python sys dir if desired. - # Ideally we should have a separate directory for these, but - # there's so few of them that this is simpler at the moment.. - if site_packages_target is not None: - cmd = (f"rsync --recursive " - f" --include '*.py' --include '*.{OPT_PYC_SUFFIX}'" - f" --include '*/' --exclude '*'" - f" \"{cfg.src}/pylib-site-packages/\" " - f"\"{cfg.dst}/{site_packages_target}/\"") - _run(cmd) + _sync_windows_extras(cfg) # Now standard common game data. assert cfg.dst is not None