Added python-site-packages to ba_data

.idea/ballisticacore.iml

@@ -13,6 +13,7 @@
       <excludeFolder url="file://$MODULE_DIR$/assets/src/ba_data/data/maps" />
       <excludeFolder url="file://$MODULE_DIR$/assets/src/ba_data/fonts" />
       <excludeFolder url="file://$MODULE_DIR$/assets/src/ba_data/models" />
+      <excludeFolder url="file://$MODULE_DIR$/assets/src/ba_data/python-site-packages" />
       <excludeFolder url="file://$MODULE_DIR$/assets/src/ba_data/textures" />
       <excludeFolder url="file://$MODULE_DIR$/assets/src/pylib-android" />
       <excludeFolder url="file://$MODULE_DIR$/assets/src/pylib-apple" />

Makefile

@@ -113,11 +113,11 @@ clean-list:
 
 # Prebuilt binaries for various platforms.
 
-# Assemble/run a debug build for this platform.
+# Assemble & run a debug build for this platform.
 prefab-debug: prefab-debug-build
 	${${shell tools/snippets prefab_run_var debug}}
 
-# Assemble/run a release build for this platform.
+# Assemble & run a release build for this platform.
 prefab-release: prefab-release-build
 	${${shell tools/snippets prefab_run_var release}}
 
@@ -129,11 +129,11 @@ prefab-debug-build:
 prefab-release-build:
 	@tools/snippets make_prefab release
 
-# Assemble/run a server debug build for this platform.
+# Assemble & run a server debug build for this platform.
 prefab-server-debug: prefab-server-debug-build
 	${${shell tools/snippets prefab_run_var server-debug}}
 
-# Assemble/run a server release build for this platform.
+# Assemble & run a server release build for this platform.
 prefab-server-release: prefab-server-release-build
 	${${shell tools/snippets prefab_run_var server-release}}
 

assets/.asset_manifest_2.json

@@ -939,6 +939,8 @@
  "ba_data/models/zoeTorso.bob",
  "ba_data/models/zoeUpperArm.bob",
  "ba_data/models/zoeUpperLeg.bob",
+ "ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc",
+ "ba_data/python-site-packages/typing_extensions.py",
  "ba_data/textures/achievementBoxer.dds",
  "ba_data/textures/achievementBoxer.ktx",
  "ba_data/textures/achievementBoxer.pvr",
@@ -4243,8 +4245,6 @@
  "pylib-apple/xmlrpc/server.py",
  "pylib-apple/zipapp.py",
  "pylib-apple/zipfile.py",
- "pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc",
- "pylib-site-packages/typing_extensions.py",
  "windows/Win32/DLLs/_asyncio.pyd",
  "windows/Win32/DLLs/_bz2.pyd",
  "windows/Win32/DLLs/_ctypes.pyd",

assets/Makefile

@@ -7807,10 +7807,10 @@ build/pylib-android/email/mime/__pycache__/base.cpython-37.opt-1.pyc: \
 
 
 SCRIPT_TARGETS_PY_2_COMMON = \
-  build/pylib-site-packages/typing_extensions.py
+  build/ba_data/python-site-packages/typing_extensions.py
 
 SCRIPT_TARGETS_PYC_2_COMMON = \
-  build/pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc
+  build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc
 
 # Rule to copy src asset scripts to dst.
 # (and make non-writable so I'm less likely to accidentally edit them there)
@@ -7821,8 +7821,8 @@ $(SCRIPT_TARGETS_PY_2_COMMON) : ../.efrocachemap
 # just generating explicit targets for each. Could perhaps look into using a
 # fancy for-loop instead, but perhaps listing these explicitly isn't so bad.
 
-build/pylib-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc: \
+build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-37.opt-1.pyc: \
-      build/pylib-site-packages/typing_extensions.py
+      build/ba_data/python-site-packages/typing_extensions.py
 	@echo Compiling script: $^
 	@rm -rf $@ && $(TOOLS_DIR)/snippets compile_python_files $^ && chmod 444 $@
 

assets/src/ba_data/python/_ba.py

@@ -34,7 +34,7 @@ NOTE: This file was autogenerated by gendummymodule; do not edit by hand.
 """
 
 # (hash we can use to see if this file is out of date)
-# SOURCES_HASH=163268159341929947935319762789060730391
+# SOURCES_HASH=191214134064734166962017844135940898076
 
 # I'm sorry Pylint. I know this file saddens you. Be strong.
 # pylint: disable=useless-suppression

assets/src/ba_data/python/ba/_app.py

@@ -180,14 +180,19 @@ class App:
         return self._test_build
 
     @property
-    def user_scripts_directory(self) -> str:
+    def python_directory_user(self) -> str:
-        """Path where the game is looking for custom user scripts."""
+        """Path where the app looks for custom user scripts."""
-        return self._user_scripts_directory
+        return self._python_directory_user
 
     @property
-    def system_scripts_directory(self) -> str:
+    def python_directory_ba(self) -> str:
-        """Path where the game is looking for its bundled scripts."""
+        """Path where the app looks for its bundled scripts."""
-        return self._system_scripts_directory
+        return self._python_directory_ba
+
+    @property
+    def python_directory_site(self) -> str:
+        """Path containing pip packages bundled with the app."""
+        return self._python_directory_site
 
     @property
     def config(self) -> ba.AppConfig:
@@ -293,10 +298,12 @@ class App:
         assert isinstance(self._debug_build, bool)
         self._test_build: bool = env['test_build']
         assert isinstance(self._test_build, bool)
-        self._user_scripts_directory: str = env['user_scripts_directory']
+        self._python_directory_user: str = env['python_directory_user']
-        assert isinstance(self._user_scripts_directory, str)
+        assert isinstance(self._python_directory_user, str)
-        self._system_scripts_directory: str = env['system_scripts_directory']
+        self._python_directory_ba: str = env['python_directory_ba']
-        assert isinstance(self._system_scripts_directory, str)
+        assert isinstance(self._python_directory_ba, str)
+        self._python_directory_site: str = env['python_directory_site']
+        assert isinstance(self._python_directory_site, str)
         self._platform: str = env['platform']
         assert isinstance(self._platform, str)
         self._subplatform: str = env['subplatform']
@@ -541,7 +548,7 @@ class App:
         # Notify the user if we're using custom system scripts.
         # FIXME: This no longer works since sys-scripts is an absolute path;
         #  need to just add a proper call to query this.
-        # if env['system_scripts_directory'] != 'data/scripts':
+        # if env['python_directory_ba'] != 'data/scripts':
         #     ba.screenmessage("Using custom system scripts...",
         #                     color=(0, 1, 0))
 

assets/src/ba_data/python/ba/_meta.py

@@ -56,7 +56,7 @@ def start_scan() -> None:
     app = _ba.app
     if app.metascan is not None:
         print('WARNING: meta scan run more than once.')
-    scriptdirs = [app.system_scripts_directory, app.user_scripts_directory]
+    scriptdirs = [app.python_directory_ba, app.python_directory_user]
     thread = ScanThread(scriptdirs)
     thread.start()
 

assets/src/ba_data/python/ba/_modutils.py

@@ -37,7 +37,7 @@ def get_human_readable_user_scripts_path() -> str:
     """
     from ba import _lang
     app = _ba.app
-    path: Optional[str] = app.user_scripts_directory
+    path: Optional[str] = app.python_directory_user
     if path is None:
         return '<Not Available>'
 
@@ -48,10 +48,10 @@ def get_human_readable_user_scripts_path() -> str:
         ext_storage_path: Optional[str] = (
             _ba.android_get_external_storage_path())
         if (ext_storage_path is not None
-                and app.user_scripts_directory.startswith(ext_storage_path)):
+                and app.python_directory_user.startswith(ext_storage_path)):
             path = ('<' +
                     _lang.Lstr(resource='externalStorageText').evaluate() +
-                    '>' + app.user_scripts_directory[len(ext_storage_path):])
+                    '>' + app.python_directory_user[len(ext_storage_path):])
     return path
 
 
@@ -70,8 +70,8 @@ def show_user_scripts() -> None:
         return
 
     # Secondly, if the dir doesn't exist, attempt to make it.
-    if not os.path.exists(app.user_scripts_directory):
+    if not os.path.exists(app.python_directory_user):
-        os.makedirs(app.user_scripts_directory)
+        os.makedirs(app.python_directory_user)
 
     # On android, attempt to write a file in their user-scripts dir telling
     # them about modding. This also has the side-effect of allowing us to
@@ -81,7 +81,7 @@ def show_user_scripts() -> None:
     # they can see it.
     if app.platform == 'android':
         try:
-            usd: Optional[str] = app.user_scripts_directory
+            usd: Optional[str] = app.python_directory_user
             if usd is not None and os.path.isdir(usd):
                 file_name = usd + '/about_this_folder.txt'
                 with open(file_name, 'w') as outfile:
@@ -95,7 +95,7 @@ def show_user_scripts() -> None:
 
     # On a few platforms we try to open the dir in the UI.
     if app.platform in ['mac', 'windows']:
-        _ba.open_dir_externally(app.user_scripts_directory)
+        _ba.open_dir_externally(app.python_directory_user)
 
     # Otherwise we just print a pretty version of it.
     else:
@@ -109,7 +109,7 @@ def create_user_system_scripts() -> None:
     """
     app = _ba.app
     import shutil
-    path = (app.user_scripts_directory + '/sys/' + app.version)
+    path = (app.python_directory_user + '/sys/' + app.version)
     if os.path.exists(path):
         shutil.rmtree(path)
     if os.path.exists(path + "_tmp"):
@@ -118,9 +118,10 @@ def create_user_system_scripts() -> None:
 
     # Hmm; shutil.copytree doesn't seem to work nicely on android,
     # so lets do it manually.
-    src_dir = app.system_scripts_directory
+    # NOTE: Should retry this now that we have 3.7 (this note was for 2.7)
+    src_dir = app.python_directory_ba
     dst_dir = path + "_tmp"
-    filenames = os.listdir(app.system_scripts_directory)
+    filenames = os.listdir(app.python_directory_ba)
     for fname in filenames:
         print('COPYING', src_dir + '/' + fname, '->', dst_dir)
         shutil.copyfile(src_dir + '/' + fname, dst_dir + '/' + fname)
@@ -140,7 +141,7 @@ def delete_user_system_scripts() -> None:
     """Clean out the scripts created by create_user_system_scripts()."""
     import shutil
     app = _ba.app
-    path = (app.user_scripts_directory + '/sys/' + app.version)
+    path = (app.python_directory_user + '/sys/' + app.version)
     if os.path.exists(path):
         shutil.rmtree(path)
         print(
@@ -150,6 +151,6 @@ def delete_user_system_scripts() -> None:
         print('User system scripts not found.')
 
     # If the sys path is empty, kill it.
-    dpath = app.user_scripts_directory + '/sys'
+    dpath = app.python_directory_user + '/sys'
     if os.path.isdir(dpath) and not os.listdir(dpath):
         os.rmdir(dpath)

docs/ba_module.md

@@ -1,5 +1,5 @@
 <!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
-<h4><em>last updated on 2020-04-21 for Ballistica version 1.5.0 build 20001</em></h4>
+<h4><em>last updated on 2020-04-22 for Ballistica version 1.5.0 build 20001</em></h4>
 <p>This page documents the Python classes and functions in the 'ba' module,
  which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
 <hr>
@@ -709,7 +709,7 @@ likely result in errors.</p>
 </p>
 
 <h3>Attributes:</h3>
-<h5><a href="#attr_ba_App__api_version">api_version</a>, <a href="#attr_ba_App__build_number">build_number</a>, <a href="#attr_ba_App__config">config</a>, <a href="#attr_ba_App__config_file_path">config_file_path</a>, <a href="#attr_ba_App__debug_build">debug_build</a>, <a href="#attr_ba_App__interface_type">interface_type</a>, <a href="#attr_ba_App__language">language</a>, <a href="#attr_ba_App__locale">locale</a>, <a href="#attr_ba_App__on_tv">on_tv</a>, <a href="#attr_ba_App__platform">platform</a>, <a href="#attr_ba_App__subplatform">subplatform</a>, <a href="#attr_ba_App__system_scripts_directory">system_scripts_directory</a>, <a href="#attr_ba_App__test_build">test_build</a>, <a href="#attr_ba_App__ui_bounds">ui_bounds</a>, <a href="#attr_ba_App__user_agent_string">user_agent_string</a>, <a href="#attr_ba_App__user_scripts_directory">user_scripts_directory</a>, <a href="#attr_ba_App__version">version</a>, <a href="#attr_ba_App__vr_mode">vr_mode</a></h5>
+<h5><a href="#attr_ba_App__api_version">api_version</a>, <a href="#attr_ba_App__build_number">build_number</a>, <a href="#attr_ba_App__config">config</a>, <a href="#attr_ba_App__config_file_path">config_file_path</a>, <a href="#attr_ba_App__debug_build">debug_build</a>, <a href="#attr_ba_App__interface_type">interface_type</a>, <a href="#attr_ba_App__language">language</a>, <a href="#attr_ba_App__locale">locale</a>, <a href="#attr_ba_App__on_tv">on_tv</a>, <a href="#attr_ba_App__platform">platform</a>, <a href="#attr_ba_App__python_directory_ba">python_directory_ba</a>, <a href="#attr_ba_App__python_directory_site">python_directory_site</a>, <a href="#attr_ba_App__python_directory_user">python_directory_user</a>, <a href="#attr_ba_App__subplatform">subplatform</a>, <a href="#attr_ba_App__test_build">test_build</a>, <a href="#attr_ba_App__ui_bounds">ui_bounds</a>, <a href="#attr_ba_App__user_agent_string">user_agent_string</a>, <a href="#attr_ba_App__version">version</a>, <a href="#attr_ba_App__vr_mode">vr_mode</a></h5>
 <dl>
 <dt><h4><a name="attr_ba_App__api_version">api_version</a></h4></dt><dd>
 <p><span>int</span></p>
@@ -789,6 +789,21 @@ likely result in errors.</p>
 
 <p>        Examples are: 'mac', 'windows', android'.</p>
 
+</dd>
+<dt><h4><a name="attr_ba_App__python_directory_ba">python_directory_ba</a></h4></dt><dd>
+<p><span>str</span></p>
+<p>Path where the app looks for its bundled scripts.</p>
+
+</dd>
+<dt><h4><a name="attr_ba_App__python_directory_site">python_directory_site</a></h4></dt><dd>
+<p><span>str</span></p>
+<p>Path containing pip packages bundled with the app.</p>
+
+</dd>
+<dt><h4><a name="attr_ba_App__python_directory_user">python_directory_user</a></h4></dt><dd>
+<p><span>str</span></p>
+<p>Path where the app looks for custom user scripts.</p>
+
 </dd>
 <dt><h4><a name="attr_ba_App__subplatform">subplatform</a></h4></dt><dd>
 <p><span>str</span></p>
@@ -797,11 +812,6 @@ likely result in errors.</p>
 <p>        Can be empty. For the 'android' platform, subplatform may
         be 'google', 'amazon', etc.</p>
 
-</dd>
-<dt><h4><a name="attr_ba_App__system_scripts_directory">system_scripts_directory</a></h4></dt><dd>
-<p><span>str</span></p>
-<p>Path where the game is looking for its bundled scripts.</p>
-
 </dd>
 <dt><h4><a name="attr_ba_App__test_build">test_build</a></h4></dt><dd>
 <p><span>bool</span></p>
@@ -822,11 +832,6 @@ likely result in errors.</p>
 <p><span>str</span></p>
 <p>String containing various bits of info about OS/device/etc.</p>
 
-</dd>
-<dt><h4><a name="attr_ba_App__user_scripts_directory">user_scripts_directory</a></h4></dt><dd>
-<p><span>str</span></p>
-<p>Path where the game is looking for custom user scripts.</p>
-
 </dd>
 <dt><h4><a name="attr_ba_App__version">version</a></h4></dt><dd>
 <p><span>str</span></p>

tools/batools/build.py

@@ -437,7 +437,8 @@ def get_current_prefab_platform() -> str:
     system = platform.system()
     machine = platform.machine()
     if system == 'Darwin':
-        # Currently there's just x86_64 on mac; will need to revisit when arm
+        # Currently there's just x86_64 on mac;
+        # will need to revisit when arm
         # cpus happen.
         return 'mac'
     if system == 'Linux':
@@ -455,13 +456,6 @@ def get_current_prefab_platform() -> str:
                        f' {platform.system()}.')
 
 
-def make_prefab(target: PrefabTarget) -> None:
-    """Make a prefab build for the current platform."""
-    from efrotools import run
-    platform = get_current_prefab_platform()
-    run(f'make prefab-{platform}-{target.value}-build')
-
-
 def _vstr(nums: Sequence[int]) -> str:
     return '.'.join(str(i) for i in nums)
 

tools/efrotools/pybuild.py

@@ -425,19 +425,6 @@ def gather() -> None:
     for existing_dir in existing_dirs:
         efrotools.run('rm -rf "' + existing_dir + '"')
 
-    # Build our set of site-packages that we'll bundle in addition
-    # to the base system.
-    # FIXME: Should we perhaps make this part more explicit?..
-    #  we might get unexpected changes sneaking if we're just
-    #  pulling from installed python. But then again, anytime we're doing
-    #  a new python build/gather we should expect *some* changes even if
-    #  only at the build-system level since we pull some of that directly
-    #  from latest git stuff.
-    efrotools.run('mkdir -p "assets/src/pylib-site-packages"')
-    efrotools.run('cp "/usr/local/lib/python' + PYTHON_VERSION_MAJOR +
-                  '/site-packages/typing_extensions.py"'
-                  ' "assets/src/pylib-site-packages/"')
-
     for buildtype in ['debug', 'release']:
         debug = buildtype == 'debug'
         bsuffix = '_debug' if buildtype == 'debug' else ''

tools/snippets

@@ -194,15 +194,15 @@ def androidaddr() -> None:
     sdkutils = os.path.abspath(
         os.path.join(os.path.dirname(sys.argv[0]), 'android_sdk_utils'))
     rootdir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
-    ndkpath = subprocess.check_output([sdkutils, 'get-ndk-path'
+    ndkpath = subprocess.check_output([sdkutils,
-                                       ]).decode('utf-8').strip()
+                                       'get-ndk-path']).decode().strip()
     if not os.path.isdir(ndkpath):
         print("ERROR: ndk-path '" + ndkpath + '" does not exist')
         sys.exit(255)
     lines = subprocess.check_output(
         ['find',
          os.path.join(ndkpath, 'toolchains'), '-name',
-         '*addr2line']).decode('utf-8').strip().splitlines()
+         '*addr2line']).decode().strip().splitlines()
     lines = [line for line in lines if archs[arch]['prefix'] in line]
     if len(lines) != 1:
         print("ERROR: couldn't find addr2line binary")
@@ -558,12 +558,15 @@ def prefab_run_var() -> None:
 
 def make_prefab() -> None:
     """Run prefab builds for the current platform."""
+    import subprocess
     import batools.build
     if len(sys.argv) != 3:
         raise RuntimeError('Expected one argument')
     target = batools.build.PrefabTarget(sys.argv[2])
+    platform = batools.build.get_current_prefab_platform()
     try:
-        batools.build.make_prefab(target)
+        subprocess.run(['make', f'prefab-{platform}-{target.value}-build'],
+                       check=True)
     except (Exception, KeyboardInterrupt) as exc:
         if str(exc):
             print(f'make_prefab failed with error: {exc}')

tools/stage_assets

@@ -221,7 +221,7 @@ def _write_payload_file(assets_root: str, full: bool) -> None:
         pass
 
 
-def _sync_windows_extras(cfg: Config) -> str:
+def _sync_windows_extras(cfg: Config) -> None:
     assert cfg.win_extras_src is not None
     if not os.path.isdir(cfg.win_extras_src):
         raise Exception('win extras src dir not found: ' + cfg.win_extras_src)
@@ -258,11 +258,8 @@ def _sync_windows_extras(cfg: Config) -> str:
     # unless they're marked executable, so do that here.
     _run(f'chmod +x {cfg.dst}/*.exe')
 
-    # We may want to add some site-packages on top; say where they should go.
-    return 'Lib'
 
-
+def _sync_pylib(cfg: Config) -> None:
-def _sync_pylib(cfg: Config) -> str:
     assert cfg.pylib_src_name is not None
     assert cfg.dst is not None
     _run(f'mkdir -p "{cfg.dst}/pylib"')
@@ -274,9 +271,6 @@ def _sync_pylib(cfg: Config) -> str:
            f"\"{cfg.dst}/pylib/\"")
     _run(cmd)
 
-    # We may want to add some site-packages on top; say where they should go.
-    return 'pylib'
-
 
 def main() -> None:
     """Stage assets for a build."""
@@ -291,27 +285,14 @@ def main() -> None:
     # we no longer have to try to preserve timestamps to get .pyc files
     # to behave (hooray!)
 
-    site_packages_target: Optional[str] = None
-
     # Do our stripped down pylib dir for platforms that use that.
     if cfg.include_pylib:
-        site_packages_target = _sync_pylib(cfg)
+        _sync_pylib(cfg)
 
     # On windows we need to pull in some dlls and this and that
     # (we also include a non-stripped-down set of python libs).
     if cfg.win_extras_src is not None:
-        site_packages_target = _sync_windows_extras(cfg)
+        _sync_windows_extras(cfg)
-
-    # Lastly, drop our site-packages into the python sys dir if desired.
-    # Ideally we should have a separate directory for these, but
-    # there's so few of them that this is simpler at the moment..
-    if site_packages_target is not None:
-        cmd = (f"rsync --recursive "
-               f" --include '*.py' --include '*.{OPT_PYC_SUFFIX}'"
-               f" --include '*/' --exclude '*'"
-               f" \"{cfg.src}/pylib-site-packages/\" "
-               f"\"{cfg.dst}/{site_packages_target}/\"")
-        _run(cmd)
 
     # Now standard common game data.
     assert cfg.dst is not None