ssl work and lang updates

.idea/dictionaries/ericf.xml

@@ -292,6 +292,7 @@
       <w>bylw</w>
       <w>bytecount</w>
       <w>byteswap</w>
+      <w>cacert</w>
       <w>cachable</w>
       <w>cachebasename</w>
       <w>cacheentry</w>
@@ -327,6 +328,7 @@
       <w>cend</w>
       <w>centeuro</w>
       <w>centiseconds</w>
+      <w>certifi</w>
       <w>cfconfig</w>
       <w>cfenv</w>
       <w>cfgdir</w>
@@ -426,6 +428,7 @@
       <w>columnwidget</w>
       <w>colw</w>
       <w>commitconfig</w>
+      <w>comms</w>
       <w>compat</w>
       <w>compileall</w>
       <w>compilelocations</w>
@@ -482,6 +485,7 @@
       <w>creditslist</w>
       <w>cresult</w>
       <w>cryptmodule</w>
+      <w>cryptosimple</w>
       <w>cspbd</w>
       <w>cspnf</w>
       <w>cspre</w>
@@ -731,6 +735,7 @@
       <w>excstr</w>
       <w>exec'ed</w>
       <w>execcode</w>
+      <w>execed</w>
       <w>execing</w>
       <w>execlocals</w>
       <w>executils</w>
@@ -898,6 +903,7 @@
       <w>futimens</w>
       <w>fval</w>
       <w>fverts</w>
+      <w>fwefocjwerj</w>
       <w>gameactivity</w>
       <w>gamebutton</w>
       <w>gameclass</w>
@@ -1145,6 +1151,7 @@
       <w>iometa</w>
       <w>ioprep</w>
       <w>ioprepped</w>
+      <w>ioprepping</w>
       <w>ipaddress</w>
       <w>ipos</w>
       <w>iprof</w>
@@ -1168,6 +1175,7 @@
       <w>jdict</w>
       <w>jenkinsfile</w>
       <w>jexport</w>
+      <w>jfwe</w>
       <w>jisx</w>
       <w>jite</w>
       <w>jittering</w>
@@ -1936,6 +1944,7 @@
       <w>recv</w>
       <w>redist</w>
       <w>redistributables</w>
+      <w>regionid</w>
       <w>regtp</w>
       <w>reimported</w>
       <w>relpath</w>
@@ -1998,6 +2007,7 @@
       <w>runonly</w>
       <w>runpy</w>
       <w>runpylint</w>
+      <w>runseconds</w>
       <w>runswindows</w>
       <w>rval</w>
       <w>safecolor</w>
@@ -2426,6 +2436,7 @@
       <w>tracemalloc</w>
       <w>tradeoff</w>
       <w>trailcolor</w>
+      <w>transportagentrequest</w>
       <w>transtime</w>
       <w>trapeznikov</w>
       <w>tref</w>
@@ -2477,6 +2488,7 @@
       <w>uiupkeeptimer</w>
       <w>unallowed</w>
       <w>uname</w>
+      <w>unbased</w>
       <w>unbounds</w>
       <w>uncollectible</w>
       <w>underruns</w>

CHANGELOG.md

@@ -1,13 +1,14 @@
-### 1.6.8 (20444)
+### 1.6.8 (20449)
 - Added Filipino language (Thanks David!)
 - Restored pre-v1.5 jump behaviour.
+- All communication with the master-server should now be secure (https) using root certificates from the [certifi](https://github.com/certifi/python-certifi) project. Please holler if you run into any connection issues with this version.
 
 ### 1.6.7 (20436)
 - Fixed a vulnerability which could expose device-account uuids.
 - Now generating Linux Arm64 server and test builds (currently built against Ubuntu 20).
 - Mac test builds are now Universal binaries (Arm64 & x86-64 versions bundled together).
 - Mac test builds are now notarized and distributed via a snazzy .dmg instead of a zip file, so the OS should no longer try to prevent you from running them.
-- Now, when pushing new builds to https://files.ballistica.net/bombsquad/builds , corresponding checksums are written to a different server and can be accessed via https://tools.ballistica.net/checksums
+- Test builds can now be found at <https://ballistica.net/builds> - this page shows more info about the builds, including file checksums (stored on a separate server from the actual files for increased security).
 
 ### 1.6.6 (20394)
 - Beginning work on moving to new asset system.
@@ -18,7 +19,7 @@
 - Added co-op support to server builds (thanks Dliwk!)
 - Updated everything from Python 3.8 to Python 3.9. The biggest immediate impact to our code is that basic types such as list, dict, and tuple can be used in annotations, eliminating the need to import typing.Dict, typing.List, etc. See python.org for more changes.
 - Note: accessing mods on external storage on Android will not work in this release. This functionality has not been working in recent versions of Android due to increased security features anyway and I am in the process of replacing it with a cloud based system for installing mods. More on this soon.
-- Python 3.9 no longer supports Windows 7 or earlier (according to https://www.python.org/downloads/windows/) so if you are running such a version of Windows you will need to stick to older builds.
+- Python 3.9 no longer supports Windows 7 or earlier (according to <https://www.python.org/downloads/windows/>) so if you are running such a version of Windows you will need to stick to older builds.
 
 ### 1.6.4 (20382)
 - Some cleanups in the Favorites tab of the gather window.

assets/.asset_manifest_private.json

@@ -946,6 +946,13 @@
  "ba_data/python-site-packages/__pycache__/typing_extensions.cpython-39.opt-1.pyc",
  "ba_data/python-site-packages/_yaml/__init__.py",
  "ba_data/python-site-packages/_yaml/__pycache__/__init__.cpython-39.opt-1.pyc",
+ "ba_data/python-site-packages/certifi/__init__.py",
+ "ba_data/python-site-packages/certifi/__main__.py",
+ "ba_data/python-site-packages/certifi/__pycache__/__init__.cpython-39.opt-1.pyc",
+ "ba_data/python-site-packages/certifi/__pycache__/__main__.cpython-39.opt-1.pyc",
+ "ba_data/python-site-packages/certifi/__pycache__/core.cpython-39.opt-1.pyc",
+ "ba_data/python-site-packages/certifi/cacert.pem",
+ "ba_data/python-site-packages/certifi/core.py",
  "ba_data/python-site-packages/typing_extensions.py",
  "ba_data/python-site-packages/yaml/__init__.py",
  "ba_data/python-site-packages/yaml/__pycache__/__init__.cpython-39.opt-1.pyc",

assets/Makefile

@@ -2475,6 +2475,9 @@ $(eval $(call make-opt-pyc-target,$(element))))
 
 SCRIPT_TARGETS_PY_PRIVATE_COMMON = \
   build/ba_data/python-site-packages/_yaml/__init__.py \
+  build/ba_data/python-site-packages/certifi/__init__.py \
+  build/ba_data/python-site-packages/certifi/__main__.py \
+  build/ba_data/python-site-packages/certifi/core.py \
   build/ba_data/python-site-packages/typing_extensions.py \
   build/ba_data/python-site-packages/yaml/__init__.py \
   build/ba_data/python-site-packages/yaml/composer.py \
@@ -2496,6 +2499,9 @@ SCRIPT_TARGETS_PY_PRIVATE_COMMON = \
 
 SCRIPT_TARGETS_PYC_PRIVATE_COMMON = \
   build/ba_data/python-site-packages/_yaml/__pycache__/__init__.cpython-39.opt-1.pyc \
+  build/ba_data/python-site-packages/certifi/__pycache__/__init__.cpython-39.opt-1.pyc \
+  build/ba_data/python-site-packages/certifi/__pycache__/__main__.cpython-39.opt-1.pyc \
+  build/ba_data/python-site-packages/certifi/__pycache__/core.cpython-39.opt-1.pyc \
   build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-39.opt-1.pyc \
   build/ba_data/python-site-packages/yaml/__pycache__/__init__.cpython-39.opt-1.pyc \
   build/ba_data/python-site-packages/yaml/__pycache__/composer.cpython-39.opt-1.pyc \
@@ -4785,6 +4791,9 @@ FONT_TARGETS = \
   build/ba_data/fonts/fontSmall6.fdata \
   build/ba_data/fonts/fontSmall7.fdata
 
+PEM_TARGETS = \
+  build/ba_data/python-site-packages/certifi/cacert.pem
+
 DATA_TARGETS = \
   build/ba_data/data/langdata.json \
   build/ba_data/data/languages/arabic.json \
@@ -7075,6 +7084,9 @@ build/%.ogg : ../.efrocachemap
 build/%.fdata : ../.efrocachemap
 	@cd .. && tools/pcommand efrocache_get assets/$@
 
+build/%.pem : ../.efrocachemap
+	@cd .. && tools/pcommand efrocache_get assets/$@
+
 # Langdata one-off json file.
 build/ba_data/data/langdata.json : ../.efrocachemap
 	@cd .. && tools/pcommand efrocache_get assets/$@
@@ -7155,7 +7167,7 @@ SCRIPT_TARGETS_ANDROID = $(SCRIPT_TARGETS_PY_PRIVATE_ANDROID) \
 SCRIPT_TARGETS_COMMON = $(SCRIPT_TARGETS_PY_PUBLIC) \
  $(SCRIPT_TARGETS_PYC_PUBLIC) $(SCRIPT_TARGETS_PY_PUBLIC_TOOLS) \
  $(SCRIPT_TARGETS_PYC_PUBLIC_TOOLS) $(SCRIPT_TARGETS_PY_PRIVATE_COMMON) \
- $(SCRIPT_TARGETS_PYC_PRIVATE_COMMON)
+ $(SCRIPT_TARGETS_PYC_PRIVATE_COMMON) $(PEM_TARGETS)
 
 # Build scripts for a specific platform.
 scripts-cmake: $(SCRIPT_TARGETS_CMAKE) $(SCRIPT_TARGETS_COMMON)

assets/src/ba_data/python/ba/_app.py

@@ -3,8 +3,9 @@
 """Functionality related to the high level state of the app."""
 from __future__ import annotations
 
-from enum import Enum
 import random
+import logging
+from enum import Enum
 from typing import TYPE_CHECKING
 
 import _ba
@@ -184,6 +185,9 @@ class App:
 
         self.state = self.State.LAUNCHING
 
+        self._app_launched = False
+        self._app_paused = False
+
         # Config.
         self.config_file_healthy = False
 
@@ -366,22 +370,40 @@ class App:
         self.accounts.on_app_launch()
         self.plugins.on_app_launch()
 
-        self.state = self.State.RUNNING
+        # See note below in on_app_pause.
+        if self.state != self.State.LAUNCHING:
+            logging.error('on_app_launch found state %s; expected LAUNCHING.',
+                          self.state)
+
+        self._app_launched = True
+        self._update_state()
 
         # from ba._dependency import test_depset
         # test_depset()
         if bool(False):
             self._test_https()
 
+    def _update_state(self) -> None:
+        if self._app_paused:
+            self.state = self.State.PAUSED
+        else:
+            if self._app_launched:
+                self.state = self.State.RUNNING
+            else:
+                self.state = self.State.LAUNCHING
+
     def on_app_pause(self) -> None:
         """Called when the app goes to a suspended state."""
-        self.state = self.State.PAUSED
+
+        self._app_paused = True
+        self._update_state()
         self.plugins.on_app_pause()
 
     def on_app_resume(self) -> None:
         """Run when the app resumes from a suspended state."""
 
-        self.state = self.State.RUNNING
+        self._app_paused = False
+        self._update_state()
         self.fg_state += 1
         self.accounts.on_app_resume()
         self.music.on_app_resume()

assets/src/ba_data/python/ba/_asyncio.py

@@ -60,12 +60,13 @@ def setup_asyncio() -> None:
                                timetype=TimeType.REAL,
                                repeat=True)
 
-    async def aio_test() -> None:
-        print('TEST AIO TASK STARTING')
-        assert _asyncio_event_loop is not None
-        assert asyncio.get_running_loop() is _asyncio_event_loop
-        await asyncio.sleep(2.0)
-        print('TEST AIO TASK ENDING')
-
     if bool(False):
+
+        async def aio_test() -> None:
+            print('TEST AIO TASK STARTING')
+            assert _asyncio_event_loop is not None
+            assert asyncio.get_running_loop() is _asyncio_event_loop
+            await asyncio.sleep(2.0)
+            print('TEST AIO TASK ENDING')
+
         _asyncio_event_loop.create_task(aio_test())

assets/src/ba_data/python/ba/_net.py

@@ -25,6 +25,9 @@ class NetworkSubsystem:
     """Network related app subsystem."""
 
     def __init__(self) -> None:
+
+        # Anyone accessing/modifying region_pings should hold this lock.
+        self.region_pings_lock = threading.Lock()
         self.region_pings: dict[str, float] = {}
 
 

ballisticacore-cmake/.idea/dictionaries/ericf.xml

@@ -152,6 +152,7 @@
       <w>buttonup</w>
       <w>buttonwidget</w>
       <w>bwst</w>
+      <w>cacert</w>
       <w>calced</w>
       <w>calcing</w>
       <w>calcs</w>
@@ -172,6 +173,7 @@
       <w>ccylinder</w>
       <w>cend</w>
       <w>centiseconds</w>
+      <w>certifi</w>
       <w>cfgdir</w>
       <w>cfgpath</w>
       <w>changeme</w>
@@ -210,6 +212,7 @@
       <w>collidable</w>
       <w>collider</w>
       <w>columnwidget</w>
+      <w>comms</w>
       <w>connectattr</w>
       <w>containerwidget</w>
       <w>controlfp</w>
@@ -229,6 +232,7 @@
       <w>crom</w>
       <w>crosswire</w>
       <w>crvel</w>
+      <w>cryptosimple</w>
       <w>csize</w>
       <w>cspr</w>
       <w>cspre</w>
@@ -349,6 +353,7 @@
       <w>exargs</w>
       <w>exctype</w>
       <w>exec'ed</w>
+      <w>execed</w>
       <w>execinfo</w>
       <w>execing</w>
       <w>exhash</w>
@@ -423,6 +428,7 @@
       <w>funcname</w>
       <w>fval</w>
       <w>fvals</w>
+      <w>fwefocjwerj</w>
       <w>gamecenter</w>
       <w>gamedata</w>
       <w>gamepacket</w>
@@ -545,6 +551,7 @@
       <w>iometa</w>
       <w>ioprep</w>
       <w>ioprepped</w>
+      <w>ioprepping</w>
       <w>iserverget</w>
       <w>iserverput</w>
       <w>isinst</w>
@@ -566,6 +573,7 @@
       <w>jaxis</w>
       <w>jcjwf</w>
       <w>jdict</w>
+      <w>jfwe</w>
       <w>jmessage</w>
       <w>jnames</w>
       <w>json's</w>
@@ -928,6 +936,7 @@
       <w>redundants</w>
       <w>refcounted</w>
       <w>refl</w>
+      <w>regionid</w>
       <w>regtp</w>
       <w>rehel</w>
       <w>reimported</w>
@@ -962,6 +971,7 @@
       <w>rtypes</w>
       <w>rtypevar</w>
       <w>runnables</w>
+      <w>runseconds</w>
       <w>rvec</w>
       <w>rvel</w>
       <w>safecolor</w>
@@ -1145,6 +1155,7 @@
       <w>tradeoff</w>
       <w>trailcolor</w>
       <w>transobj</w>
+      <w>transportagentrequest</w>
       <w>treturn</w>
       <w>trifunovic</w>
       <w>trilinear</w>
@@ -1166,6 +1177,7 @@
       <w>udif</w>
       <w>uibounds</w>
       <w>uiid</w>
+      <w>unbased</w>
       <w>unblessed</w>
       <w>uncas</w>
       <w>unchecking</w>

config/config.json

@@ -33,7 +33,8 @@
     "cpplint",
     "ansiwrap",
     "filelock",
-    "Cocoa"
+    "Cocoa",
+    "certifi"
   ],
   "python_paths": [
     "assets/src/ba_data/python",

src/ballistica/ballistica.cc

@@ -21,7 +21,7 @@
 namespace ballistica {
 
 // These are set automatically via script; don't modify them here.
-const int kAppBuildNumber = 20446;
+const int kAppBuildNumber = 20452;
 const char* kAppVersion = "1.6.8";
 
 // Our standalone globals.

src/ballistica/python/methods/python_methods_system.cc

@@ -238,6 +238,16 @@ auto PyHasUserMods(PyObject* self, PyObject* args) -> PyObject* {
   BA_PYTHON_CATCH;
 }
 
+auto PyContainsPythonDist(PyObject* self, PyObject* args) -> PyObject* {
+  BA_PYTHON_TRY;
+  Platform::SetLastPyCall("contains_python_dist");
+  if (g_platform->ContainsPythonDist()) {
+    Py_RETURN_TRUE;
+  }
+  Py_RETURN_FALSE;
+  BA_PYTHON_CATCH;
+}
+
 auto PyValueTest(PyObject* self, PyObject* args, PyObject* keywds)
     -> PyObject* {
   BA_PYTHON_TRY;
@@ -1039,6 +1049,11 @@ auto PythonMethodsSystem::GetMethods() -> std::vector<PyMethodDef> {
        "\n"
        "(internal)"},
 
+      {"contains_python_dist", PyContainsPythonDist, METH_VARARGS,
+       "contains_python_dist() -> bool\n"
+       "\n"
+       "(internal)"},
+
       {"get_idle_time", PyGetIdleTime, METH_VARARGS,
        "get_idle_time() -> int\n"
        "\n"

src/meta/bameta/python_embedded/bootstrap.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import os
 import sys
 import signal
 import threading
@@ -100,6 +101,20 @@ if debug_build != sys.flags.dev_mode:
     print(f'WARNING: Mismatch in debug_build {debug_build}'
           f' and sys.flags.dev_mode {sys.flags.dev_mode}')
 
+# In embedded situations (when we're providing our own Python) let's
+# also provide our own root certs so ssl works. We can consider overriding
+# this in particular embedded cases if we can verify that system certs
+# are working.
+# (We also allow forcing this via an env var if the user desires)
+# pylint: disable=wrong-import-position
+if (_ba.contains_python_dist()
+        or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'):
+    import certifi
+
+    # Let both OpenSSL and requests (if present) know to use this.
+    os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = (
+        certifi.where())
+
 # FIXME: I think we should init Python in the main thread, which should
 #  also avoid these issues. (and also might help us play better with
 #  Python debuggers?)
@@ -136,7 +151,6 @@ if debug_build:
     del testthread
 
 # Clear out the standard quit/exit messages since they don't work for us.
-# pylint: disable=wrong-import-position
 # pylint: disable=c-extension-no-member
 if not TYPE_CHECKING:
     import __main__

tools/batools/assetsmakefile.py

@@ -331,6 +331,7 @@ def update_assets_makefile(projroot: str, check: bool) -> None:
                          all_targets_private),
             _get_targets('FONT_TARGETS', '.fdata', '.fdata',
                          all_targets_private),
+            _get_targets('PEM_TARGETS', '.pem', '.pem', all_targets_private),
             _get_targets('DATA_TARGETS',
                          '.json',
                          '.json',

tools/batools/assetstaging.py

@@ -337,7 +337,8 @@ def _sync_standard_game_data(cfg: Config) -> None:
            ' --prune-empty-dirs')
 
     if cfg.include_scripts:
-        cmd += " --include '*.py' --include '*." + OPT_PYC_SUFFIX + "'"
+        cmd += (f" --include '*.py' --include '*.pem'"
+                f" --include '*.{OPT_PYC_SUFFIX}'")
 
     if cfg.include_textures:
         assert cfg.tex_suffix is not None

tools/batools/build.py

@@ -30,6 +30,12 @@ class PipRequirement:
 
 # Note: we look directly for modules when possible instead of just pip
 # entries; this accounts for manual installations or other nonstandard setups.
+
+# Note 2: We can probably just replace this with a simple requirements.txt
+# file, can't we? Feels like we're mostly reinventing the wheel here.
+# We just need a clean way to check/list missing stuff without necessarily
+# installing it. And as far as manually-installed bits, pip itself must
+# have some way to allow for that, right?...
 PIP_REQUIREMENTS = [
     PipRequirement(modulename='pylint', minversion=[2, 12, 2]),
     PipRequirement(modulename='mypy', minversion=[0, 931]),
@@ -45,6 +51,8 @@ PIP_REQUIREMENTS = [
     PipRequirement(pipname='types-requests', minversion=[2, 27, 7]),
     PipRequirement(pipname='types-pytz', minversion=[2021, 3, 4]),
     PipRequirement(pipname='types-PyYAML', minversion=[6, 0, 3]),
+    PipRequirement(pipname='certifi', minversion=[2021, 10, 8]),
+    PipRequirement(pipname='types-certifi', minversion=[2021, 10, 8, 1]),
 ]
 
 # Parts of full-tests suite we only run on particular days.

tools/efro/dataclassio/_prep.py

@@ -52,7 +52,7 @@ def ioprep(cls: type, globalns: dict = None) -> None:
     with localns set to the class dict (so that types defined in the class
     can be used) and globalns set to the containing module's class.
     It is possible to override globalns for special cases such as when
-    prepping happens as part of an exec'ed string instead of within a
+    prepping happens as part of an execed string instead of within a
     module.
     """
     PrepSession(explicit=True,

tools/efro/error.py

@@ -84,6 +84,12 @@ def is_urllib_network_error(exc: BaseException) -> bool:
             exc,
         (urllib.error.URLError, ConnectionError, http.client.IncompleteRead,
          http.client.BadStatusLine, socket.timeout)):
+        # Special case: although an HTTPError is a subclass of URLError,
+        # we don't return True for it. It means we have successfully
+        # communicated with the server but what we are asking for is
+        # not there/etc.
+        if isinstance(exc, urllib.error.HTTPError):
+            return False
         return True
     if isinstance(exc, OSError):
         if exc.errno == 10051:  # Windows unreachable network error.