From f5df30ba20e9a1def91123b665e676f7126732ce Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Wed, 16 Feb 2022 12:57:26 -0600 Subject: [PATCH] ssl work and lang updates --- .idea/dictionaries/ericf.xml | 12 ++++++++ CHANGELOG.md | 7 +++-- assets/.asset_manifest_private.json | 7 +++++ assets/Makefile | 14 ++++++++- assets/src/ba_data/python/ba/_app.py | 30 ++++++++++++++++--- assets/src/ba_data/python/ba/_asyncio.py | 15 +++++----- assets/src/ba_data/python/ba/_net.py | 3 ++ .../.idea/dictionaries/ericf.xml | 12 ++++++++ config/config.json | 3 +- src/ballistica/ballistica.cc | 2 +- .../python/methods/python_methods_system.cc | 15 ++++++++++ src/meta/bameta/python_embedded/bootstrap.py | 16 +++++++++- tools/batools/assetsmakefile.py | 1 + tools/batools/assetstaging.py | 3 +- tools/batools/build.py | 8 +++++ tools/efro/dataclassio/_prep.py | 2 +- tools/efro/error.py | 6 ++++ 17 files changed, 136 insertions(+), 20 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 83c39a0c..32dc7073 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -292,6 +292,7 @@ bylw bytecount byteswap + cacert cachable cachebasename cacheentry @@ -327,6 +328,7 @@ cend centeuro centiseconds + certifi cfconfig cfenv cfgdir @@ -426,6 +428,7 @@ columnwidget colw commitconfig + comms compat compileall compilelocations @@ -482,6 +485,7 @@ creditslist cresult cryptmodule + cryptosimple cspbd cspnf cspre @@ -731,6 +735,7 @@ excstr exec'ed execcode + execed execing execlocals executils @@ -898,6 +903,7 @@ futimens fval fverts + fwefocjwerj gameactivity gamebutton gameclass @@ -1145,6 +1151,7 @@ iometa ioprep ioprepped + ioprepping ipaddress ipos iprof @@ -1168,6 +1175,7 @@ jdict jenkinsfile jexport + jfwe jisx jite jittering @@ -1936,6 +1944,7 @@ recv redist redistributables + regionid regtp reimported relpath @@ -1998,6 +2007,7 @@ runonly runpy runpylint + runseconds runswindows rval safecolor @@ -2426,6 +2436,7 @@ tracemalloc tradeoff trailcolor + transportagentrequest transtime trapeznikov tref @@ -2477,6 +2488,7 @@ uiupkeeptimer unallowed uname + unbased unbounds uncollectible underruns diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fa6460b..45d2b5ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ -### 1.6.8 (20444) +### 1.6.8 (20449) - Added Filipino language (Thanks David!) - Restored pre-v1.5 jump behaviour. +- All communication with the master-server should now be secure (https) using root certificates from the [certifi](https://github.com/certifi/python-certifi) project. Please holler if you run into any connection issues with this version. ### 1.6.7 (20436) - Fixed a vulnerability which could expose device-account uuids. - Now generating Linux Arm64 server and test builds (currently built against Ubuntu 20). - Mac test builds are now Universal binaries (Arm64 & x86-64 versions bundled together). - Mac test builds are now notarized and distributed via a snazzy .dmg instead of a zip file, so the OS should no longer try to prevent you from running them. -- Now, when pushing new builds to https://files.ballistica.net/bombsquad/builds , corresponding checksums are written to a different server and can be accessed via https://tools.ballistica.net/checksums +- Test builds can now be found at - this page shows more info about the builds, including file checksums (stored on a separate server from the actual files for increased security). ### 1.6.6 (20394) - Beginning work on moving to new asset system. @@ -18,7 +19,7 @@ - Added co-op support to server builds (thanks Dliwk!) - Updated everything from Python 3.8 to Python 3.9. The biggest immediate impact to our code is that basic types such as list, dict, and tuple can be used in annotations, eliminating the need to import typing.Dict, typing.List, etc. See python.org for more changes. - Note: accessing mods on external storage on Android will not work in this release. This functionality has not been working in recent versions of Android due to increased security features anyway and I am in the process of replacing it with a cloud based system for installing mods. More on this soon. -- Python 3.9 no longer supports Windows 7 or earlier (according to https://www.python.org/downloads/windows/) so if you are running such a version of Windows you will need to stick to older builds. +- Python 3.9 no longer supports Windows 7 or earlier (according to ) so if you are running such a version of Windows you will need to stick to older builds. ### 1.6.4 (20382) - Some cleanups in the Favorites tab of the gather window. diff --git a/assets/.asset_manifest_private.json b/assets/.asset_manifest_private.json index dcf73da1..7b6c3d5d 100644 --- a/assets/.asset_manifest_private.json +++ b/assets/.asset_manifest_private.json @@ -946,6 +946,13 @@ "ba_data/python-site-packages/__pycache__/typing_extensions.cpython-39.opt-1.pyc", "ba_data/python-site-packages/_yaml/__init__.py", "ba_data/python-site-packages/_yaml/__pycache__/__init__.cpython-39.opt-1.pyc", + "ba_data/python-site-packages/certifi/__init__.py", + "ba_data/python-site-packages/certifi/__main__.py", + "ba_data/python-site-packages/certifi/__pycache__/__init__.cpython-39.opt-1.pyc", + "ba_data/python-site-packages/certifi/__pycache__/__main__.cpython-39.opt-1.pyc", + "ba_data/python-site-packages/certifi/__pycache__/core.cpython-39.opt-1.pyc", + "ba_data/python-site-packages/certifi/cacert.pem", + "ba_data/python-site-packages/certifi/core.py", "ba_data/python-site-packages/typing_extensions.py", "ba_data/python-site-packages/yaml/__init__.py", "ba_data/python-site-packages/yaml/__pycache__/__init__.cpython-39.opt-1.pyc", diff --git a/assets/Makefile b/assets/Makefile index 5b0786b7..ba284356 100644 --- a/assets/Makefile +++ b/assets/Makefile @@ -2475,6 +2475,9 @@ $(eval $(call make-opt-pyc-target,$(element)))) SCRIPT_TARGETS_PY_PRIVATE_COMMON = \ build/ba_data/python-site-packages/_yaml/__init__.py \ + build/ba_data/python-site-packages/certifi/__init__.py \ + build/ba_data/python-site-packages/certifi/__main__.py \ + build/ba_data/python-site-packages/certifi/core.py \ build/ba_data/python-site-packages/typing_extensions.py \ build/ba_data/python-site-packages/yaml/__init__.py \ build/ba_data/python-site-packages/yaml/composer.py \ @@ -2496,6 +2499,9 @@ SCRIPT_TARGETS_PY_PRIVATE_COMMON = \ SCRIPT_TARGETS_PYC_PRIVATE_COMMON = \ build/ba_data/python-site-packages/_yaml/__pycache__/__init__.cpython-39.opt-1.pyc \ + build/ba_data/python-site-packages/certifi/__pycache__/__init__.cpython-39.opt-1.pyc \ + build/ba_data/python-site-packages/certifi/__pycache__/__main__.cpython-39.opt-1.pyc \ + build/ba_data/python-site-packages/certifi/__pycache__/core.cpython-39.opt-1.pyc \ build/ba_data/python-site-packages/__pycache__/typing_extensions.cpython-39.opt-1.pyc \ build/ba_data/python-site-packages/yaml/__pycache__/__init__.cpython-39.opt-1.pyc \ build/ba_data/python-site-packages/yaml/__pycache__/composer.cpython-39.opt-1.pyc \ @@ -4785,6 +4791,9 @@ FONT_TARGETS = \ build/ba_data/fonts/fontSmall6.fdata \ build/ba_data/fonts/fontSmall7.fdata +PEM_TARGETS = \ + build/ba_data/python-site-packages/certifi/cacert.pem + DATA_TARGETS = \ build/ba_data/data/langdata.json \ build/ba_data/data/languages/arabic.json \ @@ -7075,6 +7084,9 @@ build/%.ogg : ../.efrocachemap build/%.fdata : ../.efrocachemap @cd .. && tools/pcommand efrocache_get assets/$@ +build/%.pem : ../.efrocachemap + @cd .. && tools/pcommand efrocache_get assets/$@ + # Langdata one-off json file. build/ba_data/data/langdata.json : ../.efrocachemap @cd .. && tools/pcommand efrocache_get assets/$@ @@ -7155,7 +7167,7 @@ SCRIPT_TARGETS_ANDROID = $(SCRIPT_TARGETS_PY_PRIVATE_ANDROID) \ SCRIPT_TARGETS_COMMON = $(SCRIPT_TARGETS_PY_PUBLIC) \ $(SCRIPT_TARGETS_PYC_PUBLIC) $(SCRIPT_TARGETS_PY_PUBLIC_TOOLS) \ $(SCRIPT_TARGETS_PYC_PUBLIC_TOOLS) $(SCRIPT_TARGETS_PY_PRIVATE_COMMON) \ - $(SCRIPT_TARGETS_PYC_PRIVATE_COMMON) + $(SCRIPT_TARGETS_PYC_PRIVATE_COMMON) $(PEM_TARGETS) # Build scripts for a specific platform. scripts-cmake: $(SCRIPT_TARGETS_CMAKE) $(SCRIPT_TARGETS_COMMON) diff --git a/assets/src/ba_data/python/ba/_app.py b/assets/src/ba_data/python/ba/_app.py index d9bd3383..8d2cdebc 100644 --- a/assets/src/ba_data/python/ba/_app.py +++ b/assets/src/ba_data/python/ba/_app.py @@ -3,8 +3,9 @@ """Functionality related to the high level state of the app.""" from __future__ import annotations -from enum import Enum import random +import logging +from enum import Enum from typing import TYPE_CHECKING import _ba @@ -184,6 +185,9 @@ class App: self.state = self.State.LAUNCHING + self._app_launched = False + self._app_paused = False + # Config. self.config_file_healthy = False @@ -366,22 +370,40 @@ class App: self.accounts.on_app_launch() self.plugins.on_app_launch() - self.state = self.State.RUNNING + # See note below in on_app_pause. + if self.state != self.State.LAUNCHING: + logging.error('on_app_launch found state %s; expected LAUNCHING.', + self.state) + + self._app_launched = True + self._update_state() # from ba._dependency import test_depset # test_depset() if bool(False): self._test_https() + def _update_state(self) -> None: + if self._app_paused: + self.state = self.State.PAUSED + else: + if self._app_launched: + self.state = self.State.RUNNING + else: + self.state = self.State.LAUNCHING + def on_app_pause(self) -> None: """Called when the app goes to a suspended state.""" - self.state = self.State.PAUSED + + self._app_paused = True + self._update_state() self.plugins.on_app_pause() def on_app_resume(self) -> None: """Run when the app resumes from a suspended state.""" - self.state = self.State.RUNNING + self._app_paused = False + self._update_state() self.fg_state += 1 self.accounts.on_app_resume() self.music.on_app_resume() diff --git a/assets/src/ba_data/python/ba/_asyncio.py b/assets/src/ba_data/python/ba/_asyncio.py index 76206e6f..79407a79 100644 --- a/assets/src/ba_data/python/ba/_asyncio.py +++ b/assets/src/ba_data/python/ba/_asyncio.py @@ -60,12 +60,13 @@ def setup_asyncio() -> None: timetype=TimeType.REAL, repeat=True) - async def aio_test() -> None: - print('TEST AIO TASK STARTING') - assert _asyncio_event_loop is not None - assert asyncio.get_running_loop() is _asyncio_event_loop - await asyncio.sleep(2.0) - print('TEST AIO TASK ENDING') - if bool(False): + + async def aio_test() -> None: + print('TEST AIO TASK STARTING') + assert _asyncio_event_loop is not None + assert asyncio.get_running_loop() is _asyncio_event_loop + await asyncio.sleep(2.0) + print('TEST AIO TASK ENDING') + _asyncio_event_loop.create_task(aio_test()) diff --git a/assets/src/ba_data/python/ba/_net.py b/assets/src/ba_data/python/ba/_net.py index ccbfc558..d7d2885d 100644 --- a/assets/src/ba_data/python/ba/_net.py +++ b/assets/src/ba_data/python/ba/_net.py @@ -25,6 +25,9 @@ class NetworkSubsystem: """Network related app subsystem.""" def __init__(self) -> None: + + # Anyone accessing/modifying region_pings should hold this lock. + self.region_pings_lock = threading.Lock() self.region_pings: dict[str, float] = {} diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml index 2d8c997b..10be43f6 100644 --- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml @@ -152,6 +152,7 @@ buttonup buttonwidget bwst + cacert calced calcing calcs @@ -172,6 +173,7 @@ ccylinder cend centiseconds + certifi cfgdir cfgpath changeme @@ -210,6 +212,7 @@ collidable collider columnwidget + comms connectattr containerwidget controlfp @@ -229,6 +232,7 @@ crom crosswire crvel + cryptosimple csize cspr cspre @@ -349,6 +353,7 @@ exargs exctype exec'ed + execed execinfo execing exhash @@ -423,6 +428,7 @@ funcname fval fvals + fwefocjwerj gamecenter gamedata gamepacket @@ -545,6 +551,7 @@ iometa ioprep ioprepped + ioprepping iserverget iserverput isinst @@ -566,6 +573,7 @@ jaxis jcjwf jdict + jfwe jmessage jnames json's @@ -928,6 +936,7 @@ redundants refcounted refl + regionid regtp rehel reimported @@ -962,6 +971,7 @@ rtypes rtypevar runnables + runseconds rvec rvel safecolor @@ -1145,6 +1155,7 @@ tradeoff trailcolor transobj + transportagentrequest treturn trifunovic trilinear @@ -1166,6 +1177,7 @@ udif uibounds uiid + unbased unblessed uncas unchecking diff --git a/config/config.json b/config/config.json index 0a17a714..87d3aa6b 100644 --- a/config/config.json +++ b/config/config.json @@ -33,7 +33,8 @@ "cpplint", "ansiwrap", "filelock", - "Cocoa" + "Cocoa", + "certifi" ], "python_paths": [ "assets/src/ba_data/python", diff --git a/src/ballistica/ballistica.cc b/src/ballistica/ballistica.cc index 92afd09f..420da088 100644 --- a/src/ballistica/ballistica.cc +++ b/src/ballistica/ballistica.cc @@ -21,7 +21,7 @@ namespace ballistica { // These are set automatically via script; don't modify them here. -const int kAppBuildNumber = 20446; +const int kAppBuildNumber = 20452; const char* kAppVersion = "1.6.8"; // Our standalone globals. diff --git a/src/ballistica/python/methods/python_methods_system.cc b/src/ballistica/python/methods/python_methods_system.cc index 0a64f147..41c61843 100644 --- a/src/ballistica/python/methods/python_methods_system.cc +++ b/src/ballistica/python/methods/python_methods_system.cc @@ -238,6 +238,16 @@ auto PyHasUserMods(PyObject* self, PyObject* args) -> PyObject* { BA_PYTHON_CATCH; } +auto PyContainsPythonDist(PyObject* self, PyObject* args) -> PyObject* { + BA_PYTHON_TRY; + Platform::SetLastPyCall("contains_python_dist"); + if (g_platform->ContainsPythonDist()) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; + BA_PYTHON_CATCH; +} + auto PyValueTest(PyObject* self, PyObject* args, PyObject* keywds) -> PyObject* { BA_PYTHON_TRY; @@ -1039,6 +1049,11 @@ auto PythonMethodsSystem::GetMethods() -> std::vector { "\n" "(internal)"}, + {"contains_python_dist", PyContainsPythonDist, METH_VARARGS, + "contains_python_dist() -> bool\n" + "\n" + "(internal)"}, + {"get_idle_time", PyGetIdleTime, METH_VARARGS, "get_idle_time() -> int\n" "\n" diff --git a/src/meta/bameta/python_embedded/bootstrap.py b/src/meta/bameta/python_embedded/bootstrap.py index 8688d2db..2e743aa1 100644 --- a/src/meta/bameta/python_embedded/bootstrap.py +++ b/src/meta/bameta/python_embedded/bootstrap.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import sys import signal import threading @@ -100,6 +101,20 @@ if debug_build != sys.flags.dev_mode: print(f'WARNING: Mismatch in debug_build {debug_build}' f' and sys.flags.dev_mode {sys.flags.dev_mode}') +# In embedded situations (when we're providing our own Python) let's +# also provide our own root certs so ssl works. We can consider overriding +# this in particular embedded cases if we can verify that system certs +# are working. +# (We also allow forcing this via an env var if the user desires) +# pylint: disable=wrong-import-position +if (_ba.contains_python_dist() + or os.environ.get('BA_USE_BUNDLED_ROOT_CERTS') == '1'): + import certifi + + # Let both OpenSSL and requests (if present) know to use this. + os.environ['SSL_CERT_FILE'] = os.environ['REQUESTS_CA_BUNDLE'] = ( + certifi.where()) + # FIXME: I think we should init Python in the main thread, which should # also avoid these issues. (and also might help us play better with # Python debuggers?) @@ -136,7 +151,6 @@ if debug_build: del testthread # Clear out the standard quit/exit messages since they don't work for us. -# pylint: disable=wrong-import-position # pylint: disable=c-extension-no-member if not TYPE_CHECKING: import __main__ diff --git a/tools/batools/assetsmakefile.py b/tools/batools/assetsmakefile.py index c65029c7..ae60e54e 100755 --- a/tools/batools/assetsmakefile.py +++ b/tools/batools/assetsmakefile.py @@ -331,6 +331,7 @@ def update_assets_makefile(projroot: str, check: bool) -> None: all_targets_private), _get_targets('FONT_TARGETS', '.fdata', '.fdata', all_targets_private), + _get_targets('PEM_TARGETS', '.pem', '.pem', all_targets_private), _get_targets('DATA_TARGETS', '.json', '.json', diff --git a/tools/batools/assetstaging.py b/tools/batools/assetstaging.py index 9c231bc7..46f75ebd 100755 --- a/tools/batools/assetstaging.py +++ b/tools/batools/assetstaging.py @@ -337,7 +337,8 @@ def _sync_standard_game_data(cfg: Config) -> None: ' --prune-empty-dirs') if cfg.include_scripts: - cmd += " --include '*.py' --include '*." + OPT_PYC_SUFFIX + "'" + cmd += (f" --include '*.py' --include '*.pem'" + f" --include '*.{OPT_PYC_SUFFIX}'") if cfg.include_textures: assert cfg.tex_suffix is not None diff --git a/tools/batools/build.py b/tools/batools/build.py index 8158c662..f197f77e 100644 --- a/tools/batools/build.py +++ b/tools/batools/build.py @@ -30,6 +30,12 @@ class PipRequirement: # Note: we look directly for modules when possible instead of just pip # entries; this accounts for manual installations or other nonstandard setups. + +# Note 2: We can probably just replace this with a simple requirements.txt +# file, can't we? Feels like we're mostly reinventing the wheel here. +# We just need a clean way to check/list missing stuff without necessarily +# installing it. And as far as manually-installed bits, pip itself must +# have some way to allow for that, right?... PIP_REQUIREMENTS = [ PipRequirement(modulename='pylint', minversion=[2, 12, 2]), PipRequirement(modulename='mypy', minversion=[0, 931]), @@ -45,6 +51,8 @@ PIP_REQUIREMENTS = [ PipRequirement(pipname='types-requests', minversion=[2, 27, 7]), PipRequirement(pipname='types-pytz', minversion=[2021, 3, 4]), PipRequirement(pipname='types-PyYAML', minversion=[6, 0, 3]), + PipRequirement(pipname='certifi', minversion=[2021, 10, 8]), + PipRequirement(pipname='types-certifi', minversion=[2021, 10, 8, 1]), ] # Parts of full-tests suite we only run on particular days. diff --git a/tools/efro/dataclassio/_prep.py b/tools/efro/dataclassio/_prep.py index aaf348df..57785b13 100644 --- a/tools/efro/dataclassio/_prep.py +++ b/tools/efro/dataclassio/_prep.py @@ -52,7 +52,7 @@ def ioprep(cls: type, globalns: dict = None) -> None: with localns set to the class dict (so that types defined in the class can be used) and globalns set to the containing module's class. It is possible to override globalns for special cases such as when - prepping happens as part of an exec'ed string instead of within a + prepping happens as part of an execed string instead of within a module. """ PrepSession(explicit=True, diff --git a/tools/efro/error.py b/tools/efro/error.py index 4627ffcc..3bc1a546 100644 --- a/tools/efro/error.py +++ b/tools/efro/error.py @@ -84,6 +84,12 @@ def is_urllib_network_error(exc: BaseException) -> bool: exc, (urllib.error.URLError, ConnectionError, http.client.IncompleteRead, http.client.BadStatusLine, socket.timeout)): + # Special case: although an HTTPError is a subclass of URLError, + # we don't return True for it. It means we have successfully + # communicated with the server but what we are asking for is + # not there/etc. + if isinstance(exc, urllib.error.HTTPError): + return False return True if isinstance(exc, OSError): if exc.errno == 10051: # Windows unreachable network error.