From a6b940160cdae64c3e4b9af3ea2800184e69d78f Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Wed, 11 Mar 2020 14:18:00 -0700 Subject: [PATCH] Syncing latest changes between public/private. --- .idea/dictionaries/ericf.xml | 5 + assets/src/ba_data/python/ba/_assetmanager.py | 179 +++++++++++++++++- .../src/ba_data/python/efro/entity/_entity.py | 5 +- assets/src/ba_data/python/efro/jsonutils.py | 9 +- assets/src/server/server.py | 1 + docs/ba_module.md | 4 +- tests/test_ba/test_assetmanager.py | 22 ++- tools/snippets | 2 +- 8 files changed, 212 insertions(+), 15 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 130b2d3d..801176da 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -45,6 +45,7 @@ allpaths allsettings allteams + aman aname anamorphosis andr @@ -105,6 +106,7 @@ asyncio asynciomodule asyncore + atexit attrdict attrdocs attrinfo @@ -463,6 +465,7 @@ duckdns dummymodule dummyname + dummytoken dummyval dups dxml @@ -1662,6 +1665,7 @@ testcapimodule testclass testd + testdl testfoo testfooooo testfull @@ -1797,6 +1801,7 @@ uploadargs uploadcmd uptime + ureq useragent useragentstring userbase diff --git a/assets/src/ba_data/python/ba/_assetmanager.py b/assets/src/ba_data/python/ba/_assetmanager.py index c933747f..49fc0ea9 100644 --- a/assets/src/ba_data/python/ba/_assetmanager.py +++ b/assets/src/ba_data/python/ba/_assetmanager.py @@ -20,12 +20,189 @@ # ----------------------------------------------------------------------------- """Functionality related to managing cloud based assets.""" +from __future__ import annotations + +from typing import TYPE_CHECKING +from pathlib import Path +import urllib.request +import logging +import weakref +import time +import os +import sys +# import atexit + +from efro import entity + +if TYPE_CHECKING: + from bacommon.assets import AssetPackageFlavor + from typing import List + + +class FileValue(entity.CompoundValue): + """State for an individual file.""" + + +class State(entity.Entity): + """Holds all persistent state for the asset-manager.""" + + files = entity.CompoundDictField('files', str, FileValue()) + class AssetManager: """Wrangles all assets.""" - def __init__(self) -> None: + _state: State + + def __init__(self, rootdir: Path) -> None: print('AssetManager()') + assert isinstance(rootdir, Path) + self._rootdir = rootdir + self._shutting_down = False + if not self._rootdir.is_dir(): + raise RuntimeError(f'Provided rootdir does not exist: "{rootdir}"') + + self.load_state() + + # atexit.register(self._at_exit) def __del__(self) -> None: + self._shutting_down = True + self.update() print('~AssetManager()') + + # @staticmethod + # def _at_exit() -> None: + # print('HELLO FROM SHUTDOWN') + + def launch_gather( + self, + packages: List[str], + flavor: AssetPackageFlavor, + account_token: str, + ) -> AssetGather: + """Spawn an asset-gather operation from this manager.""" + print('would gather', packages, 'and flavor', flavor, 'with token', + account_token) + return AssetGather(self) + + def update(self) -> None: + """Can be called periodically to perform upkeep.""" + + # Currently we always write state when shutting down. + if self._shutting_down: + self.save_state() + + @property + def rootdir(self) -> Path: + """The root directory for this manager.""" + return self._rootdir + + @property + def state_path(self) -> Path: + """The path of the state file.""" + return Path(self._rootdir, 'state') + + def load_state(self) -> None: + """Loads state from disk. Resets to default state if unable to.""" + print('AMAN LOADING STATE') + try: + state_path = self.state_path + if state_path.exists(): + with open(self.state_path) as infile: + self._state = State.from_json_str(infile.read()) + return + except Exception: + logging.exception('Error loading existing AssetManager state') + self._state = State() + + def save_state(self) -> None: + """Save state to disk (if possible).""" + print('AMAN SAVING STATE') + try: + with open(self.state_path, 'w') as outfile: + outfile.write(self._state.to_json_str()) + except Exception: + logging.exception('Error writing AssetManager state') + + +class AssetGather: + """Wrangles a gather of assets.""" + + def __init__(self, manager: AssetManager) -> None: + self._manager = weakref.ref(manager) + self._valid = True + print('AssetGather()') + fetch_url("http://www.python.org/ftp/python/2.7.3/Python-2.7.3.tgz", + filename=Path(manager.rootdir, 'testdl'), + asset_gather=self) + print('fetch success') + + @property + def valid(self) -> bool: + """Whether this gather is still valid. + + A gather becomes in valid if its originating AssetManager dies. + """ + return True + + def __del__(self) -> None: + print('~AssetGather()') + + +def fetch_url(url: str, filename: Path, asset_gather: AssetGather) -> None: + """Fetch a given url to a given filename for a given AssetGather. + + This """ + + import socket + + # We don't want to keep the provided AssetGather alive, but we want + # to abort if it dies. + assert isinstance(asset_gather, AssetGather) + weak_gather = weakref.ref(asset_gather) + + # Pass a very short timeout to urllib so we have opportunities + # to cancel even with network blockage. + ureq = urllib.request.urlopen(url, None, 1) + file_size = int(ureq.headers["Content-Length"]) + print(f"\nDownloading: {filename} Bytes: {file_size:,}") + + with open(filename, 'wb') as outfile: + file_size_dl = 0 + + # I'm guessing we want this decently big so we're running fewer cycles + # of this loop during downloads and keeping our load lower. Our timeout + # should ensure a minimum rate for the loop and this will affect + # the maximum. Perhaps we should aim for a few cycles per second on + # an average connection?.. + block_sz = 1024 * 100 * 2 + time_outs = 0 + while True: + try: + data = ureq.read(block_sz) + except socket.timeout: + + # File has not had activity in max seconds. + if time_outs > 3: + print("\n\n\nsorry -- try back later") + os.unlink(filename) + raise + print("\nHmmm... little issue... " + "I'll wait a couple of seconds") + time.sleep(3) + time_outs += 1 + continue + + # We reached the end of the download! + if not data: + sys.stdout.write('\rDone!\n\n') + sys.stdout.flush() + break + + file_size_dl += len(data) + outfile.write(data) + percent = file_size_dl * 1.0 / file_size + status = f'{file_size_dl:20,} Bytes [{percent:.2%}] received' + sys.stdout.write('\r' + status) + sys.stdout.flush() diff --git a/assets/src/ba_data/python/efro/entity/_entity.py b/assets/src/ba_data/python/efro/entity/_entity.py index f1eb99cd..2ae00bb2 100644 --- a/assets/src/ba_data/python/efro/entity/_entity.py +++ b/assets/src/ba_data/python/efro/entity/_entity.py @@ -119,7 +119,10 @@ class EntityMixin: f" ({type(self)}); can't steal data.") assert target.d_data is not None self.d_data = target.d_data - target.d_data = None + + # Make sure target blows up if someone tries to use it. + # noinspection PyTypeHints + target.d_data = None # type: ignore def pruned_data(self) -> Dict[str, Any]: """Return a pruned version of this instance's data. diff --git a/assets/src/ba_data/python/efro/jsonutils.py b/assets/src/ba_data/python/efro/jsonutils.py index f3a68fbf..7a2aad6f 100644 --- a/assets/src/ba_data/python/efro/jsonutils.py +++ b/assets/src/ba_data/python/efro/jsonutils.py @@ -71,11 +71,10 @@ class ExtendedJSONDecoder(json.JSONDecoder): """Custom json decoder supporting extended types.""" def __init__(self, *args: Any, **kwargs: Any): - json.JSONDecoder.__init__( # type: ignore - self, - object_hook=self.object_hook, - *args, - **kwargs) + json.JSONDecoder.__init__(self, + object_hook=self.object_hook, + *args, + **kwargs) def object_hook(self, obj: Any) -> Any: # pylint: disable=E0202 """Custom hook.""" diff --git a/assets/src/server/server.py b/assets/src/server/server.py index 56b44146..3141f049 100755 --- a/assets/src/server/server.py +++ b/assets/src/server/server.py @@ -169,6 +169,7 @@ def _run_process_until_exit(process: subprocess.Popen, ftmp.close() # Note to self: Is there a type-safe way we could do this? + assert process.stdin is not None process.stdin.write(('from ba import _server; ' '_server.config_server(config_file=' + repr(fname) + ')\n').encode('utf-8')) diff --git a/docs/ba_module.md b/docs/ba_module.md index f7290b6a..f06af23a 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,6 +1,6 @@ - -

last updated on 2020-03-05 for Ballistica version 1.5.0 build 20001

+ +

last updated on 2020-03-11 for Ballistica version 1.5.0 build 20001

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!

diff --git a/tests/test_ba/test_assetmanager.py b/tests/test_ba/test_assetmanager.py index b65915b3..190f4d0c 100644 --- a/tests/test_ba/test_assetmanager.py +++ b/tests/test_ba/test_assetmanager.py @@ -24,9 +24,13 @@ from __future__ import annotations from typing import TYPE_CHECKING import weakref +import tempfile +from pathlib import Path # noinspection PyProtectedMember from ba._assetmanager import AssetManager +from bacommon.assets import AssetPackageFlavor + # import pytest if TYPE_CHECKING: @@ -36,9 +40,17 @@ if TYPE_CHECKING: def test_assetmanager() -> None: """Testing.""" - manager = AssetManager() - wref = weakref.ref(manager) + with tempfile.TemporaryDirectory() as tmpdir: + man = AssetManager(rootdir=Path(tmpdir)) + wref = weakref.ref(man) - # Make sure it's not keeping itself alive. - del manager - assert wref() is None + gather = man.launch_gather(packages=['a@2'], + flavor=AssetPackageFlavor.DESKTOP, + account_token='dummytoken') + wref2 = weakref.ref(gather) + + # Make sure nothing is keeping itself alive + del man + del gather + assert wref() is None + assert wref2() is None diff --git a/tools/snippets b/tools/snippets index f3a9c9e6..1c000d14 100755 --- a/tools/snippets +++ b/tools/snippets @@ -77,7 +77,7 @@ DO_SPARSE_TEST_BUILDS = 'ballistica' + 'core' == 'ballisticacore' # (module name, required version, pip package (if it differs from module name)) REQUIRED_PYTHON_MODULES = [ ('pylint', [2, 4, 4], None), - ('mypy', [0, 761], None), + ('mypy', [0, 770], None), ('yapf', [0, 29, 0], None), ('typing_extensions', None, None), ('pytz', None, None),