Syncing latest changes between public/private.

.idea/dictionaries/ericf.xml

@@ -45,6 +45,7 @@
       <w>allpaths</w>
       <w>allsettings</w>
       <w>allteams</w>
+      <w>aman</w>
       <w>aname</w>
       <w>anamorphosis</w>
       <w>andr</w>
@@ -105,6 +106,7 @@
       <w>asyncio</w>
       <w>asynciomodule</w>
       <w>asyncore</w>
+      <w>atexit</w>
       <w>attrdict</w>
       <w>attrdocs</w>
       <w>attrinfo</w>
@@ -463,6 +465,7 @@
       <w>duckdns</w>
       <w>dummymodule</w>
       <w>dummyname</w>
+      <w>dummytoken</w>
       <w>dummyval</w>
       <w>dups</w>
       <w>dxml</w>
@@ -1662,6 +1665,7 @@
       <w>testcapimodule</w>
       <w>testclass</w>
       <w>testd</w>
+      <w>testdl</w>
       <w>testfoo</w>
       <w>testfooooo</w>
       <w>testfull</w>
@@ -1797,6 +1801,7 @@
       <w>uploadargs</w>
       <w>uploadcmd</w>
       <w>uptime</w>
+      <w>ureq</w>
       <w>useragent</w>
       <w>useragentstring</w>
       <w>userbase</w>

assets/src/ba_data/python/ba/_assetmanager.py

@@ -20,12 +20,189 @@
 # -----------------------------------------------------------------------------
 """Functionality related to managing cloud based assets."""
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from pathlib import Path
+import urllib.request
+import logging
+import weakref
+import time
+import os
+import sys
+# import atexit
+
+from efro import entity
+
+if TYPE_CHECKING:
+    from bacommon.assets import AssetPackageFlavor
+    from typing import List
+
+
+class FileValue(entity.CompoundValue):
+    """State for an individual file."""
+
+
+class State(entity.Entity):
+    """Holds all persistent state for the asset-manager."""
+
+    files = entity.CompoundDictField('files', str, FileValue())
+
 
 class AssetManager:
     """Wrangles all assets."""
 
-    def __init__(self) -> None:
+    _state: State
+
+    def __init__(self, rootdir: Path) -> None:
         print('AssetManager()')
+        assert isinstance(rootdir, Path)
+        self._rootdir = rootdir
+        self._shutting_down = False
+        if not self._rootdir.is_dir():
+            raise RuntimeError(f'Provided rootdir does not exist: "{rootdir}"')
+
+        self.load_state()
+
+        # atexit.register(self._at_exit)
 
     def __del__(self) -> None:
+        self._shutting_down = True
+        self.update()
         print('~AssetManager()')
+
+    # @staticmethod
+    # def _at_exit() -> None:
+    #     print('HELLO FROM SHUTDOWN')
+
+    def launch_gather(
+            self,
+            packages: List[str],
+            flavor: AssetPackageFlavor,
+            account_token: str,
+    ) -> AssetGather:
+        """Spawn an asset-gather operation from this manager."""
+        print('would gather', packages, 'and flavor', flavor, 'with token',
+              account_token)
+        return AssetGather(self)
+
+    def update(self) -> None:
+        """Can be called periodically to perform upkeep."""
+
+        # Currently we always write state when shutting down.
+        if self._shutting_down:
+            self.save_state()
+
+    @property
+    def rootdir(self) -> Path:
+        """The root directory for this manager."""
+        return self._rootdir
+
+    @property
+    def state_path(self) -> Path:
+        """The path of the state file."""
+        return Path(self._rootdir, 'state')
+
+    def load_state(self) -> None:
+        """Loads state from disk. Resets to default state if unable to."""
+        print('AMAN LOADING STATE')
+        try:
+            state_path = self.state_path
+            if state_path.exists():
+                with open(self.state_path) as infile:
+                    self._state = State.from_json_str(infile.read())
+                    return
+        except Exception:
+            logging.exception('Error loading existing AssetManager state')
+        self._state = State()
+
+    def save_state(self) -> None:
+        """Save state to disk (if possible)."""
+        print('AMAN SAVING STATE')
+        try:
+            with open(self.state_path, 'w') as outfile:
+                outfile.write(self._state.to_json_str())
+        except Exception:
+            logging.exception('Error writing AssetManager state')
+
+
+class AssetGather:
+    """Wrangles a gather of assets."""
+
+    def __init__(self, manager: AssetManager) -> None:
+        self._manager = weakref.ref(manager)
+        self._valid = True
+        print('AssetGather()')
+        fetch_url("http://www.python.org/ftp/python/2.7.3/Python-2.7.3.tgz",
+                  filename=Path(manager.rootdir, 'testdl'),
+                  asset_gather=self)
+        print('fetch success')
+
+    @property
+    def valid(self) -> bool:
+        """Whether this gather is still valid.
+
+        A gather becomes in valid if its originating AssetManager dies.
+        """
+        return True
+
+    def __del__(self) -> None:
+        print('~AssetGather()')
+
+
+def fetch_url(url: str, filename: Path, asset_gather: AssetGather) -> None:
+    """Fetch a given url to a given filename for a given AssetGather.
+
+    This """
+
+    import socket
+
+    # We don't want to keep the provided AssetGather alive, but we want
+    # to abort if it dies.
+    assert isinstance(asset_gather, AssetGather)
+    weak_gather = weakref.ref(asset_gather)
+
+    # Pass a very short timeout to urllib so we have opportunities
+    # to cancel even with network blockage.
+    ureq = urllib.request.urlopen(url, None, 1)
+    file_size = int(ureq.headers["Content-Length"])
+    print(f"\nDownloading: {filename} Bytes: {file_size:,}")
+
+    with open(filename, 'wb') as outfile:
+        file_size_dl = 0
+
+        # I'm guessing we want this decently big so we're running fewer cycles
+        # of this loop during downloads and keeping our load lower. Our timeout
+        # should ensure a minimum rate for the loop and this will affect
+        # the maximum. Perhaps we should aim for a few cycles per second on
+        # an average connection?..
+        block_sz = 1024 * 100 * 2
+        time_outs = 0
+        while True:
+            try:
+                data = ureq.read(block_sz)
+            except socket.timeout:
+
+                # File has not had activity in max seconds.
+                if time_outs > 3:
+                    print("\n\n\nsorry -- try back later")
+                    os.unlink(filename)
+                    raise
+                print("\nHmmm... little issue... "
+                      "I'll wait a couple of seconds")
+                time.sleep(3)
+                time_outs += 1
+                continue
+
+            # We reached the end of the download!
+            if not data:
+                sys.stdout.write('\rDone!\n\n')
+                sys.stdout.flush()
+                break
+
+            file_size_dl += len(data)
+            outfile.write(data)
+            percent = file_size_dl * 1.0 / file_size
+            status = f'{file_size_dl:20,} Bytes [{percent:.2%}] received'
+            sys.stdout.write('\r' + status)
+            sys.stdout.flush()

assets/src/ba_data/python/efro/entity/_entity.py

@@ -119,7 +119,10 @@ class EntityMixin:
                 f" ({type(self)}); can't steal data.")
         assert target.d_data is not None
         self.d_data = target.d_data
-        target.d_data = None
+
+        # Make sure target blows up if someone tries to use it.
+        # noinspection PyTypeHints
+        target.d_data = None  # type: ignore
 
     def pruned_data(self) -> Dict[str, Any]:
         """Return a pruned version of this instance's data.

assets/src/ba_data/python/efro/jsonutils.py

@@ -71,11 +71,10 @@ class ExtendedJSONDecoder(json.JSONDecoder):
     """Custom json decoder supporting extended types."""
 
     def __init__(self, *args: Any, **kwargs: Any):
-        json.JSONDecoder.__init__(  # type: ignore
+        json.JSONDecoder.__init__(self,
-            self,
+                                  object_hook=self.object_hook,
-            object_hook=self.object_hook,
+                                  *args,
-            *args,
+                                  **kwargs)
-            **kwargs)
 
     def object_hook(self, obj: Any) -> Any:  # pylint: disable=E0202
         """Custom hook."""

assets/src/server/server.py

@@ -169,6 +169,7 @@ def _run_process_until_exit(process: subprocess.Popen,
             ftmp.close()
 
             # Note to self: Is there a type-safe way we could do this?
+            assert process.stdin is not None
             process.stdin.write(('from ba import _server; '
                                  '_server.config_server(config_file=' +
                                  repr(fname) + ')\n').encode('utf-8'))

docs/ba_module.md

@@ -1,6 +1,6 @@
 <!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
-<!--DOCSHASH=03d4e0e0a8991c9db5174ca7111ff595-->
+<!--DOCSHASH=5b1863340d0e423b597b188ee7c7ac23-->
-<h4><em>last updated on 2020-03-05 for Ballistica version 1.5.0 build 20001</em></h4>
+<h4><em>last updated on 2020-03-11 for Ballistica version 1.5.0 build 20001</em></h4>
 <p>This page documents the Python classes and functions in the 'ba' module,
  which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
 <hr>

tests/test_ba/test_assetmanager.py

@@ -24,9 +24,13 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 import weakref
+import tempfile
+from pathlib import Path
 # noinspection PyProtectedMember
 from ba._assetmanager import AssetManager
 
+from bacommon.assets import AssetPackageFlavor
+
 # import pytest
 
 if TYPE_CHECKING:
@@ -36,9 +40,17 @@ if TYPE_CHECKING:
 def test_assetmanager() -> None:
     """Testing."""
 
-    manager = AssetManager()
+    with tempfile.TemporaryDirectory() as tmpdir:
-    wref = weakref.ref(manager)
+        man = AssetManager(rootdir=Path(tmpdir))
+        wref = weakref.ref(man)
 
-    # Make sure it's not keeping itself alive.
+        gather = man.launch_gather(packages=['a@2'],
-    del manager
+                                   flavor=AssetPackageFlavor.DESKTOP,
-    assert wref() is None
+                                   account_token='dummytoken')
+        wref2 = weakref.ref(gather)
+
+        # Make sure nothing is keeping itself alive
+        del man
+        del gather
+        assert wref() is None
+        assert wref2() is None

tools/snippets

@@ -77,7 +77,7 @@ DO_SPARSE_TEST_BUILDS = 'ballistica' + 'core' == 'ballisticacore'
 # (module name, required version, pip package (if it differs from module name))
 REQUIRED_PYTHON_MODULES = [
     ('pylint', [2, 4, 4], None),
-    ('mypy', [0, 761], None),
+    ('mypy', [0, 770], None),
     ('yapf', [0, 29, 0], None),
     ('typing_extensions', None, None),
     ('pytz', None, None),