RYDE-WORK/ballistica
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ballistica.git synced 2026-01-26 08:53:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Syncing latest changes between public/private.

Browse Source
This commit is contained in:
Eric Froemling 2020-03-11 14:18:00 -07:00
parent 84338018b9
commit a6b940160c
8 changed files with 212 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.idea/dictionaries/ericf.xml generated
View File

@ -45,6 +45,7 @@
<w>allpaths</w>
<w>allsettings</w>
<w>allteams</w>
<w>aman</w>
<w>aname</w>
<w>anamorphosis</w>
<w>andr</w>
@ -105,6 +106,7 @@
<w>asyncio</w>
<w>asynciomodule</w>
<w>asyncore</w>
<w>atexit</w>
<w>attrdict</w>
<w>attrdocs</w>
<w>attrinfo</w>
@ -463,6 +465,7 @@
<w>duckdns</w>
<w>dummymodule</w>
<w>dummyname</w>
<w>dummytoken</w>
<w>dummyval</w>
<w>dups</w>
<w>dxml</w>
@ -1662,6 +1665,7 @@
<w>testcapimodule</w>
<w>testclass</w>
<w>testd</w>
<w>testdl</w>
<w>testfoo</w>
<w>testfooooo</w>
<w>testfull</w>
@ -1797,6 +1801,7 @@
<w>uploadargs</w>
<w>uploadcmd</w>
<w>uptime</w>
<w>ureq</w>
<w>useragent</w>
<w>useragentstring</w>
<w>userbase</w>

179
assets/src/ba_data/python/ba/_assetmanager.py
View File

@ -20,12 +20,189 @@
# -----------------------------------------------------------------------------
"""Functionality related to managing cloud based assets."""
from __future__ import annotations
from typing import TYPE_CHECKING
from pathlib import Path
import urllib.request
import logging
import weakref
import time
import os
import sys
# import atexit
from efro import entity
if TYPE_CHECKING:
from bacommon.assets import AssetPackageFlavor
from typing import List
class FileValue(entity.CompoundValue):
"""State for an individual file."""
class State(entity.Entity):
"""Holds all persistent state for the asset-manager."""
files = entity.CompoundDictField('files', str, FileValue())
class AssetManager:
"""Wrangles all assets."""
def __init__(self) -> None:
_state: State
def __init__(self, rootdir: Path) -> None:
print('AssetManager()')
assert isinstance(rootdir, Path)
self._rootdir = rootdir
self._shutting_down = False
if not self._rootdir.is_dir():
raise RuntimeError(f'Provided rootdir does not exist: "{rootdir}"')
self.load_state()
# atexit.register(self._at_exit)
def __del__(self) -> None:
self._shutting_down = True
self.update()
print('~AssetManager()')
# @staticmethod
# def _at_exit() -> None:
# print('HELLO FROM SHUTDOWN')
def launch_gather(
self,
packages: List[str],
flavor: AssetPackageFlavor,
account_token: str,
) -> AssetGather:
"""Spawn an asset-gather operation from this manager."""
print('would gather', packages, 'and flavor', flavor, 'with token',
account_token)
return AssetGather(self)
def update(self) -> None:
"""Can be called periodically to perform upkeep."""
# Currently we always write state when shutting down.
if self._shutting_down:
self.save_state()
@property
def rootdir(self) -> Path:
"""The root directory for this manager."""
return self._rootdir
@property
def state_path(self) -> Path:
"""The path of the state file."""
return Path(self._rootdir, 'state')
def load_state(self) -> None:
"""Loads state from disk. Resets to default state if unable to."""
print('AMAN LOADING STATE')
try:
state_path = self.state_path
if state_path.exists():
with open(self.state_path) as infile:
self._state = State.from_json_str(infile.read())
return
except Exception:
logging.exception('Error loading existing AssetManager state')
self._state = State()
def save_state(self) -> None:
"""Save state to disk (if possible)."""
print('AMAN SAVING STATE')
try:
with open(self.state_path, 'w') as outfile:
outfile.write(self._state.to_json_str())
except Exception:
logging.exception('Error writing AssetManager state')
class AssetGather:
"""Wrangles a gather of assets."""
def __init__(self, manager: AssetManager) -> None:
self._manager = weakref.ref(manager)
self._valid = True
print('AssetGather()')
fetch_url("http://www.python.org/ftp/python/2.7.3/Python-2.7.3.tgz",
filename=Path(manager.rootdir, 'testdl'),
asset_gather=self)
print('fetch success')
@property
def valid(self) -> bool:
"""Whether this gather is still valid.
A gather becomes in valid if its originating AssetManager dies.
"""
return True
def __del__(self) -> None:
print('~AssetGather()')
def fetch_url(url: str, filename: Path, asset_gather: AssetGather) -> None:
"""Fetch a given url to a given filename for a given AssetGather.
This """
import socket
# We don't want to keep the provided AssetGather alive, but we want
# to abort if it dies.
assert isinstance(asset_gather, AssetGather)
weak_gather = weakref.ref(asset_gather)
# Pass a very short timeout to urllib so we have opportunities
# to cancel even with network blockage.
ureq = urllib.request.urlopen(url, None, 1)
file_size = int(ureq.headers["Content-Length"])
print(f"\nDownloading: {filename} Bytes: {file_size:,}")
with open(filename, 'wb') as outfile:
file_size_dl = 0
# I'm guessing we want this decently big so we're running fewer cycles
# of this loop during downloads and keeping our load lower. Our timeout
# should ensure a minimum rate for the loop and this will affect
# the maximum. Perhaps we should aim for a few cycles per second on
# an average connection?..
block_sz = 1024 * 100 * 2
time_outs = 0
while True:
try:
data = ureq.read(block_sz)
except socket.timeout:
# File has not had activity in max seconds.
if time_outs > 3:
print("\n\n\nsorry -- try back later")
os.unlink(filename)
raise
print("\nHmmm... little issue... "
"I'll wait a couple of seconds")
time.sleep(3)
time_outs += 1
continue
# We reached the end of the download!
if not data:
sys.stdout.write('\rDone!\n\n')
sys.stdout.flush()
break
file_size_dl += len(data)
outfile.write(data)
percent = file_size_dl * 1.0 / file_size
status = f'{file_size_dl:20,} Bytes [{percent:.2%}] received'
sys.stdout.write('\r' + status)
sys.stdout.flush()

5
assets/src/ba_data/python/efro/entity/_entity.py
View File

@ -119,7 +119,10 @@ class EntityMixin:
f" ({type(self)}); can't steal data.")
assert target.d_data is not None
self.d_data = target.d_data
target.d_data = None
# Make sure target blows up if someone tries to use it.
# noinspection PyTypeHints
target.d_data = None # type: ignore
def pruned_data(self) -> Dict[str, Any]:
"""Return a pruned version of this instance's data.

9
assets/src/ba_data/python/efro/jsonutils.py
View File

@ -71,11 +71,10 @@ class ExtendedJSONDecoder(json.JSONDecoder):
"""Custom json decoder supporting extended types."""
def __init__(self, *args: Any, **kwargs: Any):
json.JSONDecoder.__init__( # type: ignore
self,
object_hook=self.object_hook,
*args,
**kwargs)
json.JSONDecoder.__init__(self,
object_hook=self.object_hook,
*args,
**kwargs)
def object_hook(self, obj: Any) -> Any: # pylint: disable=E0202
"""Custom hook."""

1
assets/src/server/server.py
View File

@ -169,6 +169,7 @@ def _run_process_until_exit(process: subprocess.Popen,
ftmp.close()
# Note to self: Is there a type-safe way we could do this?
assert process.stdin is not None
process.stdin.write(('from ba import _server; '
'_server.config_server(config_file=' +
repr(fname) + ')\n').encode('utf-8'))

4
docs/ba_module.md
View File

@ -1,6 +1,6 @@
<!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
<!--DOCSHASH=03d4e0e0a8991c9db5174ca7111ff595-->
<h4><em>last updated on 2020-03-05 for Ballistica version 1.5.0 build 20001</em></h4>
<!--DOCSHASH=5b1863340d0e423b597b188ee7c7ac23-->
<h4><em>last updated on 2020-03-11 for Ballistica version 1.5.0 build 20001</em></h4>
<p>This page documents the Python classes and functions in the 'ba' module,
which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
<hr>

22
tests/test_ba/test_assetmanager.py
View File

@ -24,9 +24,13 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import weakref
import tempfile
from pathlib import Path
# noinspection PyProtectedMember
from ba._assetmanager import AssetManager
from bacommon.assets import AssetPackageFlavor
# import pytest
if TYPE_CHECKING:
@ -36,9 +40,17 @@ if TYPE_CHECKING:
def test_assetmanager() -> None:
"""Testing."""
manager = AssetManager()
wref = weakref.ref(manager)
with tempfile.TemporaryDirectory() as tmpdir:
man = AssetManager(rootdir=Path(tmpdir))
wref = weakref.ref(man)
# Make sure it's not keeping itself alive.
del manager
assert wref() is None
gather = man.launch_gather(packages=['a@2'],
flavor=AssetPackageFlavor.DESKTOP,
account_token='dummytoken')
wref2 = weakref.ref(gather)
# Make sure nothing is keeping itself alive
del man
del gather
assert wref() is None
assert wref2() is None

2
tools/snippets
View File

@ -77,7 +77,7 @@ DO_SPARSE_TEST_BUILDS = 'ballistica' + 'core' == 'ballisticacore'
# (module name, required version, pip package (if it differs from module name))
REQUIRED_PYTHON_MODULES = [
('pylint', [2, 4, 4], None),
('mypy', [0, 761], None),
('mypy', [0, 770], None),
('yapf', [0, 29, 0], None),
('typing_extensions', None, None),
('pytz', None, None),