diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml
index 6d79fcbe..4f46ea41 100644
--- a/.idea/dictionaries/ericf.xml
+++ b/.idea/dictionaries/ericf.xml
@@ -554,6 +554,7 @@
fnames
fnmatch
fnode
+ fnum
foof
foos
fopen
@@ -687,6 +688,7 @@
grumbledorf
guitarguy
gval
+ gzpath
hacktastic
hacky
halign
@@ -728,6 +730,7 @@
hspacing
hurtiness
hval
+ iasset
icls
icns
iconpicker
@@ -1247,6 +1250,8 @@
pushlist
putasset
putassetmanifest
+ putassetupload
+ putfiles
pval
pvars
pvrtc
diff --git a/config/config.json b/config/config.json
index eb36cd72..3c700dfc 100644
--- a/config/config.json
+++ b/config/config.json
@@ -26,7 +26,8 @@
"bs_mapdefs_tower_d",
"bs_mapdefs_hockey_stadium",
"bs_mapdefs_roundabout",
- "yaml"
+ "yaml",
+ "requests"
],
"python_paths": [
"assets/src/data/scripts",
diff --git a/tools/cloudtool b/tools/cloudtool
index aee3ef2f..01d09bea 100755
--- a/tools/cloudtool
+++ b/tools/cloudtool
@@ -30,15 +30,15 @@ import os
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING
-import urllib.request
-import urllib.parse
-import urllib.error
from dataclasses import dataclass
import json
import subprocess
+import tempfile
+
+import requests
if TYPE_CHECKING:
- from typing import Optional, Dict, Any, Tuple
+ from typing import Optional, Dict, Any, Tuple, List, BinaryIO
# Version is sent to the master-server with all commands. Can be incremented
# if we need to change behavior server-side to go along with client changes.
@@ -125,7 +125,7 @@ class AssetPackage:
@classmethod
def load_from_disk(cls, path: Path) -> AssetPackage:
- """Load an asset package from an existing one on disk."""
+ """Load an asset package from files on disk."""
import yaml
indexfilename = 'assetpackage.yaml'
package = AssetPackage()
@@ -153,24 +153,26 @@ class AssetPackage:
f'Invalid asset type for {assetpath} in {indexfilename}')
assettype = AssetType(assettypestr)
- print('looking at', assetpath, assetdata)
+ print('Looking at asset:', assetpath, assetdata)
package.assets[assetpath] = Asset(package, assettype, assetpath)
return package
def get_manifest(self) -> Dict:
"""Build a manifest of hashes and other info for files on disk."""
- from efrotools import get_files_hash
+ import hashlib
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count
manifest: Dict = {'files': {}}
def _get_asset_info(iasset: Asset) -> Tuple[Asset, Dict]:
- hval = get_files_hash([iasset.filepath], hashtype='sha256')
+ sha = hashlib.sha256()
+ with open(iasset.filepath, 'rb') as infile:
+ sha.update(infile.read())
if not os.path.isfile(iasset.filepath):
raise Exception(f'Asset file not found: "{iasset.filepath}"')
- info_out: Dict = {'hash': hval}
+ info_out: Dict = {'hash': sha.hexdigest()}
return iasset, info_out
# Use all procs to hash files for extra speedy goodness.
@@ -240,26 +242,34 @@ class App:
with open(CACHE_DATA_PATH, 'w') as outfile:
outfile.write(json.dumps(self._state.__dict__))
- def _servercmd(self, cmd: str, data: Dict) -> Response:
+ def _servercmd(self,
+ cmd: str,
+ data: Dict,
+ files: Dict[str, BinaryIO] = None) -> Response:
"""Issue a command to the server and get a response."""
- # We do all communication through POST requests to the server.
- response_raw = urllib.request.urlopen(
- urllib.request.Request(
- (MASTER_SERVER_ADDRESS + '/cloudtoolcmd'),
- urllib.parse.urlencode({
- 'c': cmd,
- 'v': VERSION,
- 't': json.dumps(self._state.login_token),
- 'd': json.dumps(data)
- }).encode(), {'User-Agent': USER_AGENT_STRING}))
- output = json.loads(response_raw.read().decode())
+ response_raw_2 = requests.post(
+ (MASTER_SERVER_ADDRESS + '/cloudtoolcmd'),
+ data={
+ 'c': cmd,
+ 'v': VERSION,
+ 't': json.dumps(self._state.login_token),
+ 'd': json.dumps(data)
+ },
+ files=files)
+ response_raw_2.raise_for_status() # Except if anything went wrong.
+ assert isinstance(response_raw_2.content, bytes)
+ output = json.loads(response_raw_2.content.decode())
assert isinstance(output, dict)
+ assert isinstance(output['m'], (str, type(None)))
+ assert isinstance(output['e'], (str, type(None)))
+ assert 'd' in output
response = Response(message=output['m'],
data=output['d'],
error=output['e'])
- # Handle errors and messages which are common to all command types.
+ # Handle errors and print messages;
+ # (functionality common to all command types).
if response.error is not None:
raise CleanError(response.error)
@@ -302,15 +312,42 @@ class App:
path = Path(sys.argv[2])
package = AssetPackage.load_from_disk(path)
- # Now send the server a manifest of everything we've got in the local
- # package.
+ # Send the server a manifest of everything we've got locally.
manifest = package.get_manifest()
print('SENDING PACKAGE MANIFEST:', manifest)
response = self._servercmd('putassetmanifest', {'m': manifest})
- # The server's response tells us what we need to upload to them.
- # Do that.
- print('WOULD UPLOAD NEEDED BITS')
+ # The server should give us an upload id and a set of files it wants.
+ # Upload each of those.
+ upload_files: List[str] = response.data['upload_files']
+ assert isinstance(upload_files, list)
+ assert all(isinstance(f, str) for f in upload_files)
+ self._putasset_upload(package, upload_files)
+
+ print('Asset upload successful!')
+
+ def _putasset_upload(self, package: AssetPackage,
+ files: List[str]) -> None:
+
+ # Upload the files one at a time.
+ # (we can potentially do this in parallel in the future).
+ for fnum, fname in enumerate(files):
+ print(
+ f'{CLRBLU}Uploading file {fnum+1} of {len(files)}: '
+ f'"{fname}"...{CLREND}',
+ flush=True)
+ with tempfile.TemporaryDirectory() as tempdir:
+ asset = package.assets[fname]
+ srcpath = Path(asset.filepath)
+ gzpath = Path(tempdir, 'file.gz')
+ subprocess.run(f'gzip --stdout "{srcpath}" > "{gzpath}"',
+ shell=True,
+ check=True)
+ with open(gzpath, 'rb') as infile:
+ putfiles: Dict = {'file': infile}
+ _response = self._servercmd('putassetupload',
+ {'path': asset.path},
+ files=putfiles)
def do_misc_command(self) -> None:
"""Run a miscellaneous command."""
diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py
index ef555569..1a2fbc03 100644
--- a/tools/efrotools/pybuild.py
+++ b/tools/efrotools/pybuild.py
@@ -34,10 +34,10 @@ if TYPE_CHECKING:
PYTHON_VERSION_MAJOR = "3.7"
# Specific version we're using on apple builds.
-PYTHON_VERSION_APPLE = "3.7.0"
+# PYTHON_VERSION_APPLE = "3.7.0"
# Specific version we're using on android builds.
-PYTHON_VERSION_ANDROID = "3.7.2"
+# PYTHON_VERSION_ANDROID = "3.7.2"
ENABLE_OPENSSL = True
@@ -163,7 +163,7 @@ def build_apple(arch: str, debug: bool = False) -> None:
txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.13')
# And equivalent iOS (11+).
- txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=7.0',
+ txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=11.0')
# Ditto for tvOS.
txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
@@ -283,7 +283,9 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
efrotools.writefile('pybuild/packages/python.py', ftxt)
# Set this to a particular cpython commit to target exact releases from git
- commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release
+ # commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release
+ commit = '5c02a39a0b31a330e06b4d6f44835afb205dc7cc' # 3.7.5 release
+
if commit is not None:
ftxt = efrotools.readfile('pybuild/source.py')
diff --git a/tools/snippets b/tools/snippets
index 4ade4673..0875079b 100755
--- a/tools/snippets
+++ b/tools/snippets
@@ -73,6 +73,18 @@ SPARSE_TESTS: List[List[str]] = [
# (whole word will get subbed out in spinoffs so this will be false)
DO_SPARSE_TESTS = 'ballistica' + 'core' == 'ballisticacore'
+# Python modules we require for this project.
+# (module name, required version, pip package (if it differs from module name))
+REQUIRED_PYTHON_MODULES = [
+ ('pylint', [2, 4, 4], None),
+ ('mypy', [0, 740], None),
+ ('yapf', [0, 29, 0], None),
+ ('typing_extensions', None, None),
+ ('pytz', None, None),
+ ('yaml', None, 'PyYAML'),
+ ('requests', None, None),
+]
+
def archive_old_builds() -> None:
"""Stuff our old public builds into the 'old' dir.
@@ -692,6 +704,16 @@ def update_docs_md() -> None:
print(f'{docs_path} is up to date.')
+def pip_req_list() -> None:
+ """List Python Pip packages needed for this project."""
+ out: List[str] = []
+ for module in REQUIRED_PYTHON_MODULES:
+ name = module[0] if module[2] is None else module[2]
+ assert isinstance(name, str)
+ out.append(name)
+ print(' '.join(out))
+
+
def checkenv() -> None:
"""Check for tools necessary to build and run the app."""
print('Checking environment...', flush=True)
@@ -711,14 +733,7 @@ def checkenv() -> None:
raise CleanError('pip (for {python_bin}) is required.')
# Check for some required python modules.
- for modname, minver, packagename in [
- ('pylint', [2, 4, 4], None),
- ('mypy', [0, 740], None),
- ('yapf', [0, 29, 0], None),
- ('typing_extensions', None, None),
- ('pytz', None, None),
- ('yaml', None, 'PyYAML'),
- ]:
+ for modname, minver, packagename in REQUIRED_PYTHON_MODULES:
if packagename is None:
packagename = modname
if minver is not None: