From 57f6dc354bd6223539070682166d161987bde771 Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Tue, 10 Dec 2019 19:21:49 -0800 Subject: [PATCH] latest binaries and cloudtool work --- .idea/dictionaries/ericf.xml | 5 ++ config/config.json | 3 +- tools/cloudtool | 91 +++++++++++++++++++++++++----------- tools/efrotools/pybuild.py | 10 ++-- tools/snippets | 31 ++++++++---- 5 files changed, 100 insertions(+), 40 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 6d79fcbe..4f46ea41 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -554,6 +554,7 @@ fnames fnmatch fnode + fnum foof foos fopen @@ -687,6 +688,7 @@ grumbledorf guitarguy gval + gzpath hacktastic hacky halign @@ -728,6 +730,7 @@ hspacing hurtiness hval + iasset icls icns iconpicker @@ -1247,6 +1250,8 @@ pushlist putasset putassetmanifest + putassetupload + putfiles pval pvars pvrtc diff --git a/config/config.json b/config/config.json index eb36cd72..3c700dfc 100644 --- a/config/config.json +++ b/config/config.json @@ -26,7 +26,8 @@ "bs_mapdefs_tower_d", "bs_mapdefs_hockey_stadium", "bs_mapdefs_roundabout", - "yaml" + "yaml", + "requests" ], "python_paths": [ "assets/src/data/scripts", diff --git a/tools/cloudtool b/tools/cloudtool index aee3ef2f..01d09bea 100755 --- a/tools/cloudtool +++ b/tools/cloudtool @@ -30,15 +30,15 @@ import os from enum import Enum from pathlib import Path from typing import TYPE_CHECKING -import urllib.request -import urllib.parse -import urllib.error from dataclasses import dataclass import json import subprocess +import tempfile + +import requests if TYPE_CHECKING: - from typing import Optional, Dict, Any, Tuple + from typing import Optional, Dict, Any, Tuple, List, BinaryIO # Version is sent to the master-server with all commands. Can be incremented # if we need to change behavior server-side to go along with client changes. @@ -125,7 +125,7 @@ class AssetPackage: @classmethod def load_from_disk(cls, path: Path) -> AssetPackage: - """Load an asset package from an existing one on disk.""" + """Load an asset package from files on disk.""" import yaml indexfilename = 'assetpackage.yaml' package = AssetPackage() @@ -153,24 +153,26 @@ class AssetPackage: f'Invalid asset type for {assetpath} in {indexfilename}') assettype = AssetType(assettypestr) - print('looking at', assetpath, assetdata) + print('Looking at asset:', assetpath, assetdata) package.assets[assetpath] = Asset(package, assettype, assetpath) return package def get_manifest(self) -> Dict: """Build a manifest of hashes and other info for files on disk.""" - from efrotools import get_files_hash + import hashlib from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count manifest: Dict = {'files': {}} def _get_asset_info(iasset: Asset) -> Tuple[Asset, Dict]: - hval = get_files_hash([iasset.filepath], hashtype='sha256') + sha = hashlib.sha256() + with open(iasset.filepath, 'rb') as infile: + sha.update(infile.read()) if not os.path.isfile(iasset.filepath): raise Exception(f'Asset file not found: "{iasset.filepath}"') - info_out: Dict = {'hash': hval} + info_out: Dict = {'hash': sha.hexdigest()} return iasset, info_out # Use all procs to hash files for extra speedy goodness. @@ -240,26 +242,34 @@ class App: with open(CACHE_DATA_PATH, 'w') as outfile: outfile.write(json.dumps(self._state.__dict__)) - def _servercmd(self, cmd: str, data: Dict) -> Response: + def _servercmd(self, + cmd: str, + data: Dict, + files: Dict[str, BinaryIO] = None) -> Response: """Issue a command to the server and get a response.""" - # We do all communication through POST requests to the server. - response_raw = urllib.request.urlopen( - urllib.request.Request( - (MASTER_SERVER_ADDRESS + '/cloudtoolcmd'), - urllib.parse.urlencode({ - 'c': cmd, - 'v': VERSION, - 't': json.dumps(self._state.login_token), - 'd': json.dumps(data) - }).encode(), {'User-Agent': USER_AGENT_STRING})) - output = json.loads(response_raw.read().decode()) + response_raw_2 = requests.post( + (MASTER_SERVER_ADDRESS + '/cloudtoolcmd'), + data={ + 'c': cmd, + 'v': VERSION, + 't': json.dumps(self._state.login_token), + 'd': json.dumps(data) + }, + files=files) + response_raw_2.raise_for_status() # Except if anything went wrong. + assert isinstance(response_raw_2.content, bytes) + output = json.loads(response_raw_2.content.decode()) assert isinstance(output, dict) + assert isinstance(output['m'], (str, type(None))) + assert isinstance(output['e'], (str, type(None))) + assert 'd' in output response = Response(message=output['m'], data=output['d'], error=output['e']) - # Handle errors and messages which are common to all command types. + # Handle errors and print messages; + # (functionality common to all command types). if response.error is not None: raise CleanError(response.error) @@ -302,15 +312,42 @@ class App: path = Path(sys.argv[2]) package = AssetPackage.load_from_disk(path) - # Now send the server a manifest of everything we've got in the local - # package. + # Send the server a manifest of everything we've got locally. manifest = package.get_manifest() print('SENDING PACKAGE MANIFEST:', manifest) response = self._servercmd('putassetmanifest', {'m': manifest}) - # The server's response tells us what we need to upload to them. - # Do that. - print('WOULD UPLOAD NEEDED BITS') + # The server should give us an upload id and a set of files it wants. + # Upload each of those. + upload_files: List[str] = response.data['upload_files'] + assert isinstance(upload_files, list) + assert all(isinstance(f, str) for f in upload_files) + self._putasset_upload(package, upload_files) + + print('Asset upload successful!') + + def _putasset_upload(self, package: AssetPackage, + files: List[str]) -> None: + + # Upload the files one at a time. + # (we can potentially do this in parallel in the future). + for fnum, fname in enumerate(files): + print( + f'{CLRBLU}Uploading file {fnum+1} of {len(files)}: ' + f'"{fname}"...{CLREND}', + flush=True) + with tempfile.TemporaryDirectory() as tempdir: + asset = package.assets[fname] + srcpath = Path(asset.filepath) + gzpath = Path(tempdir, 'file.gz') + subprocess.run(f'gzip --stdout "{srcpath}" > "{gzpath}"', + shell=True, + check=True) + with open(gzpath, 'rb') as infile: + putfiles: Dict = {'file': infile} + _response = self._servercmd('putassetupload', + {'path': asset.path}, + files=putfiles) def do_misc_command(self) -> None: """Run a miscellaneous command.""" diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py index ef555569..1a2fbc03 100644 --- a/tools/efrotools/pybuild.py +++ b/tools/efrotools/pybuild.py @@ -34,10 +34,10 @@ if TYPE_CHECKING: PYTHON_VERSION_MAJOR = "3.7" # Specific version we're using on apple builds. -PYTHON_VERSION_APPLE = "3.7.0" +# PYTHON_VERSION_APPLE = "3.7.0" # Specific version we're using on android builds. -PYTHON_VERSION_ANDROID = "3.7.2" +# PYTHON_VERSION_ANDROID = "3.7.2" ENABLE_OPENSSL = True @@ -163,7 +163,7 @@ def build_apple(arch: str, debug: bool = False) -> None: txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', 'MACOSX_DEPLOYMENT_TARGET=10.13') # And equivalent iOS (11+). - txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=7.0', + txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', 'CFLAGS-iOS=-mios-version-min=11.0') # Ditto for tvOS. txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', @@ -283,7 +283,9 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: efrotools.writefile('pybuild/packages/python.py', ftxt) # Set this to a particular cpython commit to target exact releases from git - commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release + # commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release + commit = '5c02a39a0b31a330e06b4d6f44835afb205dc7cc' # 3.7.5 release + if commit is not None: ftxt = efrotools.readfile('pybuild/source.py') diff --git a/tools/snippets b/tools/snippets index 4ade4673..0875079b 100755 --- a/tools/snippets +++ b/tools/snippets @@ -73,6 +73,18 @@ SPARSE_TESTS: List[List[str]] = [ # (whole word will get subbed out in spinoffs so this will be false) DO_SPARSE_TESTS = 'ballistica' + 'core' == 'ballisticacore' +# Python modules we require for this project. +# (module name, required version, pip package (if it differs from module name)) +REQUIRED_PYTHON_MODULES = [ + ('pylint', [2, 4, 4], None), + ('mypy', [0, 740], None), + ('yapf', [0, 29, 0], None), + ('typing_extensions', None, None), + ('pytz', None, None), + ('yaml', None, 'PyYAML'), + ('requests', None, None), +] + def archive_old_builds() -> None: """Stuff our old public builds into the 'old' dir. @@ -692,6 +704,16 @@ def update_docs_md() -> None: print(f'{docs_path} is up to date.') +def pip_req_list() -> None: + """List Python Pip packages needed for this project.""" + out: List[str] = [] + for module in REQUIRED_PYTHON_MODULES: + name = module[0] if module[2] is None else module[2] + assert isinstance(name, str) + out.append(name) + print(' '.join(out)) + + def checkenv() -> None: """Check for tools necessary to build and run the app.""" print('Checking environment...', flush=True) @@ -711,14 +733,7 @@ def checkenv() -> None: raise CleanError('pip (for {python_bin}) is required.') # Check for some required python modules. - for modname, minver, packagename in [ - ('pylint', [2, 4, 4], None), - ('mypy', [0, 740], None), - ('yapf', [0, 29, 0], None), - ('typing_extensions', None, None), - ('pytz', None, None), - ('yaml', None, 'PyYAML'), - ]: + for modname, minver, packagename in REQUIRED_PYTHON_MODULES: if packagename is None: packagename = modname if minver is not None: