RYDE-WORK/ballistica
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ballistica.git synced 2026-02-02 13:23:27 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

latest binaries and cloudtool work

Browse Source
This commit is contained in:
Eric Froemling 2019-12-10 19:21:49 -08:00
parent da2237ea69
commit 57f6dc354b
5 changed files with 100 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.idea/dictionaries/ericf.xml generated
View File

@ -554,6 +554,7 @@
<w>fnames</w>
<w>fnmatch</w>
<w>fnode</w>
<w>fnum</w>
<w>foof</w>
<w>foos</w>
<w>fopen</w>
@ -687,6 +688,7 @@
<w>grumbledorf</w>
<w>guitarguy</w>
<w>gval</w>
<w>gzpath</w>
<w>hacktastic</w>
<w>hacky</w>
<w>halign</w>
@ -728,6 +730,7 @@
<w>hspacing</w>
<w>hurtiness</w>
<w>hval</w>
<w>iasset</w>
<w>icls</w>
<w>icns</w>
<w>iconpicker</w>
@ -1247,6 +1250,8 @@
<w>pushlist</w>
<w>putasset</w>
<w>putassetmanifest</w>
<w>putassetupload</w>
<w>putfiles</w>
<w>pval</w>
<w>pvars</w>
<w>pvrtc</w>

3
config/config.json
View File

@ -26,7 +26,8 @@
"bs_mapdefs_tower_d",
"bs_mapdefs_hockey_stadium",
"bs_mapdefs_roundabout",
"yaml"
"yaml",
"requests"
],
"python_paths": [
"assets/src/data/scripts",

91
tools/cloudtool
View File

@ -30,15 +30,15 @@ import os
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING
import urllib.request
import urllib.parse
import urllib.error
from dataclasses import dataclass
import json
import subprocess
import tempfile
import requests
if TYPE_CHECKING:
from typing import Optional, Dict, Any, Tuple
from typing import Optional, Dict, Any, Tuple, List, BinaryIO
# Version is sent to the master-server with all commands. Can be incremented
# if we need to change behavior server-side to go along with client changes.
@ -125,7 +125,7 @@ class AssetPackage:
@classmethod
def load_from_disk(cls, path: Path) -> AssetPackage:
"""Load an asset package from an existing one on disk."""
"""Load an asset package from files on disk."""
import yaml
indexfilename = 'assetpackage.yaml'
package = AssetPackage()
@ -153,24 +153,26 @@ class AssetPackage:
f'Invalid asset type for {assetpath} in {indexfilename}')
assettype = AssetType(assettypestr)
print('looking at', assetpath, assetdata)
print('Looking at asset:', assetpath, assetdata)
package.assets[assetpath] = Asset(package, assettype, assetpath)
return package
def get_manifest(self) -> Dict:
"""Build a manifest of hashes and other info for files on disk."""
from efrotools import get_files_hash
import hashlib
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count
manifest: Dict = {'files': {}}
def _get_asset_info(iasset: Asset) -> Tuple[Asset, Dict]:
hval = get_files_hash([iasset.filepath], hashtype='sha256')
sha = hashlib.sha256()
with open(iasset.filepath, 'rb') as infile:
sha.update(infile.read())
if not os.path.isfile(iasset.filepath):
raise Exception(f'Asset file not found: "{iasset.filepath}"')
info_out: Dict = {'hash': hval}
info_out: Dict = {'hash': sha.hexdigest()}
return iasset, info_out
# Use all procs to hash files for extra speedy goodness.
@ -240,26 +242,34 @@ class App:
with open(CACHE_DATA_PATH, 'w') as outfile:
outfile.write(json.dumps(self._state.__dict__))
def _servercmd(self, cmd: str, data: Dict) -> Response:
def _servercmd(self,
cmd: str,
data: Dict,
files: Dict[str, BinaryIO] = None) -> Response:
"""Issue a command to the server and get a response."""
# We do all communication through POST requests to the server.
response_raw = urllib.request.urlopen(
urllib.request.Request(
(MASTER_SERVER_ADDRESS + '/cloudtoolcmd'),
urllib.parse.urlencode({
'c': cmd,
'v': VERSION,
't': json.dumps(self._state.login_token),
'd': json.dumps(data)
}).encode(), {'User-Agent': USER_AGENT_STRING}))
output = json.loads(response_raw.read().decode())
response_raw_2 = requests.post(
(MASTER_SERVER_ADDRESS + '/cloudtoolcmd'),
data={
'c': cmd,
'v': VERSION,
't': json.dumps(self._state.login_token),
'd': json.dumps(data)
},
files=files)
response_raw_2.raise_for_status() # Except if anything went wrong.
assert isinstance(response_raw_2.content, bytes)
output = json.loads(response_raw_2.content.decode())
assert isinstance(output, dict)
assert isinstance(output['m'], (str, type(None)))
assert isinstance(output['e'], (str, type(None)))
assert 'd' in output
response = Response(message=output['m'],
data=output['d'],
error=output['e'])
# Handle errors and messages which are common to all command types.
# Handle errors and print messages;
# (functionality common to all command types).
if response.error is not None:
raise CleanError(response.error)
@ -302,15 +312,42 @@ class App:
path = Path(sys.argv[2])
package = AssetPackage.load_from_disk(path)
# Now send the server a manifest of everything we've got in the local
# package.
# Send the server a manifest of everything we've got locally.
manifest = package.get_manifest()
print('SENDING PACKAGE MANIFEST:', manifest)
response = self._servercmd('putassetmanifest', {'m': manifest})
# The server's response tells us what we need to upload to them.
# Do that.
print('WOULD UPLOAD NEEDED BITS')
# The server should give us an upload id and a set of files it wants.
# Upload each of those.
upload_files: List[str] = response.data['upload_files']
assert isinstance(upload_files, list)
assert all(isinstance(f, str) for f in upload_files)
self._putasset_upload(package, upload_files)
print('Asset upload successful!')
def _putasset_upload(self, package: AssetPackage,
files: List[str]) -> None:
# Upload the files one at a time.
# (we can potentially do this in parallel in the future).
for fnum, fname in enumerate(files):
print(
f'{CLRBLU}Uploading file {fnum+1} of {len(files)}: '
f'"{fname}"...{CLREND}',
flush=True)
with tempfile.TemporaryDirectory() as tempdir:
asset = package.assets[fname]
srcpath = Path(asset.filepath)
gzpath = Path(tempdir, 'file.gz')
subprocess.run(f'gzip --stdout "{srcpath}" > "{gzpath}"',
shell=True,
check=True)
with open(gzpath, 'rb') as infile:
putfiles: Dict = {'file': infile}
_response = self._servercmd('putassetupload',
{'path': asset.path},
files=putfiles)
def do_misc_command(self) -> None:
"""Run a miscellaneous command."""

10
tools/efrotools/pybuild.py
View File

@ -34,10 +34,10 @@ if TYPE_CHECKING:
PYTHON_VERSION_MAJOR = "3.7"
# Specific version we're using on apple builds.
PYTHON_VERSION_APPLE = "3.7.0"
# PYTHON_VERSION_APPLE = "3.7.0"
# Specific version we're using on android builds.
PYTHON_VERSION_ANDROID = "3.7.2"
# PYTHON_VERSION_ANDROID = "3.7.2"
ENABLE_OPENSSL = True
@ -163,7 +163,7 @@ def build_apple(arch: str, debug: bool = False) -> None:
txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.13')
# And equivalent iOS (11+).
txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=7.0',
txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=11.0')
# Ditto for tvOS.
txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
@ -283,7 +283,9 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
efrotools.writefile('pybuild/packages/python.py', ftxt)
# Set this to a particular cpython commit to target exact releases from git
commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release
# commit = 'e09359112e250268eca209355abeb17abf822486' # 3.7.4 release
commit = '5c02a39a0b31a330e06b4d6f44835afb205dc7cc' # 3.7.5 release
if commit is not None:
ftxt = efrotools.readfile('pybuild/source.py')

31
tools/snippets
View File

@ -73,6 +73,18 @@ SPARSE_TESTS: List[List[str]] = [
# (whole word will get subbed out in spinoffs so this will be false)
DO_SPARSE_TESTS = 'ballistica' + 'core' == 'ballisticacore'
# Python modules we require for this project.
# (module name, required version, pip package (if it differs from module name))
REQUIRED_PYTHON_MODULES = [
('pylint', [2, 4, 4], None),
('mypy', [0, 740], None),
('yapf', [0, 29, 0], None),
('typing_extensions', None, None),
('pytz', None, None),
('yaml', None, 'PyYAML'),
('requests', None, None),
]
def archive_old_builds() -> None:
"""Stuff our old public builds into the 'old' dir.
@ -692,6 +704,16 @@ def update_docs_md() -> None:
print(f'{docs_path} is up to date.')
def pip_req_list() -> None:
"""List Python Pip packages needed for this project."""
out: List[str] = []
for module in REQUIRED_PYTHON_MODULES:
name = module[0] if module[2] is None else module[2]
assert isinstance(name, str)
out.append(name)
print(' '.join(out))
def checkenv() -> None:
"""Check for tools necessary to build and run the app."""
print('Checking environment...', flush=True)
@ -711,14 +733,7 @@ def checkenv() -> None:
raise CleanError('pip (for {python_bin}) is required.')
# Check for some required python modules.
for modname, minver, packagename in [
('pylint', [2, 4, 4], None),
('mypy', [0, 740], None),
('yapf', [0, 29, 0], None),
('typing_extensions', None, None),
('pytz', None, None),
('yaml', None, 'PyYAML'),
]:
for modname, minver, packagename in REQUIRED_PYTHON_MODULES:
if packagename is None:
packagename = modname
if minver is not None: