From 59b48f03c68e507ff2bffe52be8a35674f7fcf01 Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Wed, 8 Jan 2020 17:16:01 -0800 Subject: [PATCH] latest cloudtool work --- .idea/dictionaries/ericf.xml | 8 +++++++ tools/cloudtool | 45 +++++++++++++++++++++++++----------- tools/efrotools/pybuild.py | 41 +++++++++++++++++++++++++------- 3 files changed, 72 insertions(+), 22 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index b199ff6b..9d1a8c2b 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -58,6 +58,8 @@ apichanges apis apks + appath + appathout appcfg appconfig appdelegate @@ -109,6 +111,7 @@ autodesk autogenerate autonoassets + autopoint autoremove autoretain autoselect @@ -193,6 +196,7 @@ buttondown buttonwidget bval + bytecount byteswap cachable cachebasename @@ -647,6 +651,7 @@ gametype gametypes gameutils + gbytecount gearvr gendocs gendummymodule @@ -660,6 +665,7 @@ getconf getconfig getcurrency + getcwd getdata getlevelname getmaps @@ -832,6 +838,7 @@ jsonstrbase jsontools jsonutils + kbytecount keepaway keeprefs keylayout @@ -964,6 +971,7 @@ maxval maxw maxwidth + mbytecount mdiv mdocs mdocslines diff --git a/tools/cloudtool b/tools/cloudtool index 770271c3..e654a537 100755 --- a/tools/cloudtool +++ b/tools/cloudtool @@ -94,6 +94,14 @@ class AssetType(Enum): """Types for asset files.""" TEXTURE = 'texture' SOUND = 'sound' + DATA = 'data' + + +ASSET_SOURCE_FILE_EXTS = { + AssetType.TEXTURE: 'png', + AssetType.SOUND: 'wav', + AssetType.DATA: 'yaml', +} class Asset: @@ -103,8 +111,8 @@ class Asset: path: str) -> None: self.assettype = assettype self.path = path - exts = {AssetType.TEXTURE: '.png', AssetType.SOUND: '.wav'} - self.filepath = os.path.join(package.path, path + exts[assettype]) + self.filepath = os.path.join( + package.path, path + '.' + ASSET_SOURCE_FILE_EXTS[assettype]) def get_tz_offset_seconds() -> float: @@ -158,6 +166,7 @@ class AssetPackage: self.assets: Dict[str, Asset] = {} self.path = Path('') self.name = 'untitled' + self.index = '' @classmethod def load_from_disk(cls, path: Path) -> AssetPackage: @@ -170,14 +179,20 @@ class AssetPackage: package.path = path with open(Path(path, indexfilename)) as infile: - index = yaml.safe_load(infile) + package.index = infile.read() + index = yaml.safe_load(package.index) if not isinstance(index, dict): raise CleanError(f'Root dict not found in {indexfilename}') + + # Pull our name from the index file. + # (NOTE: can probably just let the server do this) name = index.get('name') if not isinstance(name, str): raise CleanError(f'No "name" str found in {indexfilename}') validate_asset_package_name(name) package.name = name + + # Build our list of Asset objs from the index. assets = index.get('assets') if not isinstance(assets, dict): raise CleanError(f'No "assets" dict found in {indexfilename}') @@ -191,8 +206,6 @@ class AssetPackage: raise CleanError( f'Invalid asset type for {assetpath} in {indexfilename}') assettype = AssetType(assettypestr) - - # print('Looking at asset:', assetpath, assetdata) package.assets[assetpath] = Asset(package, assettype, assetpath) return package @@ -203,7 +216,7 @@ class AssetPackage: from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count - manifest: Dict = {'name': self.name, 'files': {}} + manifest: Dict = {'name': self.name, 'files': {}, 'index': self.index} def _get_asset_info(iasset: Asset) -> Tuple[Asset, Dict]: sha = hashlib.sha256() @@ -213,7 +226,11 @@ class AssetPackage: sha.update(filebytes) if not os.path.isfile(iasset.filepath): raise Exception(f'Asset file not found: "{iasset.filepath}"') - info_out: Dict = {'hash': sha.hexdigest(), 'size': filesize} + info_out: Dict = { + 'hash': sha.hexdigest(), + 'size': filesize, + 'ext': ASSET_SOURCE_FILE_EXTS[iasset.assettype] + } return iasset, info_out # Use all procs to hash files for extra speedy goodness. @@ -250,8 +267,9 @@ class App: self._load_cache() if len(sys.argv) < 2: - raise CleanError( - f'Invalid args. Run "cloudtool help" for usage info.') + print(f'{CLRRED}You must provide one or more arguments.{CLREND}') + self.do_misc_command(['help']) + raise CleanError() cmd = sys.argv[1] if cmd == CMD_LOGIN: @@ -263,7 +281,7 @@ class App: self.do_assetpack_put() else: # For all other commands, simply pass them to the server verbatim. - self.do_misc_command() + self.do_misc_command(sys.argv[1:]) self._save_cache() @@ -373,7 +391,6 @@ class App: _response = self._servercmd('assetpackputfinish', { 'packageversion': version, }) - # print(f'{CLRGRN}Created asset package: {version}{CLREND}') def _assetpack_put_upload(self, package: AssetPackage, version: str, files: List[str]) -> None: @@ -383,7 +400,7 @@ class App: for fnum, fname in enumerate(files): print( f'{CLRBLU}Uploading file {fnum+1} of {len(files)}: ' - f'"{fname}"...{CLREND}', + f'{fname}{CLREND}', flush=True) with tempfile.TemporaryDirectory() as tempdir: asset = package.assets[fname] @@ -403,12 +420,12 @@ class App: files=putfiles, ) - def do_misc_command(self) -> None: + def do_misc_command(self, args: List[str]) -> None: """Run a miscellaneous command.""" # We don't do anything special with the response here; the normal # error-handling/message-printing is all that happens. - self._servercmd('misc', {'a': sys.argv[1:]}) + self._servercmd('misc', {'a': args}) if __name__ == '__main__': diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py index 1a2fbc03..84079955 100644 --- a/tools/efrotools/pybuild.py +++ b/tools/efrotools/pybuild.py @@ -215,6 +215,20 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: '"') os.chdir(builddir) + # It seems we now need 'autopoint' as part of this build, but on mac it + # is not available on the normal path, but only as part of the keg-only + # gettext homebrew formula. + if (subprocess.run('which autopoint', shell=True, check=False).returncode + != 0): + print("Updating path for mac autopoint...") + appath = subprocess.run('brew ls gettext | grep bin/autopoint', + shell=True, + check=True, + capture_output=True) + appathout = os.path.dirname(appath.stdout.decode().strip()) + os.environ['PATH'] += (':' + appathout) + print(f'ADDED "{appathout}" TO SYS PATH...') + # Commit from Dec 6th, 2018. Looks like right after this one the repo # switched to ndk r19 beta 2 and now seems to require r19, so we can # try switching back to master one r19 comes down the pipe. @@ -265,8 +279,8 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: "'./configure', '--with-pydebug',") # We don't use this stuff so lets strip it out to simplify. - ftxt = efrotools.replace_one(ftxt, "'--with-system-ffi',", "") - ftxt = efrotools.replace_one(ftxt, "'--with-system-expat',", "") + # ftxt = efrotools.replace_one(ftxt, "'--with-system-ffi',", "") + # ftxt = efrotools.replace_one(ftxt, "'--with-system-expat',", "") ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", "") # This builds all modules as dynamic libs, but we want to be consistent @@ -274,11 +288,17 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: # need... so to change that we'll need to add a hook for ourself after # python is downloaded but before it is built so we can muck with it. ftxt = efrotools.replace_one( - ftxt, ' def prepare(self):', - ' def prepare(self):\n import os\n' + ftxt, ' def build(self):', + ' def build(self):\n import os\n' ' if os.system(\'"' + rootdir + '/tools/snippets" python_android_patch "' + os.getcwd() + '"\') != 0: raise Exception("patch apply failed")') + # ftxt = efrotools.replace_one( + # ftxt, ' def prepare(self):', + # ' def prepare(self):\n import os\n' + # ' if os.system(\'"' + rootdir + + # '/tools/snippets" python_android_patch "' + os.getcwd() + + # '"\') != 0: raise Exception("patch apply failed")') efrotools.writefile('pybuild/packages/python.py', ftxt) @@ -291,10 +311,15 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: # Check out a particular commit right after the clone. ftxt = efrotools.replace_one( - ftxt, - "'git', 'clone', '-b', self.branch, self.source_url, self.dest])", - "'git', 'clone', '-b', self.branch, self.source_url, self.dest])\n" - " run_in_dir(['git', 'checkout', '" + commit + + ftxt, "'git', 'clone', '--single-branch', '-b'," + " self.branch, self.source_url, self.dest])", + "'git', 'clone', '-b'," + " self.branch, self.source_url, self.dest])\n" + " # efro: hack to get the python we want.\n" + " print('DOING URL', self.source_url)\n" + " if self.source_url == " + "'https://github.com/python/cpython/':\n" + " run_in_dir(['git', 'checkout', '" + commit + "'], self.source_dir)") efrotools.writefile('pybuild/source.py', ftxt) ftxt = efrotools.readfile('pybuild/util.py')