diff --git a/.efrocachemap b/.efrocachemap index ae607fc4..72d22bcb 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -420,8 +420,8 @@ "assets/build/ba_data/audio/zoeOw.ogg": "https://files.ballistica.net/cache/ba1/75/1d/868bb04cf691736035c917d02762", "assets/build/ba_data/audio/zoePickup01.ogg": "https://files.ballistica.net/cache/ba1/44/2a/8535b446284235cb503947ece074", "assets/build/ba_data/audio/zoeScream01.ogg": "https://files.ballistica.net/cache/ba1/f5/d3/8e941851c4310465646c4167afc1", - "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/de/c0/7197e7658bef48309741dfaf86c7", - "assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/30/8e/38c5f21b9251ea1111bed554035f", + "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/cd/7a/dcf7f0f9436884167abdcb126716", + "assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/a9/3c/63857b3f5f943205d7a5d8f8e476", "assets/build/ba_data/data/languages/belarussian.json": "https://files.ballistica.net/cache/ba1/49/5f/b29bb65369040892fe6601801637", "assets/build/ba_data/data/languages/chinese.json": "https://files.ballistica.net/cache/ba1/ea/14/6485c200c717f82cca7c01dee0b3", "assets/build/ba_data/data/languages/chinesetraditional.json": "https://files.ballistica.net/cache/ba1/a0/ca/b4e3c4ea2c76e462b7e657b2b1c2", @@ -436,22 +436,22 @@ "assets/build/ba_data/data/languages/gibberish.json": "https://files.ballistica.net/cache/ba1/cb/10/5d94df639e3e0cb405711e1b907f", "assets/build/ba_data/data/languages/greek.json": "https://files.ballistica.net/cache/ba1/17/78/3fd0dca40e632ce53d03a944e7fa", "assets/build/ba_data/data/languages/hindi.json": "https://files.ballistica.net/cache/ba1/11/56/ed2b07866104596338f7ce582d64", - "assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/4d/4b/0790110201c9adb1b521e9a55e63", + "assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/87/2d/027aa239eb66ea8f496562f4fd83", "assets/build/ba_data/data/languages/indonesian.json": "https://files.ballistica.net/cache/ba1/a3/b6/8987b9fc6a7cb0f61364f567c901", "assets/build/ba_data/data/languages/italian.json": "https://files.ballistica.net/cache/ba1/ff/3f/f1c11e1f22454ceceb7b4d277a2a", "assets/build/ba_data/data/languages/korean.json": "https://files.ballistica.net/cache/ba1/0a/84/bbb6ed2abf66509406f534cbbb52", "assets/build/ba_data/data/languages/persian.json": "https://files.ballistica.net/cache/ba1/bd/45/ffb2d9d9ce9578ae11de9bb5123d", "assets/build/ba_data/data/languages/polish.json": "https://files.ballistica.net/cache/ba1/e1/cc/13a4ba409b7d6f113f703be1aa8a", - "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/9c/e3/834faf6869f5cd175edd4a0244b9", + "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/e5/0a/6ee92ae2cb50f12eeb21059c6659", "assets/build/ba_data/data/languages/romanian.json": "https://files.ballistica.net/cache/ba1/44/3c/7cc06ca8d5475e1687d0ed05bdbf", "assets/build/ba_data/data/languages/russian.json": "https://files.ballistica.net/cache/ba1/57/bb/33e8bd738c3ee97122186471be75", "assets/build/ba_data/data/languages/serbian.json": "https://files.ballistica.net/cache/ba1/e7/d8/ace32888249fc8b8cca0e2edb48b", "assets/build/ba_data/data/languages/slovak.json": "https://files.ballistica.net/cache/ba1/b7/0a/fab820b96e7aa587ee56427ecdc2", - "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/c9/6b/dadd4329de69d0b7bfe1aa5f31d6", + "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/73/ef/624000057ce55f5a511ba18fd433", "assets/build/ba_data/data/languages/swedish.json": "https://files.ballistica.net/cache/ba1/50/9f/be006ba19be6a69a57837eb6dca0", "assets/build/ba_data/data/languages/turkish.json": "https://files.ballistica.net/cache/ba1/82/7c/5363de81ac6823935098df95dc66", "assets/build/ba_data/data/languages/ukrainian.json": "https://files.ballistica.net/cache/ba1/9d/fb/6c2feb78bdabbec049a71ba40d78", - "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/f1/00/c4ccd5969084505359e07f927b3a", + "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/96/96/b2e468ceec8800b7b0f1e1c5977c", "assets/build/ba_data/data/maps/big_g.json": "https://files.ballistica.net/cache/ba1/47/0a/a617cc85d927b576c4e6fc1091ed", "assets/build/ba_data/data/maps/bridgit.json": "https://files.ballistica.net/cache/ba1/03/4b/57ee9b42854b26f23f81bd8c58ef", "assets/build/ba_data/data/maps/courtyard.json": "https://files.ballistica.net/cache/ba1/03/38/344dd05bfef7bbdf464035ec5aa2", @@ -4135,16 +4135,16 @@ "assets/build/windows/x64/vc_redist.x64.exe": "https://files.ballistica.net/cache/ba1/ea/19/8b8787d81abcdce158ba608cd24f", "assets/build/windows/x64/vcruntime140_1d.dll": "https://files.ballistica.net/cache/ba1/11/d8/ff6344b429b00c24d9a1930d4338", "assets/build/windows/x64/vcruntime140d.dll": "https://files.ballistica.net/cache/ba1/20/33/0825e11e6518f87ece3009309933", - "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b2/16/0414b009ce117f73bb2bc8991d3b", - "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/66/07/d82a3ec80ff3243321e373439210", - "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/12/5f/f87158c3281a2616897fdd2165cb", - "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/69/23/a2d029f81db60cf89ad5f6510ddb", - "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c4/e3/5aba369252ed1977be8950e5a1ce", - "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/f7/ef/c28554f863485e9cd54f0d8d4e6f", - "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/68/af/f060b3463bc31c202a830fb0b20a", - "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/96/46/a239821d00e782303077192817c5", - "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/dc/15/fdd9195afe2bf299405ec49ed238", - "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/68/f9/c42b1f9b5af34cb371f80250e16b", - "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5d/9b/213c602a9aa9cbc28872d2753f51", - "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/2f/a4/e6a6a6e0e2b64c5232cccddffdd3" + "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/05/00/2dd9723909746d8039c6bdf8cddb", + "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/3e/62/6c310233f4b2c224a46914a4b78e", + "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/0c/f8/d381993bd58c1f50f0da6d8e2be3", + "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/a5/b5/d2ec28b09124404731b7f08008c8", + "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/cb/82/4520a3fec142215ed0673a1811d9", + "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b9/0b/060333370a987cee92bdd081a8ca", + "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/11/31/9f1bb364db47563a1d86f0165549", + "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/11/62/4d784f02705e438f9bd519ff53f4", + "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/8a/ff/abcf2602dc64f718408efeaf4d3a", + "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/04/2d/1f8c7ea677bae4cc7be8e967c344", + "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/26/b8/f19e24bd0e43ed9661ca62c10e6c", + "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/02/0b/d2fab33376bc7ae1c8797cae5c0f" } \ No newline at end of file diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 9bcbca70..d45094c3 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -1613,6 +1613,7 @@ pythonw pytree pytz + pyver qrcode qrencode qual @@ -1788,6 +1789,7 @@ simplesubclasses sincelaunch singledispatch + singledispatchmethod sirplus sitebuiltins skey diff --git a/CHANGELOG.md b/CHANGELOG.md index e7a1696f..8289ca3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +### 1.5.24 (20159) +- Misc bug fixes.. + ### 1.5.23 (20146) - Fixed the shebang line in `bombsquad_server` file by using `-S` flag for `/usr/bin/env`. - Fixed a bug with hardware keyboards emitting extra characters in the in-game console (~ or F2) diff --git a/assets/src/ba_data/python/ba/_netutils.py b/assets/src/ba_data/python/ba/_netutils.py index c414d7a4..72cf8dfe 100644 --- a/assets/src/ba_data/python/ba/_netutils.py +++ b/assets/src/ba_data/python/ba/_netutils.py @@ -56,7 +56,7 @@ def get_ip_address_type(addr: str) -> socket.AddressFamily: except OSError: pass if socket_type is None: - raise ValueError('addr seems to be neither v4 or v6: ' + str(addr)) + raise ValueError(f'addr seems to be neither v4 or v6: {addr}') return socket_type @@ -152,8 +152,10 @@ class ServerCallThread(threading.Thread): response_data = None # Ignore common network errors; note unexpected ones. - if isinstance(exc, (urllib.error.URLError, ConnectionError, - http.client.IncompleteRead)): + if isinstance( + exc, + (urllib.error.URLError, ConnectionError, + http.client.IncompleteRead, http.client.BadStatusLine)): pass elif isinstance(exc, OSError): if exc.errno == 10051: # Windows unreachable network error. diff --git a/assets/src/ba_data/python/bastd/actor/bomb.py b/assets/src/ba_data/python/bastd/actor/bomb.py index fd4fa485..4218666a 100644 --- a/assets/src/ba_data/python/bastd/actor/bomb.py +++ b/assets/src/ba_data/python/bastd/actor/bomb.py @@ -887,10 +887,11 @@ class Bomb(ba.Actor): def _handle_impact(self) -> None: node = ba.getcollision().opposingnode - # If we're an impact bomb and we came from this node, don't explode... - # alternately if we're hitting another impact-bomb from the same - # source, don't explode... - # try: + # If we're an impact bomb and we came from this node, don't explode. + # (otherwise we blow up on our own head when jumping). + # Alternately if we're hitting another impact-bomb from the same + # source, don't explode. (can cause accidental explosions if rapidly + # throwing/etc.) node_delegate = node.getdelegate(object) if node: if (self.bomb_type == 'impact' and diff --git a/assets/src/ba_data/python/bastd/ui/gather.py b/assets/src/ba_data/python/bastd/ui/gather.py index 5df5bf23..e98d6e96 100644 --- a/assets/src/ba_data/python/bastd/ui/gather.py +++ b/assets/src/ba_data/python/bastd/ui/gather.py @@ -1673,7 +1673,7 @@ class GatherWindow(ba.Window): # Ignore harmless errors. if exc.errno in { errno.EHOSTUNREACH, errno.ENETUNREACH, - errno.EINVAL, errno.EPERM + errno.EINVAL, errno.EPERM, errno.EACCES }: pass elif exc.errno == 10022: diff --git a/assets/src/ba_data/python/bastd/ui/settings/gamepad.py b/assets/src/ba_data/python/bastd/ui/settings/gamepad.py index bdc9d4d9..fdfe2514 100644 --- a/assets/src/ba_data/python/bastd/ui/settings/gamepad.py +++ b/assets/src/ba_data/python/bastd/ui/settings/gamepad.py @@ -838,7 +838,8 @@ class AwaitGamepadInputWindow(ba.Window): assert isinstance(input_device, ba.InputDevice) # Update - we now allow *any* input device of this type. - if input_device.exists() and input_device.name == self._input.name: + if (self._input and input_device + and input_device.name == self._input.name): self._callback(self._capture_button, event, self) def _decrement(self) -> None: diff --git a/assets/src/ba_data/python/bastd/ui/settings/keyboard.py b/assets/src/ba_data/python/bastd/ui/settings/keyboard.py index 8ed9fc65..c68e5e3c 100644 --- a/assets/src/ba_data/python/bastd/ui/settings/keyboard.py +++ b/assets/src/ba_data/python/bastd/ui/settings/keyboard.py @@ -236,6 +236,11 @@ class ConfigKeyboardWindow(ba.Window): ba.containerwidget(edit=self._root_widget, transition='out_right') ba.playsound(ba.getsound('gunCocking')) + + # There's a chance the device disappeared; handle that gracefully. + if not self._input: + return + dst = get_input_device_config(self._input, default=False) dst2: Dict[str, Any] = dst[0][dst[1]] dst2.clear() diff --git a/docs/ba_module.md b/docs/ba_module.md index 2c401f48..b8d91550 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,5 +1,5 @@ -

last updated on 2020-07-22 for Ballistica version 1.5.23 build 20152

+

last updated on 2020-07-26 for Ballistica version 1.5.24 build 20158

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!


diff --git a/tools/batools/assetstaging.py b/tools/batools/assetstaging.py index 1fa0ed07..8a281d81 100755 --- a/tools/batools/assetstaging.py +++ b/tools/batools/assetstaging.py @@ -30,7 +30,7 @@ import subprocess from functools import partial from typing import TYPE_CHECKING -from efrotools.pybuild import PYTHON_VERSION_MAJOR +from efrotools import PYVER if TYPE_CHECKING: from typing import Optional, List @@ -40,8 +40,7 @@ if TYPE_CHECKING: # Note: this means anyone wanting to modify .py files in a build # will need to wipe out the existing .pyc files first or the changes # will be ignored. -OPT_PYC_SUFFIX = ('cpython-' + PYTHON_VERSION_MAJOR.replace('.', '') + - '.opt-1.pyc') +OPT_PYC_SUFFIX = ('cpython-' + PYVER.replace('.', '') + '.opt-1.pyc') class Config: diff --git a/tools/batools/pcommand.py b/tools/batools/pcommand.py index b9248d75..a6cc67a0 100644 --- a/tools/batools/pcommand.py +++ b/tools/batools/pcommand.py @@ -43,6 +43,7 @@ def stage_server_file() -> None: import batools.build from efro.error import CleanError from efrotools import replace_one + from efrotools import PYVER if len(sys.argv) != 5: raise CleanError('Expected 3 args (mode, infile, outfile).') mode, infilename, outfilename = sys.argv[2], sys.argv[3], sys.argv[4] @@ -56,13 +57,15 @@ def stage_server_file() -> None: # Inject all available config values into the config file. batools.build.filter_server_config(str(PROJROOT), infilename, outfilename) + elif basename == 'ballisticacore_server.py': # Run Python in opt mode for release builds. with open(infilename) as infile: lines = infile.read().splitlines() if mode == 'release': - lines[0] = replace_one(lines[0], '#!/usr/bin/env python3.7', - '#!/usr/bin/env -S python3.7 -O') + lines[0] = replace_one(lines[0], + f'#!/usr/bin/env python{PYVER}', + f'#!/usr/bin/env -S python{PYVER} -O') with open(outfilename, 'w') as outfile: outfile.write('\n'.join(lines) + '\n') subprocess.run(['chmod', '+x', outfilename], check=True) diff --git a/tools/batools/updateproject.py b/tools/batools/updateproject.py index d1aede13..8b6cdb39 100755 --- a/tools/batools/updateproject.py +++ b/tools/batools/updateproject.py @@ -377,7 +377,7 @@ class Updater: def _check_python_file(self, fname: str) -> None: # pylint: disable=too-many-branches - from efrotools import get_public_license + from efrotools import get_public_license, PYVER with open(fname) as infile: contents = infile.read() lines = contents.splitlines() @@ -390,7 +390,7 @@ class Updater: if fname not in [ 'tools/devtool', 'tools/version_utils', 'tools/vmshell' ]: - if not contents.startswith('#!/usr/bin/env python3.7'): + if not contents.startswith(f'#!/usr/bin/env python{PYVER}'): print(f'{Clr.RED}Incorrect shebang (first line) for ' f'{fname}.{Clr.RST}') sys.exit(255) diff --git a/tools/efro/util.py b/tools/efro/util.py index 5ce9a3c3..906f17d1 100644 --- a/tools/efro/util.py +++ b/tools/efro/util.py @@ -115,9 +115,10 @@ def dispatchmethod( # All we do here is recreate the end of functools.singledispatch # where it returns a wrapper except instead of the wrapper using the # first arg to the function ours uses the second (to skip 'self'). - # This was made with Python 3.7; we should probably check up on + # This was made against Python 3.7; we should probably check up on # this in later versions in case anything has changed. # (or hopefully they'll add this functionality to their version) + # NOTE: sounds like we can use functools singledispatchmethod in 3.8 def wrapper(*args: Any, **kw: Any) -> Any: if not args or len(args) < 2: raise TypeError(f'{funcname} requires at least ' diff --git a/tools/efrotools/__init__.py b/tools/efrotools/__init__.py index b89d640b..51c93b35 100644 --- a/tools/efrotools/__init__.py +++ b/tools/efrotools/__init__.py @@ -40,8 +40,11 @@ if TYPE_CHECKING: from typing import Dict, Union, Sequence, Optional, Any from typing_extensions import Literal +# Python major version we're using for all this stuff. +PYVER = '3.7' + # Python binary assumed by these tools. -PYTHON_BIN = 'python3.7' if platform.system() != 'Windows' else 'python' +PYTHON_BIN = f'python{PYVER}' if platform.system() != 'Windows' else 'python' MIT_LICENSE = """Copyright (c) 2011-2020 Eric Froemling diff --git a/tools/efrotools/code.py b/tools/efrotools/code.py index 28dfd52b..11c999b1 100644 --- a/tools/efrotools/code.py +++ b/tools/efrotools/code.py @@ -86,7 +86,7 @@ def cpplint(projroot: Path, full: bool) -> None: import tempfile from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count - from efrotools import getconfig + from efrotools import getconfig, PYVER from efro.terminal import Clr from efro.error import CleanError @@ -150,7 +150,8 @@ def cpplint(projroot: Path, full: bool) -> None: def lint_file(filename: str) -> None: result = subprocess.call( - ['python3.7', '-m', 'cpplint', '--root=src', filename], env=env) + [f'python{PYVER}', '-m', 'cpplint', '--root=src', filename], + env=env) if result != 0: raise CleanError( f'{Clr.RED}Cpplint failed for {filename}.{Clr.RST}') @@ -197,7 +198,7 @@ def formatscripts(projroot: Path, full: bool) -> None: import time from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count - from efrotools import get_files_hash + from efrotools import get_files_hash, PYVER os.chdir(projroot) cachepath = Path(projroot, 'config/.cache-formatscripts') if full and cachepath.exists(): @@ -215,7 +216,7 @@ def formatscripts(projroot: Path, full: bool) -> None: def format_file(filename: str) -> None: start_time = time.time() result = subprocess.call( - ['python3.7', '-m', 'yapf', '--in-place', filename]) + [f'python{PYVER}', '-m', 'yapf', '--in-place', filename]) if result != 0: raise Exception(f'Formatting failed for {filename}') duration = time.time() - start_time diff --git a/tools/efrotools/pcommand.py b/tools/efrotools/pcommand.py index 5609ec37..6af449ec 100644 --- a/tools/efrotools/pcommand.py +++ b/tools/efrotools/pcommand.py @@ -174,6 +174,12 @@ def spelling() -> None: _spelling(sys.argv[2:]) +def pyver() -> None: + """Prints the Python version used by this project.""" + from efrotools import PYVER + print(PYVER, end='') + + def check_clean_safety() -> None: """Ensure all files are are added to git or in gitignore. diff --git a/tools/efrotools/pybuild.py b/tools/efrotools/pybuild.py index e7362d69..6f32f418 100644 --- a/tools/efrotools/pybuild.py +++ b/tools/efrotools/pybuild.py @@ -25,14 +25,11 @@ from __future__ import annotations import os from typing import TYPE_CHECKING -import efrotools +from efrotools import PYVER, run, readfile, writefile, replace_one if TYPE_CHECKING: from typing import List, Dict, Any -# Overall version we're using for the game currently. -PYTHON_VERSION_MAJOR = '3.7' - ENABLE_OPENSSL = True @@ -51,23 +48,22 @@ def build_apple(arch: str, debug: bool = False) -> None: raise CleanError('NEED TO TEMP-KILL GETTEXT') builddir = 'build/python_apple_' + arch + ('_debug' if debug else '') - efrotools.run('rm -rf "' + builddir + '"') - efrotools.run('mkdir -p build') - efrotools.run('git clone ' - 'git@github.com:pybee/Python-Apple-support.git "' + - builddir + '"') + run('rm -rf "' + builddir + '"') + run('mkdir -p build') + run('git clone ' + 'git@github.com:pybee/Python-Apple-support.git "' + builddir + '"') os.chdir(builddir) # TEMP: Check out a particular commit while the branch head is broken. # We can actually fix this to use the current one, but something # broke in the underlying build even on old commits so keeping it # locked for now... - # efrotools.run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6') - efrotools.run(f'git checkout {PYTHON_VERSION_MAJOR}') + # run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6') + run(f'git checkout {PYVER}') # On mac we currently have to add the _scproxy module or urllib will # fail. - txt = efrotools.readfile('patch/Python/Setup.embedded') + txt = readfile('patch/Python/Setup.embedded') if arch == 'mac': txt += ('\n' '# ericf added - mac urllib needs this\n' @@ -76,19 +72,19 @@ def build_apple(arch: str, debug: bool = False) -> None: '-framework CoreFoundation') # Turn off sqlite module. (scratch that; leaving it in.) - # txt = efrotools.replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(') + # txt = replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(') # txt = txt.replace(' _sqlite/', '# _sqlite/') # Turn off xz compression module. (scratch that; leaving it in.) - # txt = efrotools.replace_one(txt, '_lzma _', '#_lzma _') + # txt = replace_one(txt, '_lzma _', '#_lzma _') # Turn off bzip2 module. - txt = efrotools.replace_one(txt, '_bz2 _b', '#_bz2 _b') + txt = replace_one(txt, '_bz2 _b', '#_bz2 _b') # Turn off openssl module (only if not doing openssl). if not ENABLE_OPENSSL: - txt = efrotools.replace_one(txt, '_hashlib _hashopenssl.c', - '#_hashlib _hashopenssl.c') + txt = replace_one(txt, '_hashlib _hashopenssl.c', + '#_hashlib _hashopenssl.c') # Turn off various other stuff we don't use. for line in [ @@ -123,7 +119,7 @@ def build_apple(arch: str, debug: bool = False) -> None: '_testimportmultiple _testimportmultiple.c', '_crypt _cryptmodule.c', # not on android so disabling here too ]: - txt = efrotools.replace_one(txt, line, '#' + line) + txt = replace_one(txt, line, '#' + line) if ENABLE_OPENSSL: @@ -138,28 +134,28 @@ def build_apple(arch: str, debug: bool = False) -> None: '_sha256 sha256module.c', '_sha512 sha512module.c', ]: - txt = efrotools.replace_one(txt, line, '#' + line) + txt = replace_one(txt, line, '#' + line) else: - txt = efrotools.replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c') - efrotools.writefile('patch/Python/Setup.embedded', txt) + txt = replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c') + writefile('patch/Python/Setup.embedded', txt) - txt = efrotools.readfile('Makefile') + txt = readfile('Makefile') # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!) - txt = efrotools.replace_one( + txt = replace_one( txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .', '&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .') # Remove makefile dependencies so we don't build the # libs we're not using. srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' - txt = efrotools.replace_one( + txt = replace_one( txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' + ('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') + 'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt) srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.' '$(BUILD_NUMBER).tar.gz: ') - txt = efrotools.replace_one( + txt = replace_one( txt, srctxt, 'dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz:' ' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt) @@ -169,14 +165,14 @@ def build_apple(arch: str, debug: bool = False) -> None: # Set mac/ios version reqs # (see issue with utimensat and futimens). - txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', - 'MACOSX_DEPLOYMENT_TARGET=10.14') + txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', + 'MACOSX_DEPLOYMENT_TARGET=10.14') # And equivalent iOS (11+). - txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', - 'CFLAGS-iOS=-mios-version-min=12.0') + txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', + 'CFLAGS-iOS=-mios-version-min=12.0') # Ditto for tvOS. - txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', - 'CFLAGS-tvOS=-mtvos-version-min=12.0') + txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', + 'CFLAGS-tvOS=-mtvos-version-min=12.0') if debug: @@ -196,13 +192,13 @@ def build_apple(arch: str, debug: bool = False) -> None: raise RuntimeError(f'Unexpected configure line count {splitlen}.') txt = txt.replace(dline, 'python$(PYTHON_VER)dm') - efrotools.writefile('Makefile', txt) + writefile('Makefile', txt) # Ok; let 'er rip. # (we run these in parallel so limit to 1 job a piece; # otherwise they inherit the -j12 or whatever from the top level) # (also this build seems to fail with multiple threads) - efrotools.run('make -j1 ' + { + run('make -j1 ' + { 'mac': 'Python-macOS', 'ios': 'Python-iOS', 'tvos': 'Python-tvOS' @@ -217,11 +213,10 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: """ import subprocess builddir = 'build/python_android_' + arch + ('_debug' if debug else '') - efrotools.run('rm -rf "' + builddir + '"') - efrotools.run('mkdir -p build') - efrotools.run('git clone ' - 'git@github.com:yan12125/python3-android.git "' + builddir + - '"') + run('rm -rf "' + builddir + '"') + run('mkdir -p build') + run('git clone ' + 'git@github.com:yan12125/python3-android.git "' + builddir + '"') os.chdir(builddir) # It seems we now need 'autopoint' as part of this build, but on mac it @@ -242,69 +237,69 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: # a a completely new minimal one which will take some work to update here. # Punting on that for now... if True: # pylint: disable=using-constant-test - efrotools.run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae') - ftxt = efrotools.readfile('pybuild/env.py') + run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae') + ftxt = readfile('pybuild/env.py') # Set the packages we build. - ftxt = efrotools.replace_one( + ftxt = replace_one( ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," + (" 'openssl'" if ENABLE_OPENSSL else '') + ')\n# packages = (') # Don't wanna bother with gpg signing stuff. - ftxt = efrotools.replace_one(ftxt, 'verify_source = True', - 'verify_source = False') + ftxt = replace_one(ftxt, 'verify_source = True', 'verify_source = False') # Sub in the min api level we're targeting. - ftxt = efrotools.replace_one(ftxt, 'android_api_level = 21', - 'android_api_level = 21') - ftxt = efrotools.replace_one(ftxt, "target_arch = 'arm'", - "target_arch = '" + arch + "'") - efrotools.writefile('pybuild/env.py', ftxt) - ftxt = efrotools.readfile('Makefile') + ftxt = replace_one(ftxt, 'android_api_level = 21', + 'android_api_level = 21') + ftxt = replace_one(ftxt, "target_arch = 'arm'", + "target_arch = '" + arch + "'") + writefile('pybuild/env.py', ftxt) + ftxt = readfile('Makefile') # This needs to be python3 for us. - ftxt = efrotools.replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n') - efrotools.writefile('Makefile', ftxt) - ftxt = efrotools.readfile('pybuild/packages/python.py') + ftxt = replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n') + writefile('Makefile', ftxt) + ftxt = readfile('pybuild/packages/python.py') # We currently build as a static lib. - ftxt = efrotools.replace_one(ftxt, " '--enable-shared',\n", '') - ftxt = efrotools.replace_one( + ftxt = replace_one(ftxt, " '--enable-shared',\n", '') + ftxt = replace_one( ftxt, "super().__init__('https://github.com/python/cpython/')", - "super().__init__('https://github.com/python/cpython/', branch='3.7')") + "super().__init__('https://github.com/python/cpython/'" + f", branch='{PYVER}')") # Turn ipv6 on (curious why its turned off here?...) # Also, turn on low level debugging features for our debug builds. - ftxt = efrotools.replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',") + ftxt = replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',") if debug: - ftxt = efrotools.replace_one(ftxt, "'./configure',", - "'./configure', '--with-pydebug',") + ftxt = replace_one(ftxt, "'./configure',", + "'./configure', '--with-pydebug',") # We don't use this stuff so lets strip it out to simplify. - ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", '') + ftxt = replace_one(ftxt, "'--without-ensurepip',", '') # This builds all modules as dynamic libs, but we want to be consistent # with our other embedded builds and just static-build the ones we # need... so to change that we'll need to add a hook for ourself after # python is downloaded but before it is built so we can muck with it. - ftxt = efrotools.replace_one( + ftxt = replace_one( ftxt, ' def build(self):', ' def build(self):\n import os\n' ' if os.system(\'"' + rootdir + '/tools/pcommand" python_android_patch "' + os.getcwd() + '"\') != 0: raise Exception("patch apply failed")') - efrotools.writefile('pybuild/packages/python.py', ftxt) + writefile('pybuild/packages/python.py', ftxt) # Set this to a particular cpython commit to target exact releases from git # commit = 'd7c567b08f9d7d6aef21b881340a2b72731129db' # 3.7.7 release commit = '4b47a5b6ba66b02df9392feb97b8ead916f8c1fa' # 3.7.8 release if commit is not None: - ftxt = efrotools.readfile('pybuild/source.py') + ftxt = readfile('pybuild/source.py') # Check out a particular commit right after the clone. - ftxt = efrotools.replace_one( + ftxt = replace_one( ftxt, "'git', 'clone', '--single-branch', '-b'," ' self.branch, self.source_url, self.dest])', "'git', 'clone', '-b'," @@ -315,16 +310,16 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: "'https://github.com/python/cpython/':\n" " run_in_dir(['git', 'checkout', '" + commit + "'], self.source_dir)") - efrotools.writefile('pybuild/source.py', ftxt) - ftxt = efrotools.readfile('pybuild/util.py') + writefile('pybuild/source.py', ftxt) + ftxt = readfile('pybuild/util.py') # Still don't wanna bother with gpg signing stuff. - ftxt = efrotools.replace_one( + ftxt = replace_one( ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n', 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n' ' print("gpg-verify disabled by ericf")\n' ' return\n') - efrotools.writefile('pybuild/util.py', ftxt) + writefile('pybuild/util.py', ftxt) # These builds require ANDROID_NDK to be set, so make sure that's # the case. @@ -335,18 +330,18 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None: # Ok, let 'er rip # (we often run these builds in parallel so limit to 1 job a piece; # otherwise they each inherit the -j12 or whatever from the top level). - efrotools.run('make -j1') + run('make -j1') print('python build complete! (android/' + arch + ')') def android_patch() -> None: """Run necessary patches on an android archive before building.""" fname = 'src/cpython/Modules/Setup.dist' - txt = efrotools.readfile(fname) + txt = readfile(fname) # Need to switch some flags on this one. - txt = efrotools.replace_one(txt, '#zlib zlibmodule.c', - 'zlib zlibmodule.c -lz\n#zlib zlibmodule.c') + txt = replace_one(txt, '#zlib zlibmodule.c', + 'zlib zlibmodule.c -lz\n#zlib zlibmodule.c') # Just turn all these on. for enable in [ '#array arraymodule.c', '#cmath cmathmodule.c _math.c', @@ -362,10 +357,10 @@ def android_patch() -> None: '#binascii binascii.c', '#_posixsubprocess _posixsubprocess.c', '#_sha3 _sha3/sha3module.c' ]: - txt = efrotools.replace_one(txt, enable, enable[1:]) + txt = replace_one(txt, enable, enable[1:]) if ENABLE_OPENSSL: - txt = efrotools.replace_one(txt, '#_ssl _ssl.c \\', - '_ssl _ssl.c -DUSE_SSL -lssl -lcrypto') + txt = replace_one(txt, '#_ssl _ssl.c \\', + '_ssl _ssl.c -DUSE_SSL -lssl -lcrypto') else: # Note that the _md5 and _sha modules are normally only built if the # system does not have the OpenSSL libs containing an optimized @@ -374,11 +369,10 @@ def android_patch() -> None: '#_md5 md5module.c', '#_sha1 sha1module.c', '#_sha256 sha256module.c', '#_sha512 sha512module.c' ]: - txt = efrotools.replace_one(txt, enable, enable[1:]) + txt = replace_one(txt, enable, enable[1:]) # Turn this off (its just an example module). - txt = efrotools.replace_one(txt, 'xxsubtype xxsubtype.c', - '#xxsubtype xxsubtype.c') + txt = replace_one(txt, 'xxsubtype xxsubtype.c', '#xxsubtype xxsubtype.c') # For whatever reason this stuff isn't in there at all; add it. txt += '\n_json _json.c\n' @@ -403,7 +397,7 @@ def android_patch() -> None: txt += '\n\n*disabled*\n_ctypes _crypt grp' - efrotools.writefile(fname, txt) + writefile(fname, txt) # Ok, this is weird. # When applying the module Setup, python looks for any line containing *=* @@ -412,17 +406,16 @@ def android_patch() -> None: # The check used to look for [A-Z]*=* which didn't break, so let' just # change it back to that for now. fname = 'src/cpython/Modules/makesetup' - txt = efrotools.readfile(fname) - txt = efrotools.replace_one( - txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;', - ' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;') - efrotools.writefile(fname, txt) + txt = readfile(fname) + txt = replace_one(txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;', + ' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;') + writefile(fname, txt) # Add custom callbacks to Python's PyParser_ParseFileObject # and PyParser_ParseString calls to debug a crash. fname = 'src/cpython/Parser/parsetok.c' - txt = efrotools.readfile(fname) - txt = efrotools.replace_one( + txt = readfile(fname) + txt = replace_one( txt, 'node *\n' 'PyParser_ParseFileObject(FILE *fp, PyObject *filename,\n' ' const char *enc, grammar *g, int start,\n' @@ -444,7 +437,7 @@ def android_patch() -> None: ' start, ps1, ps2,\n' ' err_ret, flags);\n' ' }\n') - txt = efrotools.replace_one( + txt = replace_one( txt, 'node *\n' 'PyParser_ParseStringObject(const char *s, PyObject *filename,\n' ' grammar *g, int start,\n' @@ -462,7 +455,7 @@ def android_patch() -> None: ' PyParser_ParseStringObject_EfroCB(s, filename, g, start,\n' ' err_ret, flags);\n' ' }\n') - efrotools.writefile(fname, txt) + writefile(fname, txt) print('APPLIED EFROTOOLS ANDROID BUILD PATCHES.') @@ -485,14 +478,14 @@ def gather() -> None: if d.startswith('pylib-') ] for existing_dir in existing_dirs: - efrotools.run('rm -rf "' + existing_dir + '"') + run('rm -rf "' + existing_dir + '"') for buildtype in ['debug', 'release']: debug = buildtype == 'debug' bsuffix = '_debug' if buildtype == 'debug' else '' bsuffix2 = '-debug' if buildtype == 'debug' else '' - libname = 'python' + PYTHON_VERSION_MAJOR + ('dm' if debug else 'm') + libname = 'python' + PYVER + ('dm' if debug else 'm') bases = { 'mac': @@ -513,19 +506,15 @@ def gather() -> None: # Note: only need pylib for the first in each group. builds: List[Dict[str, Any]] = [{ - 'name': - 'macos', - 'group': - 'apple', - 'headers': - bases['mac'] + '/Support/Python/Headers', + 'name': 'macos', + 'group': 'apple', + 'headers': bases['mac'] + '/Support/Python/Headers', 'libs': [ bases['mac'] + '/Support/Python/libPython.a', bases['mac'] + '/Support/OpenSSL/libOpenSSL.a', bases['mac'] + '/Support/XZ/libxz.a' ], - 'pylib': - (bases['mac'] + '/python/lib/python' + PYTHON_VERSION_MAJOR), + 'pylib': (bases['mac'] + '/python/lib/python' + PYVER), }, { 'name': 'ios', @@ -551,12 +540,9 @@ def gather() -> None: bases['tvos'] + '/Support/XZ/libxz.a' ], }, { - 'name': - 'android_arm', - 'group': - 'android', - 'headers': - bases['android_arm'] + f'/usr/include/{libname}', + 'name': 'android_arm', + 'group': 'android', + 'headers': bases['android_arm'] + f'/usr/include/{libname}', 'libs': [ bases['android_arm'] + f'/usr/lib/lib{libname}.a', bases['android_arm'] + '/usr/lib/libssl.a', @@ -564,10 +550,8 @@ def gather() -> None: bases['android_arm'] + '/usr/lib/liblzma.a', bases['android_arm'] + '/usr/lib/libsqlite3.a' ], - 'libinst': - 'android_armeabi-v7a', - 'pylib': (bases['android_arm'] + '/usr/lib/python' + - PYTHON_VERSION_MAJOR), + 'libinst': 'android_armeabi-v7a', + 'pylib': (bases['android_arm'] + '/usr/lib/python' + PYVER), }, { 'name': 'android_arm64', 'group': 'android', @@ -616,16 +600,15 @@ def gather() -> None: # Do some setup only once per group. if not os.path.exists(builddir): - efrotools.run('mkdir -p "' + builddir + '"') - efrotools.run('mkdir -p "' + lib_dst + '"') + run('mkdir -p "' + builddir + '"') + run('mkdir -p "' + lib_dst + '"') # Only pull modules into game assets on release pass. if not debug: # Copy system modules into the src assets # dir for this group. - efrotools.run('mkdir -p "' + assets_src_dst + '"') - efrotools.run( - 'rsync --recursive --include "*.py"' + run('mkdir -p "' + assets_src_dst + '"') + run('rsync --recursive --include "*.py"' ' --exclude __pycache__ --include "*/" --exclude "*" "' + build['pylib'] + '/" "' + assets_src_dst + '"') @@ -639,30 +622,30 @@ def gather() -> None: 'unittest', 'dbm', 'venv', 'ctypes/test', 'imaplib.py', '_sysconfigdata_*' ] - efrotools.run('cd "' + assets_src_dst + '" && rm -rf ' + - ' '.join(prune)) + run('cd "' + assets_src_dst + '" && rm -rf ' + + ' '.join(prune)) # Some minor filtering to system scripts: # on iOS/tvOS, addusersitepackages() leads to a crash # due to _sysconfigdata_dm_ios_darwin module not existing, # so let's skip that. fname = f'{assets_src_dst}/site.py' - txt = efrotools.readfile(fname) - txt = efrotools.replace_one( + txt = readfile(fname) + txt = replace_one( txt, ' known_paths = addusersitepackages(known_paths)', ' # efro tweak: this craps out on ios/tvos.\n' ' # (and we don\'t use it anyway)\n' ' # known_paths = addusersitepackages(known_paths)') - efrotools.writefile(fname, txt) + writefile(fname, txt) # Copy in a base set of headers (everything in a group should # be using the same headers) - efrotools.run(f'cp -r "{build["headers"]}" "{header_dst}"') + run(f'cp -r "{build["headers"]}" "{header_dst}"') # Clear whatever pyconfigs came across; we'll build our own # universal one below. - efrotools.run('rm ' + header_dst + '/pyconfig*') + run('rm ' + header_dst + '/pyconfig*') # Write a master pyconfig header that reroutes to each # platform's actual header. @@ -701,21 +684,21 @@ def gather() -> None: # contents too (those headers can themselves include # others; ios for instance points to a arm64 and a # x86_64 variant). - contents = efrotools.readfile(build['headers'] + '/' + cfg) + contents = readfile(build['headers'] + '/' + cfg) contents = contents.replace('pyconfig', 'pyconfig-' + build['name']) - efrotools.writefile(header_dst + '/' + out, contents) + writefile(header_dst + '/' + out, contents) else: # other configs we just rename - efrotools.run('cp "' + build['headers'] + '/' + cfg + - '" "' + header_dst + '/' + out + '"') + run('cp "' + build['headers'] + '/' + cfg + '" "' + + header_dst + '/' + out + '"') # Copy in libs. If the lib gave a specific install name, # use that; otherwise use name. targetdir = lib_dst + '/' + build.get('libinst', build['name']) - efrotools.run('rm -rf "' + targetdir + '"') - efrotools.run('mkdir -p "' + targetdir + '"') + run('rm -rf "' + targetdir + '"') + run('mkdir -p "' + targetdir + '"') for lib in build['libs']: - efrotools.run('cp "' + lib + '" "' + targetdir + '"') + run('cp "' + lib + '" "' + targetdir + '"') print('Great success!') diff --git a/tools/pcommand b/tools/pcommand index 8019c1c5..db068958 100755 --- a/tools/pcommand +++ b/tools/pcommand @@ -42,7 +42,7 @@ from efrotools.pcommand import ( cpplint, pylint, runpylint, mypy, runmypy, dmypy, tool_config_install, sync, sync_all, scriptfiles, pycharm, clioncode, androidstudiocode, makefile_target_list, spelling, spelling_all, pytest, echo, - compile_python_files) + compile_python_files, pyver) from batools.pcommand import ( stage_server_file, py_examine, fix_mac_ssh, check_mac_ssh, resize_image, check_clean_safety, clean_orphaned_assets, archive_old_builds,