Prep for Python version upgrade

.efrocachemap

@@ -420,8 +420,8 @@
   "assets/build/ba_data/audio/zoeOw.ogg": "https://files.ballistica.net/cache/ba1/75/1d/868bb04cf691736035c917d02762",
   "assets/build/ba_data/audio/zoePickup01.ogg": "https://files.ballistica.net/cache/ba1/44/2a/8535b446284235cb503947ece074",
   "assets/build/ba_data/audio/zoeScream01.ogg": "https://files.ballistica.net/cache/ba1/f5/d3/8e941851c4310465646c4167afc1",
-  "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/de/c0/7197e7658bef48309741dfaf86c7",
+  "assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/cd/7a/dcf7f0f9436884167abdcb126716",
-  "assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/30/8e/38c5f21b9251ea1111bed554035f",
+  "assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/a9/3c/63857b3f5f943205d7a5d8f8e476",
   "assets/build/ba_data/data/languages/belarussian.json": "https://files.ballistica.net/cache/ba1/49/5f/b29bb65369040892fe6601801637",
   "assets/build/ba_data/data/languages/chinese.json": "https://files.ballistica.net/cache/ba1/ea/14/6485c200c717f82cca7c01dee0b3",
   "assets/build/ba_data/data/languages/chinesetraditional.json": "https://files.ballistica.net/cache/ba1/a0/ca/b4e3c4ea2c76e462b7e657b2b1c2",
@@ -436,22 +436,22 @@
   "assets/build/ba_data/data/languages/gibberish.json": "https://files.ballistica.net/cache/ba1/cb/10/5d94df639e3e0cb405711e1b907f",
   "assets/build/ba_data/data/languages/greek.json": "https://files.ballistica.net/cache/ba1/17/78/3fd0dca40e632ce53d03a944e7fa",
   "assets/build/ba_data/data/languages/hindi.json": "https://files.ballistica.net/cache/ba1/11/56/ed2b07866104596338f7ce582d64",
-  "assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/4d/4b/0790110201c9adb1b521e9a55e63",
+  "assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/87/2d/027aa239eb66ea8f496562f4fd83",
   "assets/build/ba_data/data/languages/indonesian.json": "https://files.ballistica.net/cache/ba1/a3/b6/8987b9fc6a7cb0f61364f567c901",
   "assets/build/ba_data/data/languages/italian.json": "https://files.ballistica.net/cache/ba1/ff/3f/f1c11e1f22454ceceb7b4d277a2a",
   "assets/build/ba_data/data/languages/korean.json": "https://files.ballistica.net/cache/ba1/0a/84/bbb6ed2abf66509406f534cbbb52",
   "assets/build/ba_data/data/languages/persian.json": "https://files.ballistica.net/cache/ba1/bd/45/ffb2d9d9ce9578ae11de9bb5123d",
   "assets/build/ba_data/data/languages/polish.json": "https://files.ballistica.net/cache/ba1/e1/cc/13a4ba409b7d6f113f703be1aa8a",
-  "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/9c/e3/834faf6869f5cd175edd4a0244b9",
+  "assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/e5/0a/6ee92ae2cb50f12eeb21059c6659",
   "assets/build/ba_data/data/languages/romanian.json": "https://files.ballistica.net/cache/ba1/44/3c/7cc06ca8d5475e1687d0ed05bdbf",
   "assets/build/ba_data/data/languages/russian.json": "https://files.ballistica.net/cache/ba1/57/bb/33e8bd738c3ee97122186471be75",
   "assets/build/ba_data/data/languages/serbian.json": "https://files.ballistica.net/cache/ba1/e7/d8/ace32888249fc8b8cca0e2edb48b",
   "assets/build/ba_data/data/languages/slovak.json": "https://files.ballistica.net/cache/ba1/b7/0a/fab820b96e7aa587ee56427ecdc2",
-  "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/c9/6b/dadd4329de69d0b7bfe1aa5f31d6",
+  "assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/73/ef/624000057ce55f5a511ba18fd433",
   "assets/build/ba_data/data/languages/swedish.json": "https://files.ballistica.net/cache/ba1/50/9f/be006ba19be6a69a57837eb6dca0",
   "assets/build/ba_data/data/languages/turkish.json": "https://files.ballistica.net/cache/ba1/82/7c/5363de81ac6823935098df95dc66",
   "assets/build/ba_data/data/languages/ukrainian.json": "https://files.ballistica.net/cache/ba1/9d/fb/6c2feb78bdabbec049a71ba40d78",
-  "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/f1/00/c4ccd5969084505359e07f927b3a",
+  "assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/96/96/b2e468ceec8800b7b0f1e1c5977c",
   "assets/build/ba_data/data/maps/big_g.json": "https://files.ballistica.net/cache/ba1/47/0a/a617cc85d927b576c4e6fc1091ed",
   "assets/build/ba_data/data/maps/bridgit.json": "https://files.ballistica.net/cache/ba1/03/4b/57ee9b42854b26f23f81bd8c58ef",
   "assets/build/ba_data/data/maps/courtyard.json": "https://files.ballistica.net/cache/ba1/03/38/344dd05bfef7bbdf464035ec5aa2",
@@ -4135,16 +4135,16 @@
   "assets/build/windows/x64/vc_redist.x64.exe": "https://files.ballistica.net/cache/ba1/ea/19/8b8787d81abcdce158ba608cd24f",
   "assets/build/windows/x64/vcruntime140_1d.dll": "https://files.ballistica.net/cache/ba1/11/d8/ff6344b429b00c24d9a1930d4338",
   "assets/build/windows/x64/vcruntime140d.dll": "https://files.ballistica.net/cache/ba1/20/33/0825e11e6518f87ece3009309933",
-  "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b2/16/0414b009ce117f73bb2bc8991d3b",
+  "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/05/00/2dd9723909746d8039c6bdf8cddb",
-  "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/66/07/d82a3ec80ff3243321e373439210",
+  "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/3e/62/6c310233f4b2c224a46914a4b78e",
-  "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/12/5f/f87158c3281a2616897fdd2165cb",
+  "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/0c/f8/d381993bd58c1f50f0da6d8e2be3",
-  "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/69/23/a2d029f81db60cf89ad5f6510ddb",
+  "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/a5/b5/d2ec28b09124404731b7f08008c8",
-  "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c4/e3/5aba369252ed1977be8950e5a1ce",
+  "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/cb/82/4520a3fec142215ed0673a1811d9",
-  "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/f7/ef/c28554f863485e9cd54f0d8d4e6f",
+  "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b9/0b/060333370a987cee92bdd081a8ca",
-  "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/68/af/f060b3463bc31c202a830fb0b20a",
+  "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/11/31/9f1bb364db47563a1d86f0165549",
-  "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/96/46/a239821d00e782303077192817c5",
+  "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/11/62/4d784f02705e438f9bd519ff53f4",
-  "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/dc/15/fdd9195afe2bf299405ec49ed238",
+  "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/8a/ff/abcf2602dc64f718408efeaf4d3a",
-  "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/68/f9/c42b1f9b5af34cb371f80250e16b",
+  "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/04/2d/1f8c7ea677bae4cc7be8e967c344",
-  "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5d/9b/213c602a9aa9cbc28872d2753f51",
+  "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/26/b8/f19e24bd0e43ed9661ca62c10e6c",
-  "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/2f/a4/e6a6a6e0e2b64c5232cccddffdd3"
+  "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/02/0b/d2fab33376bc7ae1c8797cae5c0f"
 }

.idea/dictionaries/ericf.xml

@@ -1613,6 +1613,7 @@
       <w>pythonw</w>
       <w>pytree</w>
       <w>pytz</w>
+      <w>pyver</w>
       <w>qrcode</w>
       <w>qrencode</w>
       <w>qual</w>
@@ -1788,6 +1789,7 @@
       <w>simplesubclasses</w>
       <w>sincelaunch</w>
       <w>singledispatch</w>
+      <w>singledispatchmethod</w>
       <w>sirplus</w>
       <w>sitebuiltins</w>
       <w>skey</w>

CHANGELOG.md

@@ -1,3 +1,6 @@
+### 1.5.24 (20159)
+- Misc bug fixes..
+
 ### 1.5.23 (20146)
 - Fixed the shebang line in `bombsquad_server` file by using `-S` flag for `/usr/bin/env`.
 - Fixed a bug with hardware keyboards emitting extra characters in the in-game console (~ or F2)

assets/src/ba_data/python/ba/_netutils.py

@@ -56,7 +56,7 @@ def get_ip_address_type(addr: str) -> socket.AddressFamily:
         except OSError:
             pass
     if socket_type is None:
-        raise ValueError('addr seems to be neither v4 or v6: ' + str(addr))
+        raise ValueError(f'addr seems to be neither v4 or v6: {addr}')
     return socket_type
 
 
@@ -152,8 +152,10 @@ class ServerCallThread(threading.Thread):
             response_data = None
 
             # Ignore common network errors; note unexpected ones.
-            if isinstance(exc, (urllib.error.URLError, ConnectionError,
+            if isinstance(
-                                http.client.IncompleteRead)):
+                    exc,
+                (urllib.error.URLError, ConnectionError,
+                 http.client.IncompleteRead, http.client.BadStatusLine)):
                 pass
             elif isinstance(exc, OSError):
                 if exc.errno == 10051:  # Windows unreachable network error.

assets/src/ba_data/python/bastd/actor/bomb.py

@@ -887,10 +887,11 @@ class Bomb(ba.Actor):
     def _handle_impact(self) -> None:
         node = ba.getcollision().opposingnode
 
-        # If we're an impact bomb and we came from this node, don't explode...
+        # If we're an impact bomb and we came from this node, don't explode.
-        # alternately if we're hitting another impact-bomb from the same
+        # (otherwise we blow up on our own head when jumping).
-        # source, don't explode...
+        # Alternately if we're hitting another impact-bomb from the same
-        # try:
+        # source, don't explode. (can cause accidental explosions if rapidly
+        # throwing/etc.)
         node_delegate = node.getdelegate(object)
         if node:
             if (self.bomb_type == 'impact' and

assets/src/ba_data/python/bastd/ui/gather.py

@@ -1673,7 +1673,7 @@ class GatherWindow(ba.Window):
                                 # Ignore harmless errors.
                                 if exc.errno in {
                                         errno.EHOSTUNREACH, errno.ENETUNREACH,
-                                        errno.EINVAL, errno.EPERM
+                                        errno.EINVAL, errno.EPERM, errno.EACCES
                                 }:
                                     pass
                                 elif exc.errno == 10022:

assets/src/ba_data/python/bastd/ui/settings/gamepad.py

@@ -838,7 +838,8 @@ class AwaitGamepadInputWindow(ba.Window):
         assert isinstance(input_device, ba.InputDevice)
 
         # Update - we now allow *any* input device of this type.
-        if input_device.exists() and input_device.name == self._input.name:
+        if (self._input and input_device
+                and input_device.name == self._input.name):
             self._callback(self._capture_button, event, self)
 
     def _decrement(self) -> None:

assets/src/ba_data/python/bastd/ui/settings/keyboard.py

@@ -236,6 +236,11 @@ class ConfigKeyboardWindow(ba.Window):
 
         ba.containerwidget(edit=self._root_widget, transition='out_right')
         ba.playsound(ba.getsound('gunCocking'))
+
+        # There's a chance the device disappeared; handle that gracefully.
+        if not self._input:
+            return
+
         dst = get_input_device_config(self._input, default=False)
         dst2: Dict[str, Any] = dst[0][dst[1]]
         dst2.clear()

docs/ba_module.md

@@ -1,5 +1,5 @@
 <!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
-<h4><em>last updated on 2020-07-22 for Ballistica version 1.5.23 build 20152</em></h4>
+<h4><em>last updated on 2020-07-26 for Ballistica version 1.5.24 build 20158</em></h4>
 <p>This page documents the Python classes and functions in the 'ba' module,
  which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
 <hr>

tools/batools/assetstaging.py

@@ -30,7 +30,7 @@ import subprocess
 from functools import partial
 from typing import TYPE_CHECKING
 
-from efrotools.pybuild import PYTHON_VERSION_MAJOR
+from efrotools import PYVER
 
 if TYPE_CHECKING:
     from typing import Optional, List
@@ -40,8 +40,7 @@ if TYPE_CHECKING:
 # Note: this means anyone wanting to modify .py files in a build
 # will need to wipe out the existing .pyc files first or the changes
 # will be ignored.
-OPT_PYC_SUFFIX = ('cpython-' + PYTHON_VERSION_MAJOR.replace('.', '') +
+OPT_PYC_SUFFIX = ('cpython-' + PYVER.replace('.', '') + '.opt-1.pyc')
-                  '.opt-1.pyc')
 
 
 class Config:

tools/batools/pcommand.py

@@ -43,6 +43,7 @@ def stage_server_file() -> None:
     import batools.build
     from efro.error import CleanError
     from efrotools import replace_one
+    from efrotools import PYVER
     if len(sys.argv) != 5:
         raise CleanError('Expected 3 args (mode, infile, outfile).')
     mode, infilename, outfilename = sys.argv[2], sys.argv[3], sys.argv[4]
@@ -56,13 +57,15 @@ def stage_server_file() -> None:
         # Inject all available config values into the config file.
         batools.build.filter_server_config(str(PROJROOT), infilename,
                                            outfilename)
+
     elif basename == 'ballisticacore_server.py':
         # Run Python in opt mode for release builds.
         with open(infilename) as infile:
             lines = infile.read().splitlines()
             if mode == 'release':
-                lines[0] = replace_one(lines[0], '#!/usr/bin/env python3.7',
+                lines[0] = replace_one(lines[0],
-                                       '#!/usr/bin/env -S python3.7 -O')
+                                       f'#!/usr/bin/env python{PYVER}',
+                                       f'#!/usr/bin/env -S python{PYVER} -O')
         with open(outfilename, 'w') as outfile:
             outfile.write('\n'.join(lines) + '\n')
         subprocess.run(['chmod', '+x', outfilename], check=True)

tools/batools/updateproject.py

@@ -377,7 +377,7 @@ class Updater:
 
     def _check_python_file(self, fname: str) -> None:
         # pylint: disable=too-many-branches
-        from efrotools import get_public_license
+        from efrotools import get_public_license, PYVER
         with open(fname) as infile:
             contents = infile.read()
             lines = contents.splitlines()
@@ -390,7 +390,7 @@ class Updater:
             if fname not in [
                     'tools/devtool', 'tools/version_utils', 'tools/vmshell'
             ]:
-                if not contents.startswith('#!/usr/bin/env python3.7'):
+                if not contents.startswith(f'#!/usr/bin/env python{PYVER}'):
                     print(f'{Clr.RED}Incorrect shebang (first line) for '
                           f'{fname}.{Clr.RST}')
                     sys.exit(255)

tools/efro/util.py

@@ -115,9 +115,10 @@ def dispatchmethod(
     # All we do here is recreate the end of functools.singledispatch
     # where it returns a wrapper except instead of the wrapper using the
     # first arg to the function ours uses the second (to skip 'self').
-    # This was made with Python 3.7; we should probably check up on
+    # This was made against Python 3.7; we should probably check up on
     # this in later versions in case anything has changed.
     # (or hopefully they'll add this functionality to their version)
+    # NOTE: sounds like we can use functools singledispatchmethod in 3.8
     def wrapper(*args: Any, **kw: Any) -> Any:
         if not args or len(args) < 2:
             raise TypeError(f'{funcname} requires at least '

tools/efrotools/__init__.py

@@ -40,8 +40,11 @@ if TYPE_CHECKING:
     from typing import Dict, Union, Sequence, Optional, Any
     from typing_extensions import Literal
 
+# Python major version we're using for all this stuff.
+PYVER = '3.7'
+
 # Python binary assumed by these tools.
-PYTHON_BIN = 'python3.7' if platform.system() != 'Windows' else 'python'
+PYTHON_BIN = f'python{PYVER}' if platform.system() != 'Windows' else 'python'
 
 MIT_LICENSE = """Copyright (c) 2011-2020 Eric Froemling
 

tools/efrotools/code.py

@@ -86,7 +86,7 @@ def cpplint(projroot: Path, full: bool) -> None:
     import tempfile
     from concurrent.futures import ThreadPoolExecutor
     from multiprocessing import cpu_count
-    from efrotools import getconfig
+    from efrotools import getconfig, PYVER
     from efro.terminal import Clr
     from efro.error import CleanError
 
@@ -150,7 +150,8 @@ def cpplint(projroot: Path, full: bool) -> None:
 
     def lint_file(filename: str) -> None:
         result = subprocess.call(
-            ['python3.7', '-m', 'cpplint', '--root=src', filename], env=env)
+            [f'python{PYVER}', '-m', 'cpplint', '--root=src', filename],
+            env=env)
         if result != 0:
             raise CleanError(
                 f'{Clr.RED}Cpplint failed for {filename}.{Clr.RST}')
@@ -197,7 +198,7 @@ def formatscripts(projroot: Path, full: bool) -> None:
     import time
     from concurrent.futures import ThreadPoolExecutor
     from multiprocessing import cpu_count
-    from efrotools import get_files_hash
+    from efrotools import get_files_hash, PYVER
     os.chdir(projroot)
     cachepath = Path(projroot, 'config/.cache-formatscripts')
     if full and cachepath.exists():
@@ -215,7 +216,7 @@ def formatscripts(projroot: Path, full: bool) -> None:
     def format_file(filename: str) -> None:
         start_time = time.time()
         result = subprocess.call(
-            ['python3.7', '-m', 'yapf', '--in-place', filename])
+            [f'python{PYVER}', '-m', 'yapf', '--in-place', filename])
         if result != 0:
             raise Exception(f'Formatting failed for {filename}')
         duration = time.time() - start_time

tools/efrotools/pcommand.py

@@ -174,6 +174,12 @@ def spelling() -> None:
     _spelling(sys.argv[2:])
 
 
+def pyver() -> None:
+    """Prints the Python version used by this project."""
+    from efrotools import PYVER
+    print(PYVER, end='')
+
+
 def check_clean_safety() -> None:
     """Ensure all files are are added to git or in gitignore.
 

tools/efrotools/pybuild.py

@@ -25,14 +25,11 @@ from __future__ import annotations
 import os
 from typing import TYPE_CHECKING
 
-import efrotools
+from efrotools import PYVER, run, readfile, writefile, replace_one
 
 if TYPE_CHECKING:
     from typing import List, Dict, Any
 
-# Overall version we're using for the game currently.
-PYTHON_VERSION_MAJOR = '3.7'
-
 ENABLE_OPENSSL = True
 
 
@@ -51,23 +48,22 @@ def build_apple(arch: str, debug: bool = False) -> None:
             raise CleanError('NEED TO TEMP-KILL GETTEXT')
 
     builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
-    efrotools.run('rm -rf "' + builddir + '"')
+    run('rm -rf "' + builddir + '"')
-    efrotools.run('mkdir -p build')
+    run('mkdir -p build')
-    efrotools.run('git clone '
+    run('git clone '
-                  'git@github.com:pybee/Python-Apple-support.git "' +
+        'git@github.com:pybee/Python-Apple-support.git "' + builddir + '"')
-                  builddir + '"')
     os.chdir(builddir)
 
     # TEMP: Check out a particular commit while the branch head is broken.
     # We can actually fix this to use the current one, but something
     # broke in the underlying build even on old commits so keeping it
     # locked for now...
-    # efrotools.run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
+    # run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
-    efrotools.run(f'git checkout {PYTHON_VERSION_MAJOR}')
+    run(f'git checkout {PYVER}')
 
     # On mac we currently have to add the _scproxy module or urllib will
     # fail.
-    txt = efrotools.readfile('patch/Python/Setup.embedded')
+    txt = readfile('patch/Python/Setup.embedded')
     if arch == 'mac':
         txt += ('\n'
                 '# ericf added - mac urllib needs this\n'
@@ -76,19 +72,19 @@ def build_apple(arch: str, debug: bool = False) -> None:
                 '-framework CoreFoundation')
 
     # Turn off sqlite module. (scratch that; leaving it in.)
-    # txt = efrotools.replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
+    # txt = replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
     # txt = txt.replace('    _sqlite/', '#    _sqlite/')
 
     # Turn off xz compression module. (scratch that; leaving it in.)
-    # txt = efrotools.replace_one(txt, '_lzma _', '#_lzma _')
+    # txt = replace_one(txt, '_lzma _', '#_lzma _')
 
     # Turn off bzip2 module.
-    txt = efrotools.replace_one(txt, '_bz2 _b', '#_bz2 _b')
+    txt = replace_one(txt, '_bz2 _b', '#_bz2 _b')
 
     # Turn off openssl module (only if not doing openssl).
     if not ENABLE_OPENSSL:
-        txt = efrotools.replace_one(txt, '_hashlib _hashopenssl.c',
+        txt = replace_one(txt, '_hashlib _hashopenssl.c',
-                                    '#_hashlib _hashopenssl.c')
+                          '#_hashlib _hashopenssl.c')
 
     # Turn off various other stuff we don't use.
     for line in [
@@ -123,7 +119,7 @@ def build_apple(arch: str, debug: bool = False) -> None:
             '_testimportmultiple _testimportmultiple.c',
             '_crypt _cryptmodule.c',  # not on android so disabling here too
     ]:
-        txt = efrotools.replace_one(txt, line, '#' + line)
+        txt = replace_one(txt, line, '#' + line)
 
     if ENABLE_OPENSSL:
 
@@ -138,28 +134,28 @@ def build_apple(arch: str, debug: bool = False) -> None:
                 '_sha256 sha256module.c',
                 '_sha512 sha512module.c',
         ]:
-            txt = efrotools.replace_one(txt, line, '#' + line)
+            txt = replace_one(txt, line, '#' + line)
     else:
-        txt = efrotools.replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
+        txt = replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
-    efrotools.writefile('patch/Python/Setup.embedded', txt)
+    writefile('patch/Python/Setup.embedded', txt)
 
-    txt = efrotools.readfile('Makefile')
+    txt = readfile('Makefile')
 
     # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
-    txt = efrotools.replace_one(
+    txt = replace_one(
         txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .',
         '&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .')
 
     # Remove makefile dependencies so we don't build the
     # libs we're not using.
     srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: '
-    txt = efrotools.replace_one(
+    txt = replace_one(
         txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' +
         ('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') +
         'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt)
     srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.'
               '$(BUILD_NUMBER).tar.gz: ')
-    txt = efrotools.replace_one(
+    txt = replace_one(
         txt, srctxt,
         'dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz:'
         ' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt)
@@ -169,14 +165,14 @@ def build_apple(arch: str, debug: bool = False) -> None:
 
     # Set mac/ios version reqs
     # (see issue with utimensat and futimens).
-    txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
+    txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
-                                'MACOSX_DEPLOYMENT_TARGET=10.14')
+                      'MACOSX_DEPLOYMENT_TARGET=10.14')
     # And equivalent iOS (11+).
-    txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
+    txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
-                                'CFLAGS-iOS=-mios-version-min=12.0')
+                      'CFLAGS-iOS=-mios-version-min=12.0')
     # Ditto for tvOS.
-    txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
+    txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
-                                'CFLAGS-tvOS=-mtvos-version-min=12.0')
+                      'CFLAGS-tvOS=-mtvos-version-min=12.0')
 
     if debug:
 
@@ -196,13 +192,13 @@ def build_apple(arch: str, debug: bool = False) -> None:
             raise RuntimeError(f'Unexpected configure line count {splitlen}.')
         txt = txt.replace(dline, 'python$(PYTHON_VER)dm')
 
-    efrotools.writefile('Makefile', txt)
+    writefile('Makefile', txt)
 
     # Ok; let 'er rip.
     # (we run these in parallel so limit to 1 job a piece;
     # otherwise they inherit the -j12 or whatever from the top level)
     # (also this build seems to fail with multiple threads)
-    efrotools.run('make -j1 ' + {
+    run('make -j1 ' + {
         'mac': 'Python-macOS',
         'ios': 'Python-iOS',
         'tvos': 'Python-tvOS'
@@ -217,11 +213,10 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
     """
     import subprocess
     builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
-    efrotools.run('rm -rf "' + builddir + '"')
+    run('rm -rf "' + builddir + '"')
-    efrotools.run('mkdir -p build')
+    run('mkdir -p build')
-    efrotools.run('git clone '
+    run('git clone '
-                  'git@github.com:yan12125/python3-android.git "' + builddir +
+        'git@github.com:yan12125/python3-android.git "' + builddir + '"')
-                  '"')
     os.chdir(builddir)
 
     # It seems we now need 'autopoint' as part of this build, but on mac it
@@ -242,69 +237,69 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
     # a a completely new minimal one which will take some work to update here.
     # Punting on that for now...
     if True:  # pylint: disable=using-constant-test
-        efrotools.run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
+        run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
-    ftxt = efrotools.readfile('pybuild/env.py')
+    ftxt = readfile('pybuild/env.py')
 
     # Set the packages we build.
-    ftxt = efrotools.replace_one(
+    ftxt = replace_one(
         ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," +
         (" 'openssl'" if ENABLE_OPENSSL else '') + ')\n# packages = (')
 
     # Don't wanna bother with gpg signing stuff.
-    ftxt = efrotools.replace_one(ftxt, 'verify_source = True',
+    ftxt = replace_one(ftxt, 'verify_source = True', 'verify_source = False')
-                                 'verify_source = False')
 
     # Sub in the min api level we're targeting.
-    ftxt = efrotools.replace_one(ftxt, 'android_api_level = 21',
+    ftxt = replace_one(ftxt, 'android_api_level = 21',
-                                 'android_api_level = 21')
+                       'android_api_level = 21')
-    ftxt = efrotools.replace_one(ftxt, "target_arch = 'arm'",
+    ftxt = replace_one(ftxt, "target_arch = 'arm'",
-                                 "target_arch = '" + arch + "'")
+                       "target_arch = '" + arch + "'")
-    efrotools.writefile('pybuild/env.py', ftxt)
+    writefile('pybuild/env.py', ftxt)
-    ftxt = efrotools.readfile('Makefile')
+    ftxt = readfile('Makefile')
 
     # This needs to be python3 for us.
-    ftxt = efrotools.replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
+    ftxt = replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
-    efrotools.writefile('Makefile', ftxt)
+    writefile('Makefile', ftxt)
-    ftxt = efrotools.readfile('pybuild/packages/python.py')
+    ftxt = readfile('pybuild/packages/python.py')
 
     # We currently build as a static lib.
-    ftxt = efrotools.replace_one(ftxt, "            '--enable-shared',\n", '')
+    ftxt = replace_one(ftxt, "            '--enable-shared',\n", '')
-    ftxt = efrotools.replace_one(
+    ftxt = replace_one(
         ftxt, "super().__init__('https://github.com/python/cpython/')",
-        "super().__init__('https://github.com/python/cpython/', branch='3.7')")
+        "super().__init__('https://github.com/python/cpython/'"
+        f", branch='{PYVER}')")
 
     # Turn ipv6 on (curious why its turned off here?...)
     # Also, turn on low level debugging features for our debug builds.
-    ftxt = efrotools.replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
+    ftxt = replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
     if debug:
-        ftxt = efrotools.replace_one(ftxt, "'./configure',",
+        ftxt = replace_one(ftxt, "'./configure',",
-                                     "'./configure', '--with-pydebug',")
+                           "'./configure', '--with-pydebug',")
 
     # We don't use this stuff so lets strip it out to simplify.
-    ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", '')
+    ftxt = replace_one(ftxt, "'--without-ensurepip',", '')
 
     # This builds all modules as dynamic libs, but we want to be consistent
     # with our other embedded builds and just static-build the ones we
     # need... so to change that we'll need to add a hook for ourself after
     # python is downloaded but before it is built so we can muck with it.
-    ftxt = efrotools.replace_one(
+    ftxt = replace_one(
         ftxt, '    def build(self):',
         '    def build(self):\n        import os\n'
         '        if os.system(\'"' + rootdir +
         '/tools/pcommand" python_android_patch "' + os.getcwd() +
         '"\') != 0: raise Exception("patch apply failed")')
 
-    efrotools.writefile('pybuild/packages/python.py', ftxt)
+    writefile('pybuild/packages/python.py', ftxt)
 
     # Set this to a particular cpython commit to target exact releases from git
     # commit = 'd7c567b08f9d7d6aef21b881340a2b72731129db'  # 3.7.7 release
     commit = '4b47a5b6ba66b02df9392feb97b8ead916f8c1fa'  # 3.7.8 release
 
     if commit is not None:
-        ftxt = efrotools.readfile('pybuild/source.py')
+        ftxt = readfile('pybuild/source.py')
 
         # Check out a particular commit right after the clone.
-        ftxt = efrotools.replace_one(
+        ftxt = replace_one(
             ftxt, "'git', 'clone', '--single-branch', '-b',"
             ' self.branch, self.source_url, self.dest])',
             "'git', 'clone', '-b',"
@@ -315,16 +310,16 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
             "'https://github.com/python/cpython/':\n"
             "            run_in_dir(['git', 'checkout', '" + commit +
             "'], self.source_dir)")
-        efrotools.writefile('pybuild/source.py', ftxt)
+        writefile('pybuild/source.py', ftxt)
-    ftxt = efrotools.readfile('pybuild/util.py')
+    ftxt = readfile('pybuild/util.py')
 
     # Still don't wanna bother with gpg signing stuff.
-    ftxt = efrotools.replace_one(
+    ftxt = replace_one(
         ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n',
         'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n'
         '    print("gpg-verify disabled by ericf")\n'
         '    return\n')
-    efrotools.writefile('pybuild/util.py', ftxt)
+    writefile('pybuild/util.py', ftxt)
 
     # These builds require ANDROID_NDK to be set, so make sure that's
     # the case.
@@ -335,18 +330,18 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
     # Ok, let 'er rip
     # (we often run these builds in parallel so limit to 1 job a piece;
     # otherwise they each inherit the -j12 or whatever from the top level).
-    efrotools.run('make -j1')
+    run('make -j1')
     print('python build complete! (android/' + arch + ')')
 
 
 def android_patch() -> None:
     """Run necessary patches on an android archive before building."""
     fname = 'src/cpython/Modules/Setup.dist'
-    txt = efrotools.readfile(fname)
+    txt = readfile(fname)
 
     # Need to switch some flags on this one.
-    txt = efrotools.replace_one(txt, '#zlib zlibmodule.c',
+    txt = replace_one(txt, '#zlib zlibmodule.c',
-                                'zlib zlibmodule.c -lz\n#zlib zlibmodule.c')
+                      'zlib zlibmodule.c -lz\n#zlib zlibmodule.c')
     # Just turn all these on.
     for enable in [
             '#array arraymodule.c', '#cmath cmathmodule.c _math.c',
@@ -362,10 +357,10 @@ def android_patch() -> None:
             '#binascii binascii.c', '#_posixsubprocess _posixsubprocess.c',
             '#_sha3 _sha3/sha3module.c'
     ]:
-        txt = efrotools.replace_one(txt, enable, enable[1:])
+        txt = replace_one(txt, enable, enable[1:])
     if ENABLE_OPENSSL:
-        txt = efrotools.replace_one(txt, '#_ssl _ssl.c \\',
+        txt = replace_one(txt, '#_ssl _ssl.c \\',
-                                    '_ssl _ssl.c -DUSE_SSL -lssl -lcrypto')
+                          '_ssl _ssl.c -DUSE_SSL -lssl -lcrypto')
     else:
         # Note that the _md5 and _sha modules are normally only built if the
         # system does not have the OpenSSL libs containing an optimized
@@ -374,11 +369,10 @@ def android_patch() -> None:
                 '#_md5 md5module.c', '#_sha1 sha1module.c',
                 '#_sha256 sha256module.c', '#_sha512 sha512module.c'
         ]:
-            txt = efrotools.replace_one(txt, enable, enable[1:])
+            txt = replace_one(txt, enable, enable[1:])
 
     # Turn this off (its just an example module).
-    txt = efrotools.replace_one(txt, 'xxsubtype xxsubtype.c',
+    txt = replace_one(txt, 'xxsubtype xxsubtype.c', '#xxsubtype xxsubtype.c')
-                                '#xxsubtype xxsubtype.c')
 
     # For whatever reason this stuff isn't in there at all; add it.
     txt += '\n_json _json.c\n'
@@ -403,7 +397,7 @@ def android_patch() -> None:
 
     txt += '\n\n*disabled*\n_ctypes _crypt grp'
 
-    efrotools.writefile(fname, txt)
+    writefile(fname, txt)
 
     # Ok, this is weird.
     # When applying the module Setup, python looks for any line containing *=*
@@ -412,17 +406,16 @@ def android_patch() -> None:
     # The check used to look for [A-Z]*=* which didn't break, so let' just
     # change it back to that for now.
     fname = 'src/cpython/Modules/makesetup'
-    txt = efrotools.readfile(fname)
+    txt = readfile(fname)
-    txt = efrotools.replace_one(
+    txt = replace_one(txt, '		*=*)	DEFS="$line$NL$DEFS"; continue;;',
-        txt, '		*=*)	DEFS="$line$NL$DEFS"; continue;;',
+                      '		[A-Z]*=*)	DEFS="$line$NL$DEFS"; continue;;')
-        '		[A-Z]*=*)	DEFS="$line$NL$DEFS"; continue;;')
+    writefile(fname, txt)
-    efrotools.writefile(fname, txt)
 
     # Add custom callbacks to Python's PyParser_ParseFileObject
     # and PyParser_ParseString calls to debug a crash.
     fname = 'src/cpython/Parser/parsetok.c'
-    txt = efrotools.readfile(fname)
+    txt = readfile(fname)
-    txt = efrotools.replace_one(
+    txt = replace_one(
         txt, 'node *\n'
         'PyParser_ParseFileObject(FILE *fp, PyObject *filename,\n'
         '                         const char *enc, grammar *g, int start,\n'
@@ -444,7 +437,7 @@ def android_patch() -> None:
         '                                       start, ps1, ps2,\n'
         '                                       err_ret, flags);\n'
         '    }\n')
-    txt = efrotools.replace_one(
+    txt = replace_one(
         txt, 'node *\n'
         'PyParser_ParseStringObject(const char *s, PyObject *filename,\n'
         '                           grammar *g, int start,\n'
@@ -462,7 +455,7 @@ def android_patch() -> None:
         '        PyParser_ParseStringObject_EfroCB(s, filename, g, start,\n'
         '                                          err_ret, flags);\n'
         '    }\n')
-    efrotools.writefile(fname, txt)
+    writefile(fname, txt)
 
     print('APPLIED EFROTOOLS ANDROID BUILD PATCHES.')
 
@@ -485,14 +478,14 @@ def gather() -> None:
         if d.startswith('pylib-')
     ]
     for existing_dir in existing_dirs:
-        efrotools.run('rm -rf "' + existing_dir + '"')
+        run('rm -rf "' + existing_dir + '"')
 
     for buildtype in ['debug', 'release']:
         debug = buildtype == 'debug'
         bsuffix = '_debug' if buildtype == 'debug' else ''
         bsuffix2 = '-debug' if buildtype == 'debug' else ''
 
-        libname = 'python' + PYTHON_VERSION_MAJOR + ('dm' if debug else 'm')
+        libname = 'python' + PYVER + ('dm' if debug else 'm')
 
         bases = {
             'mac':
@@ -513,19 +506,15 @@ def gather() -> None:
 
         # Note: only need pylib for the first in each group.
         builds: List[Dict[str, Any]] = [{
-            'name':
+            'name': 'macos',
-                'macos',
+            'group': 'apple',
-            'group':
+            'headers': bases['mac'] + '/Support/Python/Headers',
-                'apple',
-            'headers':
-                bases['mac'] + '/Support/Python/Headers',
             'libs': [
                 bases['mac'] + '/Support/Python/libPython.a',
                 bases['mac'] + '/Support/OpenSSL/libOpenSSL.a',
                 bases['mac'] + '/Support/XZ/libxz.a'
             ],
-            'pylib':
+            'pylib': (bases['mac'] + '/python/lib/python' + PYVER),
-                (bases['mac'] + '/python/lib/python' + PYTHON_VERSION_MAJOR),
         }, {
             'name':
                 'ios',
@@ -551,12 +540,9 @@ def gather() -> None:
                 bases['tvos'] + '/Support/XZ/libxz.a'
             ],
         }, {
-            'name':
+            'name': 'android_arm',
-                'android_arm',
+            'group': 'android',
-            'group':
+            'headers': bases['android_arm'] + f'/usr/include/{libname}',
-                'android',
-            'headers':
-                bases['android_arm'] + f'/usr/include/{libname}',
             'libs': [
                 bases['android_arm'] + f'/usr/lib/lib{libname}.a',
                 bases['android_arm'] + '/usr/lib/libssl.a',
@@ -564,10 +550,8 @@ def gather() -> None:
                 bases['android_arm'] + '/usr/lib/liblzma.a',
                 bases['android_arm'] + '/usr/lib/libsqlite3.a'
             ],
-            'libinst':
+            'libinst': 'android_armeabi-v7a',
-                'android_armeabi-v7a',
+            'pylib': (bases['android_arm'] + '/usr/lib/python' + PYVER),
-            'pylib': (bases['android_arm'] + '/usr/lib/python' +
-                      PYTHON_VERSION_MAJOR),
         }, {
             'name': 'android_arm64',
             'group': 'android',
@@ -616,16 +600,15 @@ def gather() -> None:
 
             # Do some setup only once per group.
             if not os.path.exists(builddir):
-                efrotools.run('mkdir -p "' + builddir + '"')
+                run('mkdir -p "' + builddir + '"')
-                efrotools.run('mkdir -p "' + lib_dst + '"')
+                run('mkdir -p "' + lib_dst + '"')
 
                 # Only pull modules into game assets on release pass.
                 if not debug:
                     # Copy system modules into the src assets
                     # dir for this group.
-                    efrotools.run('mkdir -p "' + assets_src_dst + '"')
+                    run('mkdir -p "' + assets_src_dst + '"')
-                    efrotools.run(
+                    run('rsync --recursive --include "*.py"'
-                        'rsync --recursive --include "*.py"'
                         ' --exclude __pycache__ --include "*/" --exclude "*" "'
                         + build['pylib'] + '/" "' + assets_src_dst + '"')
 
@@ -639,30 +622,30 @@ def gather() -> None:
                         'unittest', 'dbm', 'venv', 'ctypes/test', 'imaplib.py',
                         '_sysconfigdata_*'
                     ]
-                    efrotools.run('cd "' + assets_src_dst + '" && rm -rf ' +
+                    run('cd "' + assets_src_dst + '" && rm -rf ' +
-                                  ' '.join(prune))
+                        ' '.join(prune))
 
                     # Some minor filtering to system scripts:
                     # on iOS/tvOS, addusersitepackages() leads to a crash
                     # due to _sysconfigdata_dm_ios_darwin module not existing,
                     # so let's skip that.
                     fname = f'{assets_src_dst}/site.py'
-                    txt = efrotools.readfile(fname)
+                    txt = readfile(fname)
-                    txt = efrotools.replace_one(
+                    txt = replace_one(
                         txt,
                         '    known_paths = addusersitepackages(known_paths)',
                         '    # efro tweak: this craps out on ios/tvos.\n'
                         '    # (and we don\'t use it anyway)\n'
                         '    # known_paths = addusersitepackages(known_paths)')
-                    efrotools.writefile(fname, txt)
+                    writefile(fname, txt)
 
                 # Copy in a base set of headers (everything in a group should
                 # be using the same headers)
-                efrotools.run(f'cp -r "{build["headers"]}" "{header_dst}"')
+                run(f'cp -r "{build["headers"]}" "{header_dst}"')
 
                 # Clear whatever pyconfigs came across; we'll build our own
                 # universal one below.
-                efrotools.run('rm ' + header_dst + '/pyconfig*')
+                run('rm ' + header_dst + '/pyconfig*')
 
                 # Write a master pyconfig header that reroutes to each
                 # platform's actual header.
@@ -701,21 +684,21 @@ def gather() -> None:
                     # contents too (those headers can themselves include
                     # others; ios for instance points to a arm64 and a
                     # x86_64 variant).
-                    contents = efrotools.readfile(build['headers'] + '/' + cfg)
+                    contents = readfile(build['headers'] + '/' + cfg)
                     contents = contents.replace('pyconfig',
                                                 'pyconfig-' + build['name'])
-                    efrotools.writefile(header_dst + '/' + out, contents)
+                    writefile(header_dst + '/' + out, contents)
                 else:
                     # other configs we just rename
-                    efrotools.run('cp "' + build['headers'] + '/' + cfg +
+                    run('cp "' + build['headers'] + '/' + cfg + '" "' +
-                                  '" "' + header_dst + '/' + out + '"')
+                        header_dst + '/' + out + '"')
 
             # Copy in libs. If the lib gave a specific install name,
             # use that; otherwise use name.
             targetdir = lib_dst + '/' + build.get('libinst', build['name'])
-            efrotools.run('rm -rf "' + targetdir + '"')
+            run('rm -rf "' + targetdir + '"')
-            efrotools.run('mkdir -p "' + targetdir + '"')
+            run('mkdir -p "' + targetdir + '"')
             for lib in build['libs']:
-                efrotools.run('cp "' + lib + '" "' + targetdir + '"')
+                run('cp "' + lib + '" "' + targetdir + '"')
 
     print('Great success!')

tools/pcommand

@@ -42,7 +42,7 @@ from efrotools.pcommand import (
     cpplint, pylint, runpylint, mypy, runmypy, dmypy, tool_config_install,
     sync, sync_all, scriptfiles, pycharm, clioncode, androidstudiocode,
     makefile_target_list, spelling, spelling_all, pytest, echo,
-    compile_python_files)
+    compile_python_files, pyver)
 from batools.pcommand import (
     stage_server_file, py_examine, fix_mac_ssh, check_mac_ssh, resize_image,
     check_clean_safety, clean_orphaned_assets, archive_old_builds,