Prep for Python version upgrade

This commit is contained in:
Eric Froemling 2020-07-26 00:57:25 -07:00
parent 49e4da8b4f
commit 147c8ec66b
18 changed files with 178 additions and 168 deletions

View File

@ -420,8 +420,8 @@
"assets/build/ba_data/audio/zoeOw.ogg": "https://files.ballistica.net/cache/ba1/75/1d/868bb04cf691736035c917d02762",
"assets/build/ba_data/audio/zoePickup01.ogg": "https://files.ballistica.net/cache/ba1/44/2a/8535b446284235cb503947ece074",
"assets/build/ba_data/audio/zoeScream01.ogg": "https://files.ballistica.net/cache/ba1/f5/d3/8e941851c4310465646c4167afc1",
"assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/de/c0/7197e7658bef48309741dfaf86c7",
"assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/30/8e/38c5f21b9251ea1111bed554035f",
"assets/build/ba_data/data/langdata.json": "https://files.ballistica.net/cache/ba1/cd/7a/dcf7f0f9436884167abdcb126716",
"assets/build/ba_data/data/languages/arabic.json": "https://files.ballistica.net/cache/ba1/a9/3c/63857b3f5f943205d7a5d8f8e476",
"assets/build/ba_data/data/languages/belarussian.json": "https://files.ballistica.net/cache/ba1/49/5f/b29bb65369040892fe6601801637",
"assets/build/ba_data/data/languages/chinese.json": "https://files.ballistica.net/cache/ba1/ea/14/6485c200c717f82cca7c01dee0b3",
"assets/build/ba_data/data/languages/chinesetraditional.json": "https://files.ballistica.net/cache/ba1/a0/ca/b4e3c4ea2c76e462b7e657b2b1c2",
@ -436,22 +436,22 @@
"assets/build/ba_data/data/languages/gibberish.json": "https://files.ballistica.net/cache/ba1/cb/10/5d94df639e3e0cb405711e1b907f",
"assets/build/ba_data/data/languages/greek.json": "https://files.ballistica.net/cache/ba1/17/78/3fd0dca40e632ce53d03a944e7fa",
"assets/build/ba_data/data/languages/hindi.json": "https://files.ballistica.net/cache/ba1/11/56/ed2b07866104596338f7ce582d64",
"assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/4d/4b/0790110201c9adb1b521e9a55e63",
"assets/build/ba_data/data/languages/hungarian.json": "https://files.ballistica.net/cache/ba1/87/2d/027aa239eb66ea8f496562f4fd83",
"assets/build/ba_data/data/languages/indonesian.json": "https://files.ballistica.net/cache/ba1/a3/b6/8987b9fc6a7cb0f61364f567c901",
"assets/build/ba_data/data/languages/italian.json": "https://files.ballistica.net/cache/ba1/ff/3f/f1c11e1f22454ceceb7b4d277a2a",
"assets/build/ba_data/data/languages/korean.json": "https://files.ballistica.net/cache/ba1/0a/84/bbb6ed2abf66509406f534cbbb52",
"assets/build/ba_data/data/languages/persian.json": "https://files.ballistica.net/cache/ba1/bd/45/ffb2d9d9ce9578ae11de9bb5123d",
"assets/build/ba_data/data/languages/polish.json": "https://files.ballistica.net/cache/ba1/e1/cc/13a4ba409b7d6f113f703be1aa8a",
"assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/9c/e3/834faf6869f5cd175edd4a0244b9",
"assets/build/ba_data/data/languages/portuguese.json": "https://files.ballistica.net/cache/ba1/e5/0a/6ee92ae2cb50f12eeb21059c6659",
"assets/build/ba_data/data/languages/romanian.json": "https://files.ballistica.net/cache/ba1/44/3c/7cc06ca8d5475e1687d0ed05bdbf",
"assets/build/ba_data/data/languages/russian.json": "https://files.ballistica.net/cache/ba1/57/bb/33e8bd738c3ee97122186471be75",
"assets/build/ba_data/data/languages/serbian.json": "https://files.ballistica.net/cache/ba1/e7/d8/ace32888249fc8b8cca0e2edb48b",
"assets/build/ba_data/data/languages/slovak.json": "https://files.ballistica.net/cache/ba1/b7/0a/fab820b96e7aa587ee56427ecdc2",
"assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/c9/6b/dadd4329de69d0b7bfe1aa5f31d6",
"assets/build/ba_data/data/languages/spanish.json": "https://files.ballistica.net/cache/ba1/73/ef/624000057ce55f5a511ba18fd433",
"assets/build/ba_data/data/languages/swedish.json": "https://files.ballistica.net/cache/ba1/50/9f/be006ba19be6a69a57837eb6dca0",
"assets/build/ba_data/data/languages/turkish.json": "https://files.ballistica.net/cache/ba1/82/7c/5363de81ac6823935098df95dc66",
"assets/build/ba_data/data/languages/ukrainian.json": "https://files.ballistica.net/cache/ba1/9d/fb/6c2feb78bdabbec049a71ba40d78",
"assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/f1/00/c4ccd5969084505359e07f927b3a",
"assets/build/ba_data/data/languages/vietnamese.json": "https://files.ballistica.net/cache/ba1/96/96/b2e468ceec8800b7b0f1e1c5977c",
"assets/build/ba_data/data/maps/big_g.json": "https://files.ballistica.net/cache/ba1/47/0a/a617cc85d927b576c4e6fc1091ed",
"assets/build/ba_data/data/maps/bridgit.json": "https://files.ballistica.net/cache/ba1/03/4b/57ee9b42854b26f23f81bd8c58ef",
"assets/build/ba_data/data/maps/courtyard.json": "https://files.ballistica.net/cache/ba1/03/38/344dd05bfef7bbdf464035ec5aa2",
@ -4135,16 +4135,16 @@
"assets/build/windows/x64/vc_redist.x64.exe": "https://files.ballistica.net/cache/ba1/ea/19/8b8787d81abcdce158ba608cd24f",
"assets/build/windows/x64/vcruntime140_1d.dll": "https://files.ballistica.net/cache/ba1/11/d8/ff6344b429b00c24d9a1930d4338",
"assets/build/windows/x64/vcruntime140d.dll": "https://files.ballistica.net/cache/ba1/20/33/0825e11e6518f87ece3009309933",
"build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b2/16/0414b009ce117f73bb2bc8991d3b",
"build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/66/07/d82a3ec80ff3243321e373439210",
"build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/12/5f/f87158c3281a2616897fdd2165cb",
"build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/69/23/a2d029f81db60cf89ad5f6510ddb",
"build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/c4/e3/5aba369252ed1977be8950e5a1ce",
"build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/f7/ef/c28554f863485e9cd54f0d8d4e6f",
"build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/68/af/f060b3463bc31c202a830fb0b20a",
"build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/96/46/a239821d00e782303077192817c5",
"build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/dc/15/fdd9195afe2bf299405ec49ed238",
"build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/68/f9/c42b1f9b5af34cb371f80250e16b",
"build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5d/9b/213c602a9aa9cbc28872d2753f51",
"build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/2f/a4/e6a6a6e0e2b64c5232cccddffdd3"
"build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/05/00/2dd9723909746d8039c6bdf8cddb",
"build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/3e/62/6c310233f4b2c224a46914a4b78e",
"build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/0c/f8/d381993bd58c1f50f0da6d8e2be3",
"build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/a5/b5/d2ec28b09124404731b7f08008c8",
"build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/cb/82/4520a3fec142215ed0673a1811d9",
"build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/b9/0b/060333370a987cee92bdd081a8ca",
"build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/11/31/9f1bb364db47563a1d86f0165549",
"build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/11/62/4d784f02705e438f9bd519ff53f4",
"build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/8a/ff/abcf2602dc64f718408efeaf4d3a",
"build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/04/2d/1f8c7ea677bae4cc7be8e967c344",
"build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/26/b8/f19e24bd0e43ed9661ca62c10e6c",
"build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/02/0b/d2fab33376bc7ae1c8797cae5c0f"
}

View File

@ -1613,6 +1613,7 @@
<w>pythonw</w>
<w>pytree</w>
<w>pytz</w>
<w>pyver</w>
<w>qrcode</w>
<w>qrencode</w>
<w>qual</w>
@ -1788,6 +1789,7 @@
<w>simplesubclasses</w>
<w>sincelaunch</w>
<w>singledispatch</w>
<w>singledispatchmethod</w>
<w>sirplus</w>
<w>sitebuiltins</w>
<w>skey</w>

View File

@ -1,3 +1,6 @@
### 1.5.24 (20159)
- Misc bug fixes..
### 1.5.23 (20146)
- Fixed the shebang line in `bombsquad_server` file by using `-S` flag for `/usr/bin/env`.
- Fixed a bug with hardware keyboards emitting extra characters in the in-game console (~ or F2)

View File

@ -56,7 +56,7 @@ def get_ip_address_type(addr: str) -> socket.AddressFamily:
except OSError:
pass
if socket_type is None:
raise ValueError('addr seems to be neither v4 or v6: ' + str(addr))
raise ValueError(f'addr seems to be neither v4 or v6: {addr}')
return socket_type
@ -152,8 +152,10 @@ class ServerCallThread(threading.Thread):
response_data = None
# Ignore common network errors; note unexpected ones.
if isinstance(exc, (urllib.error.URLError, ConnectionError,
http.client.IncompleteRead)):
if isinstance(
exc,
(urllib.error.URLError, ConnectionError,
http.client.IncompleteRead, http.client.BadStatusLine)):
pass
elif isinstance(exc, OSError):
if exc.errno == 10051: # Windows unreachable network error.

View File

@ -887,10 +887,11 @@ class Bomb(ba.Actor):
def _handle_impact(self) -> None:
node = ba.getcollision().opposingnode
# If we're an impact bomb and we came from this node, don't explode...
# alternately if we're hitting another impact-bomb from the same
# source, don't explode...
# try:
# If we're an impact bomb and we came from this node, don't explode.
# (otherwise we blow up on our own head when jumping).
# Alternately if we're hitting another impact-bomb from the same
# source, don't explode. (can cause accidental explosions if rapidly
# throwing/etc.)
node_delegate = node.getdelegate(object)
if node:
if (self.bomb_type == 'impact' and

View File

@ -1673,7 +1673,7 @@ class GatherWindow(ba.Window):
# Ignore harmless errors.
if exc.errno in {
errno.EHOSTUNREACH, errno.ENETUNREACH,
errno.EINVAL, errno.EPERM
errno.EINVAL, errno.EPERM, errno.EACCES
}:
pass
elif exc.errno == 10022:

View File

@ -838,7 +838,8 @@ class AwaitGamepadInputWindow(ba.Window):
assert isinstance(input_device, ba.InputDevice)
# Update - we now allow *any* input device of this type.
if input_device.exists() and input_device.name == self._input.name:
if (self._input and input_device
and input_device.name == self._input.name):
self._callback(self._capture_button, event, self)
def _decrement(self) -> None:

View File

@ -236,6 +236,11 @@ class ConfigKeyboardWindow(ba.Window):
ba.containerwidget(edit=self._root_widget, transition='out_right')
ba.playsound(ba.getsound('gunCocking'))
# There's a chance the device disappeared; handle that gracefully.
if not self._input:
return
dst = get_input_device_config(self._input, default=False)
dst2: Dict[str, Any] = dst[0][dst[1]]
dst2.clear()

View File

@ -1,5 +1,5 @@
<!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->
<h4><em>last updated on 2020-07-22 for Ballistica version 1.5.23 build 20152</em></h4>
<h4><em>last updated on 2020-07-26 for Ballistica version 1.5.24 build 20158</em></h4>
<p>This page documents the Python classes and functions in the 'ba' module,
which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please <a href="mailto:support@froemling.net">let me know</a>. Happy modding!</p>
<hr>

View File

@ -30,7 +30,7 @@ import subprocess
from functools import partial
from typing import TYPE_CHECKING
from efrotools.pybuild import PYTHON_VERSION_MAJOR
from efrotools import PYVER
if TYPE_CHECKING:
from typing import Optional, List
@ -40,8 +40,7 @@ if TYPE_CHECKING:
# Note: this means anyone wanting to modify .py files in a build
# will need to wipe out the existing .pyc files first or the changes
# will be ignored.
OPT_PYC_SUFFIX = ('cpython-' + PYTHON_VERSION_MAJOR.replace('.', '') +
'.opt-1.pyc')
OPT_PYC_SUFFIX = ('cpython-' + PYVER.replace('.', '') + '.opt-1.pyc')
class Config:

View File

@ -43,6 +43,7 @@ def stage_server_file() -> None:
import batools.build
from efro.error import CleanError
from efrotools import replace_one
from efrotools import PYVER
if len(sys.argv) != 5:
raise CleanError('Expected 3 args (mode, infile, outfile).')
mode, infilename, outfilename = sys.argv[2], sys.argv[3], sys.argv[4]
@ -56,13 +57,15 @@ def stage_server_file() -> None:
# Inject all available config values into the config file.
batools.build.filter_server_config(str(PROJROOT), infilename,
outfilename)
elif basename == 'ballisticacore_server.py':
# Run Python in opt mode for release builds.
with open(infilename) as infile:
lines = infile.read().splitlines()
if mode == 'release':
lines[0] = replace_one(lines[0], '#!/usr/bin/env python3.7',
'#!/usr/bin/env -S python3.7 -O')
lines[0] = replace_one(lines[0],
f'#!/usr/bin/env python{PYVER}',
f'#!/usr/bin/env -S python{PYVER} -O')
with open(outfilename, 'w') as outfile:
outfile.write('\n'.join(lines) + '\n')
subprocess.run(['chmod', '+x', outfilename], check=True)

View File

@ -377,7 +377,7 @@ class Updater:
def _check_python_file(self, fname: str) -> None:
# pylint: disable=too-many-branches
from efrotools import get_public_license
from efrotools import get_public_license, PYVER
with open(fname) as infile:
contents = infile.read()
lines = contents.splitlines()
@ -390,7 +390,7 @@ class Updater:
if fname not in [
'tools/devtool', 'tools/version_utils', 'tools/vmshell'
]:
if not contents.startswith('#!/usr/bin/env python3.7'):
if not contents.startswith(f'#!/usr/bin/env python{PYVER}'):
print(f'{Clr.RED}Incorrect shebang (first line) for '
f'{fname}.{Clr.RST}')
sys.exit(255)

View File

@ -115,9 +115,10 @@ def dispatchmethod(
# All we do here is recreate the end of functools.singledispatch
# where it returns a wrapper except instead of the wrapper using the
# first arg to the function ours uses the second (to skip 'self').
# This was made with Python 3.7; we should probably check up on
# This was made against Python 3.7; we should probably check up on
# this in later versions in case anything has changed.
# (or hopefully they'll add this functionality to their version)
# NOTE: sounds like we can use functools singledispatchmethod in 3.8
def wrapper(*args: Any, **kw: Any) -> Any:
if not args or len(args) < 2:
raise TypeError(f'{funcname} requires at least '

View File

@ -40,8 +40,11 @@ if TYPE_CHECKING:
from typing import Dict, Union, Sequence, Optional, Any
from typing_extensions import Literal
# Python major version we're using for all this stuff.
PYVER = '3.7'
# Python binary assumed by these tools.
PYTHON_BIN = 'python3.7' if platform.system() != 'Windows' else 'python'
PYTHON_BIN = f'python{PYVER}' if platform.system() != 'Windows' else 'python'
MIT_LICENSE = """Copyright (c) 2011-2020 Eric Froemling

View File

@ -86,7 +86,7 @@ def cpplint(projroot: Path, full: bool) -> None:
import tempfile
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count
from efrotools import getconfig
from efrotools import getconfig, PYVER
from efro.terminal import Clr
from efro.error import CleanError
@ -150,7 +150,8 @@ def cpplint(projroot: Path, full: bool) -> None:
def lint_file(filename: str) -> None:
result = subprocess.call(
['python3.7', '-m', 'cpplint', '--root=src', filename], env=env)
[f'python{PYVER}', '-m', 'cpplint', '--root=src', filename],
env=env)
if result != 0:
raise CleanError(
f'{Clr.RED}Cpplint failed for {filename}.{Clr.RST}')
@ -197,7 +198,7 @@ def formatscripts(projroot: Path, full: bool) -> None:
import time
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import cpu_count
from efrotools import get_files_hash
from efrotools import get_files_hash, PYVER
os.chdir(projroot)
cachepath = Path(projroot, 'config/.cache-formatscripts')
if full and cachepath.exists():
@ -215,7 +216,7 @@ def formatscripts(projroot: Path, full: bool) -> None:
def format_file(filename: str) -> None:
start_time = time.time()
result = subprocess.call(
['python3.7', '-m', 'yapf', '--in-place', filename])
[f'python{PYVER}', '-m', 'yapf', '--in-place', filename])
if result != 0:
raise Exception(f'Formatting failed for {filename}')
duration = time.time() - start_time

View File

@ -174,6 +174,12 @@ def spelling() -> None:
_spelling(sys.argv[2:])
def pyver() -> None:
"""Prints the Python version used by this project."""
from efrotools import PYVER
print(PYVER, end='')
def check_clean_safety() -> None:
"""Ensure all files are are added to git or in gitignore.

View File

@ -25,14 +25,11 @@ from __future__ import annotations
import os
from typing import TYPE_CHECKING
import efrotools
from efrotools import PYVER, run, readfile, writefile, replace_one
if TYPE_CHECKING:
from typing import List, Dict, Any
# Overall version we're using for the game currently.
PYTHON_VERSION_MAJOR = '3.7'
ENABLE_OPENSSL = True
@ -51,23 +48,22 @@ def build_apple(arch: str, debug: bool = False) -> None:
raise CleanError('NEED TO TEMP-KILL GETTEXT')
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
efrotools.run('rm -rf "' + builddir + '"')
efrotools.run('mkdir -p build')
efrotools.run('git clone '
'git@github.com:pybee/Python-Apple-support.git "' +
builddir + '"')
run('rm -rf "' + builddir + '"')
run('mkdir -p build')
run('git clone '
'git@github.com:pybee/Python-Apple-support.git "' + builddir + '"')
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# We can actually fix this to use the current one, but something
# broke in the underlying build even on old commits so keeping it
# locked for now...
# efrotools.run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
efrotools.run(f'git checkout {PYTHON_VERSION_MAJOR}')
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
run(f'git checkout {PYVER}')
# On mac we currently have to add the _scproxy module or urllib will
# fail.
txt = efrotools.readfile('patch/Python/Setup.embedded')
txt = readfile('patch/Python/Setup.embedded')
if arch == 'mac':
txt += ('\n'
'# ericf added - mac urllib needs this\n'
@ -76,19 +72,19 @@ def build_apple(arch: str, debug: bool = False) -> None:
'-framework CoreFoundation')
# Turn off sqlite module. (scratch that; leaving it in.)
# txt = efrotools.replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
# txt = replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
# txt = txt.replace(' _sqlite/', '# _sqlite/')
# Turn off xz compression module. (scratch that; leaving it in.)
# txt = efrotools.replace_one(txt, '_lzma _', '#_lzma _')
# txt = replace_one(txt, '_lzma _', '#_lzma _')
# Turn off bzip2 module.
txt = efrotools.replace_one(txt, '_bz2 _b', '#_bz2 _b')
txt = replace_one(txt, '_bz2 _b', '#_bz2 _b')
# Turn off openssl module (only if not doing openssl).
if not ENABLE_OPENSSL:
txt = efrotools.replace_one(txt, '_hashlib _hashopenssl.c',
'#_hashlib _hashopenssl.c')
txt = replace_one(txt, '_hashlib _hashopenssl.c',
'#_hashlib _hashopenssl.c')
# Turn off various other stuff we don't use.
for line in [
@ -123,7 +119,7 @@ def build_apple(arch: str, debug: bool = False) -> None:
'_testimportmultiple _testimportmultiple.c',
'_crypt _cryptmodule.c', # not on android so disabling here too
]:
txt = efrotools.replace_one(txt, line, '#' + line)
txt = replace_one(txt, line, '#' + line)
if ENABLE_OPENSSL:
@ -138,28 +134,28 @@ def build_apple(arch: str, debug: bool = False) -> None:
'_sha256 sha256module.c',
'_sha512 sha512module.c',
]:
txt = efrotools.replace_one(txt, line, '#' + line)
txt = replace_one(txt, line, '#' + line)
else:
txt = efrotools.replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
efrotools.writefile('patch/Python/Setup.embedded', txt)
txt = replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
writefile('patch/Python/Setup.embedded', txt)
txt = efrotools.readfile('Makefile')
txt = readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = efrotools.replace_one(
txt = replace_one(
txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .',
'&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .')
# Remove makefile dependencies so we don't build the
# libs we're not using.
srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: '
txt = efrotools.replace_one(
txt = replace_one(
txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' +
('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') +
'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt)
srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.'
'$(BUILD_NUMBER).tar.gz: ')
txt = efrotools.replace_one(
txt = replace_one(
txt, srctxt,
'dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz:'
' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt)
@ -169,14 +165,14 @@ def build_apple(arch: str, debug: bool = False) -> None:
# Set mac/ios version reqs
# (see issue with utimensat and futimens).
txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.14')
txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.14')
# And equivalent iOS (11+).
txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=12.0')
txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=12.0')
# Ditto for tvOS.
txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
'CFLAGS-tvOS=-mtvos-version-min=12.0')
txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
'CFLAGS-tvOS=-mtvos-version-min=12.0')
if debug:
@ -196,13 +192,13 @@ def build_apple(arch: str, debug: bool = False) -> None:
raise RuntimeError(f'Unexpected configure line count {splitlen}.')
txt = txt.replace(dline, 'python$(PYTHON_VER)dm')
efrotools.writefile('Makefile', txt)
writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
efrotools.run('make -j1 ' + {
run('make -j1 ' + {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
@ -217,11 +213,10 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"""
import subprocess
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
efrotools.run('rm -rf "' + builddir + '"')
efrotools.run('mkdir -p build')
efrotools.run('git clone '
'git@github.com:yan12125/python3-android.git "' + builddir +
'"')
run('rm -rf "' + builddir + '"')
run('mkdir -p build')
run('git clone '
'git@github.com:yan12125/python3-android.git "' + builddir + '"')
os.chdir(builddir)
# It seems we now need 'autopoint' as part of this build, but on mac it
@ -242,69 +237,69 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
# a a completely new minimal one which will take some work to update here.
# Punting on that for now...
if True: # pylint: disable=using-constant-test
efrotools.run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
ftxt = efrotools.readfile('pybuild/env.py')
run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
ftxt = readfile('pybuild/env.py')
# Set the packages we build.
ftxt = efrotools.replace_one(
ftxt = replace_one(
ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," +
(" 'openssl'" if ENABLE_OPENSSL else '') + ')\n# packages = (')
# Don't wanna bother with gpg signing stuff.
ftxt = efrotools.replace_one(ftxt, 'verify_source = True',
'verify_source = False')
ftxt = replace_one(ftxt, 'verify_source = True', 'verify_source = False')
# Sub in the min api level we're targeting.
ftxt = efrotools.replace_one(ftxt, 'android_api_level = 21',
'android_api_level = 21')
ftxt = efrotools.replace_one(ftxt, "target_arch = 'arm'",
"target_arch = '" + arch + "'")
efrotools.writefile('pybuild/env.py', ftxt)
ftxt = efrotools.readfile('Makefile')
ftxt = replace_one(ftxt, 'android_api_level = 21',
'android_api_level = 21')
ftxt = replace_one(ftxt, "target_arch = 'arm'",
"target_arch = '" + arch + "'")
writefile('pybuild/env.py', ftxt)
ftxt = readfile('Makefile')
# This needs to be python3 for us.
ftxt = efrotools.replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
efrotools.writefile('Makefile', ftxt)
ftxt = efrotools.readfile('pybuild/packages/python.py')
ftxt = replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
writefile('Makefile', ftxt)
ftxt = readfile('pybuild/packages/python.py')
# We currently build as a static lib.
ftxt = efrotools.replace_one(ftxt, " '--enable-shared',\n", '')
ftxt = efrotools.replace_one(
ftxt = replace_one(ftxt, " '--enable-shared',\n", '')
ftxt = replace_one(
ftxt, "super().__init__('https://github.com/python/cpython/')",
"super().__init__('https://github.com/python/cpython/', branch='3.7')")
"super().__init__('https://github.com/python/cpython/'"
f", branch='{PYVER}')")
# Turn ipv6 on (curious why its turned off here?...)
# Also, turn on low level debugging features for our debug builds.
ftxt = efrotools.replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
ftxt = replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
if debug:
ftxt = efrotools.replace_one(ftxt, "'./configure',",
"'./configure', '--with-pydebug',")
ftxt = replace_one(ftxt, "'./configure',",
"'./configure', '--with-pydebug',")
# We don't use this stuff so lets strip it out to simplify.
ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", '')
ftxt = replace_one(ftxt, "'--without-ensurepip',", '')
# This builds all modules as dynamic libs, but we want to be consistent
# with our other embedded builds and just static-build the ones we
# need... so to change that we'll need to add a hook for ourself after
# python is downloaded but before it is built so we can muck with it.
ftxt = efrotools.replace_one(
ftxt = replace_one(
ftxt, ' def build(self):',
' def build(self):\n import os\n'
' if os.system(\'"' + rootdir +
'/tools/pcommand" python_android_patch "' + os.getcwd() +
'"\') != 0: raise Exception("patch apply failed")')
efrotools.writefile('pybuild/packages/python.py', ftxt)
writefile('pybuild/packages/python.py', ftxt)
# Set this to a particular cpython commit to target exact releases from git
# commit = 'd7c567b08f9d7d6aef21b881340a2b72731129db' # 3.7.7 release
commit = '4b47a5b6ba66b02df9392feb97b8ead916f8c1fa' # 3.7.8 release
if commit is not None:
ftxt = efrotools.readfile('pybuild/source.py')
ftxt = readfile('pybuild/source.py')
# Check out a particular commit right after the clone.
ftxt = efrotools.replace_one(
ftxt = replace_one(
ftxt, "'git', 'clone', '--single-branch', '-b',"
' self.branch, self.source_url, self.dest])',
"'git', 'clone', '-b',"
@ -315,16 +310,16 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"'https://github.com/python/cpython/':\n"
" run_in_dir(['git', 'checkout', '" + commit +
"'], self.source_dir)")
efrotools.writefile('pybuild/source.py', ftxt)
ftxt = efrotools.readfile('pybuild/util.py')
writefile('pybuild/source.py', ftxt)
ftxt = readfile('pybuild/util.py')
# Still don't wanna bother with gpg signing stuff.
ftxt = efrotools.replace_one(
ftxt = replace_one(
ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n',
'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n'
' print("gpg-verify disabled by ericf")\n'
' return\n')
efrotools.writefile('pybuild/util.py', ftxt)
writefile('pybuild/util.py', ftxt)
# These builds require ANDROID_NDK to be set, so make sure that's
# the case.
@ -335,18 +330,18 @@ def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
# Ok, let 'er rip
# (we often run these builds in parallel so limit to 1 job a piece;
# otherwise they each inherit the -j12 or whatever from the top level).
efrotools.run('make -j1')
run('make -j1')
print('python build complete! (android/' + arch + ')')
def android_patch() -> None:
"""Run necessary patches on an android archive before building."""
fname = 'src/cpython/Modules/Setup.dist'
txt = efrotools.readfile(fname)
txt = readfile(fname)
# Need to switch some flags on this one.
txt = efrotools.replace_one(txt, '#zlib zlibmodule.c',
'zlib zlibmodule.c -lz\n#zlib zlibmodule.c')
txt = replace_one(txt, '#zlib zlibmodule.c',
'zlib zlibmodule.c -lz\n#zlib zlibmodule.c')
# Just turn all these on.
for enable in [
'#array arraymodule.c', '#cmath cmathmodule.c _math.c',
@ -362,10 +357,10 @@ def android_patch() -> None:
'#binascii binascii.c', '#_posixsubprocess _posixsubprocess.c',
'#_sha3 _sha3/sha3module.c'
]:
txt = efrotools.replace_one(txt, enable, enable[1:])
txt = replace_one(txt, enable, enable[1:])
if ENABLE_OPENSSL:
txt = efrotools.replace_one(txt, '#_ssl _ssl.c \\',
'_ssl _ssl.c -DUSE_SSL -lssl -lcrypto')
txt = replace_one(txt, '#_ssl _ssl.c \\',
'_ssl _ssl.c -DUSE_SSL -lssl -lcrypto')
else:
# Note that the _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
@ -374,11 +369,10 @@ def android_patch() -> None:
'#_md5 md5module.c', '#_sha1 sha1module.c',
'#_sha256 sha256module.c', '#_sha512 sha512module.c'
]:
txt = efrotools.replace_one(txt, enable, enable[1:])
txt = replace_one(txt, enable, enable[1:])
# Turn this off (its just an example module).
txt = efrotools.replace_one(txt, 'xxsubtype xxsubtype.c',
'#xxsubtype xxsubtype.c')
txt = replace_one(txt, 'xxsubtype xxsubtype.c', '#xxsubtype xxsubtype.c')
# For whatever reason this stuff isn't in there at all; add it.
txt += '\n_json _json.c\n'
@ -403,7 +397,7 @@ def android_patch() -> None:
txt += '\n\n*disabled*\n_ctypes _crypt grp'
efrotools.writefile(fname, txt)
writefile(fname, txt)
# Ok, this is weird.
# When applying the module Setup, python looks for any line containing *=*
@ -412,17 +406,16 @@ def android_patch() -> None:
# The check used to look for [A-Z]*=* which didn't break, so let' just
# change it back to that for now.
fname = 'src/cpython/Modules/makesetup'
txt = efrotools.readfile(fname)
txt = efrotools.replace_one(
txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;',
' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;')
efrotools.writefile(fname, txt)
txt = readfile(fname)
txt = replace_one(txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;',
' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;')
writefile(fname, txt)
# Add custom callbacks to Python's PyParser_ParseFileObject
# and PyParser_ParseString calls to debug a crash.
fname = 'src/cpython/Parser/parsetok.c'
txt = efrotools.readfile(fname)
txt = efrotools.replace_one(
txt = readfile(fname)
txt = replace_one(
txt, 'node *\n'
'PyParser_ParseFileObject(FILE *fp, PyObject *filename,\n'
' const char *enc, grammar *g, int start,\n'
@ -444,7 +437,7 @@ def android_patch() -> None:
' start, ps1, ps2,\n'
' err_ret, flags);\n'
' }\n')
txt = efrotools.replace_one(
txt = replace_one(
txt, 'node *\n'
'PyParser_ParseStringObject(const char *s, PyObject *filename,\n'
' grammar *g, int start,\n'
@ -462,7 +455,7 @@ def android_patch() -> None:
' PyParser_ParseStringObject_EfroCB(s, filename, g, start,\n'
' err_ret, flags);\n'
' }\n')
efrotools.writefile(fname, txt)
writefile(fname, txt)
print('APPLIED EFROTOOLS ANDROID BUILD PATCHES.')
@ -485,14 +478,14 @@ def gather() -> None:
if d.startswith('pylib-')
]
for existing_dir in existing_dirs:
efrotools.run('rm -rf "' + existing_dir + '"')
run('rm -rf "' + existing_dir + '"')
for buildtype in ['debug', 'release']:
debug = buildtype == 'debug'
bsuffix = '_debug' if buildtype == 'debug' else ''
bsuffix2 = '-debug' if buildtype == 'debug' else ''
libname = 'python' + PYTHON_VERSION_MAJOR + ('dm' if debug else 'm')
libname = 'python' + PYVER + ('dm' if debug else 'm')
bases = {
'mac':
@ -513,19 +506,15 @@ def gather() -> None:
# Note: only need pylib for the first in each group.
builds: List[Dict[str, Any]] = [{
'name':
'macos',
'group':
'apple',
'headers':
bases['mac'] + '/Support/Python/Headers',
'name': 'macos',
'group': 'apple',
'headers': bases['mac'] + '/Support/Python/Headers',
'libs': [
bases['mac'] + '/Support/Python/libPython.a',
bases['mac'] + '/Support/OpenSSL/libOpenSSL.a',
bases['mac'] + '/Support/XZ/libxz.a'
],
'pylib':
(bases['mac'] + '/python/lib/python' + PYTHON_VERSION_MAJOR),
'pylib': (bases['mac'] + '/python/lib/python' + PYVER),
}, {
'name':
'ios',
@ -551,12 +540,9 @@ def gather() -> None:
bases['tvos'] + '/Support/XZ/libxz.a'
],
}, {
'name':
'android_arm',
'group':
'android',
'headers':
bases['android_arm'] + f'/usr/include/{libname}',
'name': 'android_arm',
'group': 'android',
'headers': bases['android_arm'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm'] + f'/usr/lib/lib{libname}.a',
bases['android_arm'] + '/usr/lib/libssl.a',
@ -564,10 +550,8 @@ def gather() -> None:
bases['android_arm'] + '/usr/lib/liblzma.a',
bases['android_arm'] + '/usr/lib/libsqlite3.a'
],
'libinst':
'android_armeabi-v7a',
'pylib': (bases['android_arm'] + '/usr/lib/python' +
PYTHON_VERSION_MAJOR),
'libinst': 'android_armeabi-v7a',
'pylib': (bases['android_arm'] + '/usr/lib/python' + PYVER),
}, {
'name': 'android_arm64',
'group': 'android',
@ -616,16 +600,15 @@ def gather() -> None:
# Do some setup only once per group.
if not os.path.exists(builddir):
efrotools.run('mkdir -p "' + builddir + '"')
efrotools.run('mkdir -p "' + lib_dst + '"')
run('mkdir -p "' + builddir + '"')
run('mkdir -p "' + lib_dst + '"')
# Only pull modules into game assets on release pass.
if not debug:
# Copy system modules into the src assets
# dir for this group.
efrotools.run('mkdir -p "' + assets_src_dst + '"')
efrotools.run(
'rsync --recursive --include "*.py"'
run('mkdir -p "' + assets_src_dst + '"')
run('rsync --recursive --include "*.py"'
' --exclude __pycache__ --include "*/" --exclude "*" "'
+ build['pylib'] + '/" "' + assets_src_dst + '"')
@ -639,30 +622,30 @@ def gather() -> None:
'unittest', 'dbm', 'venv', 'ctypes/test', 'imaplib.py',
'_sysconfigdata_*'
]
efrotools.run('cd "' + assets_src_dst + '" && rm -rf ' +
' '.join(prune))
run('cd "' + assets_src_dst + '" && rm -rf ' +
' '.join(prune))
# Some minor filtering to system scripts:
# on iOS/tvOS, addusersitepackages() leads to a crash
# due to _sysconfigdata_dm_ios_darwin module not existing,
# so let's skip that.
fname = f'{assets_src_dst}/site.py'
txt = efrotools.readfile(fname)
txt = efrotools.replace_one(
txt = readfile(fname)
txt = replace_one(
txt,
' known_paths = addusersitepackages(known_paths)',
' # efro tweak: this craps out on ios/tvos.\n'
' # (and we don\'t use it anyway)\n'
' # known_paths = addusersitepackages(known_paths)')
efrotools.writefile(fname, txt)
writefile(fname, txt)
# Copy in a base set of headers (everything in a group should
# be using the same headers)
efrotools.run(f'cp -r "{build["headers"]}" "{header_dst}"')
run(f'cp -r "{build["headers"]}" "{header_dst}"')
# Clear whatever pyconfigs came across; we'll build our own
# universal one below.
efrotools.run('rm ' + header_dst + '/pyconfig*')
run('rm ' + header_dst + '/pyconfig*')
# Write a master pyconfig header that reroutes to each
# platform's actual header.
@ -701,21 +684,21 @@ def gather() -> None:
# contents too (those headers can themselves include
# others; ios for instance points to a arm64 and a
# x86_64 variant).
contents = efrotools.readfile(build['headers'] + '/' + cfg)
contents = readfile(build['headers'] + '/' + cfg)
contents = contents.replace('pyconfig',
'pyconfig-' + build['name'])
efrotools.writefile(header_dst + '/' + out, contents)
writefile(header_dst + '/' + out, contents)
else:
# other configs we just rename
efrotools.run('cp "' + build['headers'] + '/' + cfg +
'" "' + header_dst + '/' + out + '"')
run('cp "' + build['headers'] + '/' + cfg + '" "' +
header_dst + '/' + out + '"')
# Copy in libs. If the lib gave a specific install name,
# use that; otherwise use name.
targetdir = lib_dst + '/' + build.get('libinst', build['name'])
efrotools.run('rm -rf "' + targetdir + '"')
efrotools.run('mkdir -p "' + targetdir + '"')
run('rm -rf "' + targetdir + '"')
run('mkdir -p "' + targetdir + '"')
for lib in build['libs']:
efrotools.run('cp "' + lib + '" "' + targetdir + '"')
run('cp "' + lib + '" "' + targetdir + '"')
print('Great success!')

View File

@ -42,7 +42,7 @@ from efrotools.pcommand import (
cpplint, pylint, runpylint, mypy, runmypy, dmypy, tool_config_install,
sync, sync_all, scriptfiles, pycharm, clioncode, androidstudiocode,
makefile_target_list, spelling, spelling_all, pytest, echo,
compile_python_files)
compile_python_files, pyver)
from batools.pcommand import (
stage_server_file, py_examine, fix_mac_ssh, check_mac_ssh, resize_image,
check_clean_safety, clean_orphaned_assets, archive_old_builds,