mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-22 14:59:25 +08:00
799 lines
32 KiB
Python
799 lines
32 KiB
Python
# Released under the MIT License. See LICENSE for details.
|
|
#
|
|
"""Functionality related to building python for ios, android, etc."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
from typing import TYPE_CHECKING
|
|
|
|
from efrotools import readfile, writefile, replace_exact
|
|
|
|
if TYPE_CHECKING:
|
|
from typing import Any
|
|
|
|
# Python version we build here (not necessarily same as we use in repo).
|
|
PY_VER = '3.10'
|
|
PY_VER_EXACT_ANDROID = '3.10.5'
|
|
PY_VER_EXACT_APPLE = '3.10.4'
|
|
|
|
ANDROID_PYTHON_REPO = 'https://github.com/GRRedWings/python3-android'
|
|
|
|
# Filenames we prune from Python lib dirs in source repo to cut down on size.
|
|
PRUNE_LIB_NAMES = [
|
|
'config-*', 'idlelib', 'lib-dynload', 'lib2to3', 'multiprocessing',
|
|
'pydoc_data', 'site-packages', 'ensurepip', 'tkinter', 'wsgiref',
|
|
'distutils', 'turtle.py', 'turtledemo', 'test', 'sqlite3/test', 'unittest',
|
|
'dbm', 'venv', 'ctypes/test', 'imaplib.py', '_sysconfigdata_*'
|
|
]
|
|
|
|
# Same but for DLLs dir (windows only)
|
|
PRUNE_DLL_NAMES = ['*.ico', '*.pdb']
|
|
|
|
|
|
def build_apple(arch: str, debug: bool = False) -> None:
|
|
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
|
|
import platform
|
|
from efro.error import CleanError
|
|
|
|
# IMPORTANT; seems we currently wind up building against /usr/local gettext
|
|
# stuff. Hopefully the maintainer fixes this, but for now I need to
|
|
# remind myself to blow it away while building.
|
|
# (via brew remove gettext --ignore-dependencies)
|
|
# NOTE: Should check to see if this is still necessary on Apple silicon
|
|
# since homebrew stuff is no longer in /usr/local there.
|
|
if ('MacBook-Fro' in platform.node()
|
|
and os.environ.get('SKIP_GETTEXT_WARNING') != '1'):
|
|
if (subprocess.run('which gettext', shell=True,
|
|
check=False).returncode == 0):
|
|
raise CleanError(
|
|
'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)')
|
|
|
|
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
|
|
subprocess.run(['rm', '-rf', builddir], check=True)
|
|
subprocess.run(['mkdir', '-p', 'build'], check=True)
|
|
subprocess.run(
|
|
[
|
|
'git', 'clone',
|
|
'https://github.com/beeware/Python-Apple-support.git', builddir
|
|
],
|
|
check=True,
|
|
)
|
|
os.chdir(builddir)
|
|
|
|
# TEMP: Check out a particular commit while the branch head is broken.
|
|
# We can actually fix this to use the current one, but something
|
|
# broke in the underlying build even on old commits so keeping it
|
|
# locked for now...
|
|
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
|
|
subprocess.run(['git', 'checkout', PY_VER], check=True)
|
|
|
|
txt = readfile('Makefile')
|
|
|
|
# Turn doc strings on; looks like it only adds a few hundred k.
|
|
txt = replace_exact(txt,
|
|
'--without-doc-strings',
|
|
'--with-doc-strings',
|
|
count=2)
|
|
|
|
# Customize our minimum version requirements
|
|
txt = replace_exact(
|
|
txt,
|
|
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
|
|
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
|
|
)
|
|
txt = replace_exact(
|
|
txt,
|
|
'CFLAGS-iOS=-mios-version-min=12.0 ',
|
|
'CFLAGS-iOS=-mios-version-min=12.0 ',
|
|
)
|
|
txt = replace_exact(
|
|
txt,
|
|
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
|
|
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
|
|
)
|
|
|
|
assert '--with-pydebug' not in txt
|
|
if debug:
|
|
|
|
# Add debug build flag
|
|
txt = replace_exact(
|
|
txt,
|
|
'--enable-ipv6 --without-ensurepip ',
|
|
'--enable-ipv6 --with-pydebug --without-ensurepip ',
|
|
count=2,
|
|
)
|
|
|
|
# Debug lib has a different name.
|
|
txt = replace_exact(txt,
|
|
'python$(PYTHON_VER).a',
|
|
'python$(PYTHON_VER)d.a',
|
|
count=2)
|
|
|
|
txt = replace_exact(txt,
|
|
'/include/python$(PYTHON_VER)',
|
|
'/include/python$(PYTHON_VER)d',
|
|
count=4)
|
|
|
|
# Inject our custom modifications to fire right after their normal
|
|
# Setup.local filtering and right before building (and pass the same
|
|
# 'slice' value they use so we can use it too).
|
|
txt = replace_exact(
|
|
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
|
|
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n',
|
|
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
|
|
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n'
|
|
'\tcd $$(PYTHON_DIR-$(target)) && '
|
|
f'../../../../../tools/pcommand python_apple_patch {arch} '
|
|
'"$$(SLICE-$$(SDK-$(target)))"\n')
|
|
txt = replace_exact(
|
|
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
|
|
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n',
|
|
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
|
|
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n'
|
|
'\tcd $$(PYTHON_DIR-$(os)) && '
|
|
f'../../../../../tools/pcommand python_apple_patch {arch} '
|
|
'"$$(SLICE-macosx)"\n')
|
|
|
|
writefile('Makefile', txt)
|
|
|
|
# Ok; let 'er rip.
|
|
# (we run these in parallel so limit to 1 job a piece;
|
|
# otherwise they inherit the -j12 or whatever from the top level)
|
|
# (also this build seems to fail with multiple threads)
|
|
subprocess.run(
|
|
[
|
|
'make', '-j1', {
|
|
'mac': 'Python-macOS',
|
|
'ios': 'Python-iOS',
|
|
'tvos': 'Python-tvOS'
|
|
}[arch]
|
|
],
|
|
check=True,
|
|
)
|
|
print('python build complete! (apple/' + arch + ')')
|
|
|
|
|
|
def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
|
|
"""Run a build for android with the given architecture.
|
|
|
|
(can be arm, arm64, x86, or x86_64)
|
|
"""
|
|
|
|
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
|
|
subprocess.run(['rm', '-rf', builddir], check=True)
|
|
subprocess.run(['mkdir', '-p', 'build'], check=True)
|
|
subprocess.run(
|
|
['git', 'clone', ANDROID_PYTHON_REPO, builddir],
|
|
check=True,
|
|
)
|
|
os.chdir(builddir)
|
|
|
|
# If we need to use a particular branch.
|
|
if bool(False):
|
|
subprocess.run(['git', 'checkout', PY_VER_EXACT_ANDROID], check=True)
|
|
|
|
# These builds require ANDROID_NDK to be set; make sure that's the case.
|
|
os.environ['ANDROID_NDK'] = subprocess.check_output(
|
|
[f'{rootdir}/tools/pcommand', 'android_sdk_utils',
|
|
'get-ndk-path']).decode().strip()
|
|
|
|
# Disable builds for dependencies we don't use.
|
|
ftxt = readfile('Android/build_deps.py')
|
|
ftxt = replace_exact(
|
|
ftxt,
|
|
' '
|
|
'BZip2, GDBM, LibFFI, LibUUID, OpenSSL, Readline, SQLite, XZ, ZLib,\n',
|
|
' '
|
|
'BZip2, LibUUID, OpenSSL, SQLite, XZ, ZLib,\n',
|
|
)
|
|
|
|
# Give ourselves a handle to patch the OpenSSL build.
|
|
ftxt = replace_exact(
|
|
ftxt,
|
|
' # OpenSSL handles NDK internal paths by itself',
|
|
' # Ericf addition: do some patching:\n'
|
|
' self.run(["../../../../../../../tools/pcommand",'
|
|
' "python_android_patch_ssl"])\n'
|
|
' # OpenSSL handles NDK internal paths by itself',
|
|
)
|
|
|
|
writefile('Android/build_deps.py', ftxt)
|
|
|
|
# Tweak some things in the base build script; grab the right version
|
|
# of Python and also inject some code to modify bits of python
|
|
# after it is extracted.
|
|
ftxt = readfile('build.sh')
|
|
|
|
ftxt = replace_exact(ftxt, 'PYVER=3.10.5', f'PYVER={PY_VER_EXACT_ANDROID}')
|
|
ftxt = replace_exact(
|
|
ftxt, ' popd\n', f' ../../../tools/pcommand'
|
|
f' python_android_patch Python-{PY_VER_EXACT_ANDROID}\n popd\n')
|
|
writefile('build.sh', ftxt)
|
|
|
|
# Ok; let 'er rip!
|
|
exargs = ' --with-pydebug' if debug else ''
|
|
subprocess.run(f'ARCH={arch} ANDROID_API=21 ./build.sh{exargs}',
|
|
shell=True,
|
|
check=True)
|
|
print('python build complete! (android/' + arch + ')')
|
|
|
|
|
|
def apple_patch(arch: str, slc: str) -> None:
|
|
"""Run necessary patches on an apple archive before building."""
|
|
|
|
# Here's the deal: we want our custom static python libraries to
|
|
# be as similar as possible on apple platforms and android, so let's
|
|
# blow away all the tweaks that this setup does to Setup.local and
|
|
# instead apply our very similar ones directly to Setup, just as we
|
|
# do for android.
|
|
with open('Modules/Setup.local', 'w', encoding='utf-8') as outfile:
|
|
outfile.write('# cleared by efrotools build\n')
|
|
|
|
_patch_setup_file('apple', arch, slc)
|
|
_patch_py_ssl()
|
|
|
|
|
|
def android_patch() -> None:
|
|
"""Run necessary patches on an android archive before building."""
|
|
_patch_setup_file('android', '?', '?')
|
|
_patch_py_ssl()
|
|
|
|
|
|
def android_patch_ssl() -> None:
|
|
"""Run necessary patches on an android ssl before building."""
|
|
|
|
# We bundle our own SSL root certificates on various platforms and use
|
|
# the OpenSSL 'SSL_CERT_FILE' env var override to get them to be used
|
|
# by default. However, OpenSSL is picky about allowing env-vars to be
|
|
# used and something about the Android environment makes it disallow
|
|
# them. So we need to force the issue. Alternately we could explicitly
|
|
# pass 'cafile' args to SSLContexts whenever we do network-y stuff
|
|
# but it seems cleaner to just have things work everywhere by default.
|
|
fname = 'crypto/getenv.c'
|
|
txt = readfile(fname)
|
|
txt = replace_exact(
|
|
txt,
|
|
('char *ossl_safe_getenv(const char *name)\n'
|
|
'{\n'),
|
|
('char *ossl_safe_getenv(const char *name)\n'
|
|
'{\n'
|
|
' // ERICF TWEAK: ALWAYS ALLOW GETENV.\n'
|
|
' return getenv(name);\n'),
|
|
)
|
|
writefile(fname, txt)
|
|
|
|
# Getting a lot of crashes in _armv7_tick, which seems to be a
|
|
# somewhat known issue with certain arm7 devices. Sounds like
|
|
# there are no major downsides to disabling this feature, so doing that.
|
|
# (Sounds like its possible to somehow disable it through an env var
|
|
# but let's just be sure and #ifdef it out in the source.
|
|
# see https://github.com/openssl/openssl/issues/17465
|
|
fname = 'crypto/armcap.c'
|
|
txt = readfile(fname)
|
|
txt = replace_exact(
|
|
txt,
|
|
' /* Things that getauxval didn\'t tell us */\n'
|
|
' if (sigsetjmp(ill_jmp, 1) == 0) {\n'
|
|
' _armv7_tick();\n'
|
|
' OPENSSL_armcap_P |= ARMV7_TICK;\n'
|
|
' }\n',
|
|
'# if 0 // ericf disabled; causing crashes on some android devices.\n'
|
|
' /* Things that getauxval didn\'t tell us */\n'
|
|
' if (sigsetjmp(ill_jmp, 1) == 0) {\n'
|
|
' _armv7_tick();\n'
|
|
' OPENSSL_armcap_P |= ARMV7_TICK;\n'
|
|
' }\n'
|
|
'# endif // 0\n',
|
|
)
|
|
writefile(fname, txt)
|
|
|
|
|
|
def _patch_py_ssl() -> None:
|
|
|
|
# I've tracked down an issue where Python's SSL module
|
|
# can spend lots of time in SSL_CTX_set_default_verify_paths()
|
|
# while holding the GIL, which hitches the game like crazy.
|
|
# On debug builds on older Android devices it can spend up to
|
|
# 1-2 seconds there. So its necessary to release the GIL during that
|
|
# call to keep things smooth. Will submit a report/patch to the
|
|
# Python folks, but for now am just patching it for our Python builds.
|
|
# NOTE TO SELF: It would also be good to look into why that call can be
|
|
# so slow and if there's anything we can do about that.
|
|
# UPDATE: This should be fixed in Python itself as of 3.10.6
|
|
# (see https://github.com/python/cpython/issues/94637)
|
|
# UPDATE 2: Have confirmed that call is expected to be slow in some
|
|
# situations.
|
|
fname = 'Modules/_ssl.c'
|
|
txt = readfile(fname)
|
|
txt = replace_exact(
|
|
txt,
|
|
' if (!SSL_CTX_set_default_verify_paths(self->ctx)) {',
|
|
' int ret = 0;\n'
|
|
'\n'
|
|
' PySSL_BEGIN_ALLOW_THREADS\n'
|
|
' ret = SSL_CTX_set_default_verify_paths(self->ctx);\n'
|
|
' PySSL_END_ALLOW_THREADS\n'
|
|
'\n'
|
|
' if (!ret) {',
|
|
)
|
|
writefile(fname, txt)
|
|
|
|
|
|
def _patch_setup_file(platform: str, arch: str, slc: str) -> None:
|
|
# pylint: disable=too-many-locals
|
|
# pylint: disable=too-many-statements
|
|
|
|
fname = 'Modules/Setup'
|
|
ftxt = readfile(fname)
|
|
|
|
if platform == 'android':
|
|
prefix = '$(srcdir)/Android/sysroot/usr'
|
|
uuid_ex = f' -L{prefix}/lib -luuid'
|
|
zlib_ex = f' -I{prefix}/include -L{prefix}/lib -lz'
|
|
bz2_ex = f' -I{prefix}/include -L{prefix}/lib -lbz2'
|
|
ssl_ex = f' -DUSE_SSL -I{prefix}/include -L{prefix}/lib -lssl -lcrypto'
|
|
sqlite_ex = f' -I{prefix}/include -L{prefix}/lib'
|
|
hash_ex = ' -DUSE_SSL -lssl -lcrypto'
|
|
lzma_ex = ' -llzma'
|
|
elif platform == 'apple':
|
|
# These should basically match what the Python-Apple-support dist
|
|
# does in its patch/Python/Setup.embedded.
|
|
# (We do our own thing and ignore the Setup.local it generates
|
|
# for the sake of cross-platform consistency, but still need to
|
|
# do what we do the same way they do what they do)
|
|
|
|
def _slrp(val: str) -> str:
|
|
# In the distro, the Makefile does this filtering to Setup.local
|
|
return val.replace('{{slice}}', slc)
|
|
|
|
uuid_ex = ''
|
|
zlib_ex = ' -I$(prefix)/include -lz'
|
|
bz2_ex = _slrp(
|
|
' -I$(srcdir)/../Support/BZip2.xcframework/{{slice}}/Headers'
|
|
' -L$(srcdir)/../Support/BZip2.xcframework/{{slice}} -lbzip2')
|
|
ssl_ex = _slrp(
|
|
' -I$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}/Headers'
|
|
' -L$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}'
|
|
' -lOpenSSL -DUSE_SSL')
|
|
sqlite_ex = ' -I$(srcdir)/Modules/_sqlite'
|
|
hash_ex = _slrp(
|
|
' -I$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}/Headers'
|
|
' -L$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}'
|
|
' -lOpenSSL -DUSE_SSL')
|
|
lzma_ex = _slrp(
|
|
' -I$(srcdir)/../Support/XZ.xcframework/{{slice}}/Headers'
|
|
' -L$(srcdir)/../Support/XZ.xcframework/{{slice}} -lxz')
|
|
else:
|
|
raise RuntimeError(f'Unknown platform {platform}')
|
|
|
|
# This list should contain all possible compiled modules to start.
|
|
# If any .so files are coming out of builds, their names should be
|
|
# added here to stop that.
|
|
cmodules = [
|
|
'_asyncio', '_bisect', '_blake2', '_codecs_cn', '_codecs_hk',
|
|
'_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw',
|
|
'_contextvars', '_crypt', '_csv', '_ctypes_test', '_ctypes',
|
|
'_curses_panel', '_curses', '_datetime', '_decimal', '_elementtree',
|
|
'_heapq', '_json', '_lsprof', '_lzma', '_md5', '_multibytecodec',
|
|
'_multiprocessing', '_opcode', '_pickle', '_posixsubprocess', '_queue',
|
|
'_random', '_sha1', '_sha3', '_sha256', '_sha512', '_socket',
|
|
'_statistics', '_struct', '_testbuffer', '_testcapi',
|
|
'_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_uuid',
|
|
'_xxsubinterpreters', '_xxtestfuzz', '_zoneinfo', 'array', 'audioop',
|
|
'binascii', 'cmath', 'fcntl', 'grp', 'math', 'mmap', 'ossaudiodev',
|
|
'parser', 'pyexpat', 'resource', 'select', 'syslog', 'termios',
|
|
'unicodedata', 'xxlimited', 'zlib'
|
|
]
|
|
|
|
# Selectively uncomment some existing modules for static compilation.
|
|
enables = [
|
|
'_asyncio', 'array', 'cmath', 'math', '_contextvars', '_struct',
|
|
'_random', '_elementtree', '_pickle', '_datetime', '_zoneinfo',
|
|
'_bisect', '_heapq', '_json', '_statistics', 'unicodedata', 'fcntl',
|
|
'select', 'mmap', '_csv', '_socket', '_sha3', '_blake2', 'binascii',
|
|
'_posixsubprocess'
|
|
]
|
|
# Note that the _md5 and _sha modules are normally only built if the
|
|
# system does not have the OpenSSL libs containing an optimized
|
|
# version.
|
|
if bool(False):
|
|
enables += ['_md5']
|
|
|
|
for enable in enables:
|
|
ftxt = replace_exact(ftxt, f'#{enable} ', f'{enable} ')
|
|
cmodules.remove(enable)
|
|
|
|
# Disable ones that were enabled:
|
|
disables = ['xxsubtype']
|
|
for disable in disables:
|
|
ftxt = replace_exact(ftxt, f'\n{disable} ', f'\n#{disable} ')
|
|
|
|
# Additions:
|
|
ftxt += '\n# Additions by efrotools:\n'
|
|
|
|
if bool(True):
|
|
ftxt += f'_uuid _uuidmodule.c{uuid_ex}\n'
|
|
cmodules.remove('_uuid')
|
|
|
|
ftxt += f'zlib zlibmodule.c{zlib_ex}\n'
|
|
|
|
cmodules.remove('zlib')
|
|
|
|
# Why isn't this getting built as a shared lib by default?
|
|
# Do we need it for sure?
|
|
ftxt += f'_hashlib _hashopenssl.c{hash_ex}\n'
|
|
|
|
ftxt += f'_lzma _lzmamodule.c{lzma_ex}\n'
|
|
cmodules.remove('_lzma')
|
|
|
|
ftxt += f'_bz2 _bz2module.c{bz2_ex}\n'
|
|
|
|
ftxt += f'_ssl _ssl.c{ssl_ex}\n'
|
|
|
|
ftxt += (f'_sqlite3'
|
|
f' _sqlite/cache.c'
|
|
f' _sqlite/connection.c'
|
|
f' _sqlite/cursor.c'
|
|
f' _sqlite/microprotocols.c'
|
|
f' _sqlite/module.c'
|
|
f' _sqlite/prepare_protocol.c'
|
|
f' _sqlite/row.c'
|
|
f' _sqlite/statement.c'
|
|
f' _sqlite/util.c'
|
|
f'{sqlite_ex}'
|
|
f' -DMODULE_NAME=\'\\"sqlite3\\"\''
|
|
f' -DSQLITE_OMIT_LOAD_EXTENSION'
|
|
f' -lsqlite3\n')
|
|
|
|
# Mac needs this:
|
|
if arch == 'mac':
|
|
ftxt += ('\n'
|
|
'# efrotools: mac urllib needs this:\n'
|
|
'_scproxy _scproxy.c '
|
|
'-framework SystemConfiguration '
|
|
'-framework CoreFoundation\n')
|
|
|
|
# Explicitly mark the remaining ones as disabled
|
|
# (so Python won't try to build them as dynamic libs).
|
|
remaining_disabled = ' '.join(cmodules)
|
|
ftxt += ('\n# Disabled by efrotools build:\n'
|
|
'*disabled*\n'
|
|
f'{remaining_disabled}\n')
|
|
writefile(fname, ftxt)
|
|
|
|
# Ok, this is weird.
|
|
# When applying the module Setup, python looks for any line containing *=*
|
|
# and interprets the whole thing a a global define?...
|
|
# This breaks things for our static sqlite compile above.
|
|
# The check used to look for [A-Z]*=* which didn't break, so let' just
|
|
# change it back to that for now.
|
|
# UPDATE: Currently this seems to only be necessary on Android;
|
|
# perhaps this broke between 3.9.6 and 3.9.7 or perhaps the apple
|
|
# bundle already patches it ¯\_(ツ)_/¯
|
|
fname = 'Modules/makesetup'
|
|
txt = readfile(fname)
|
|
if platform == 'android':
|
|
txt = replace_exact(txt, ' *=*)'
|
|
' DEFS="$line$NL$DEFS"; continue;;',
|
|
' [A-Z]*=*) DEFS="$line$NL$DEFS";'
|
|
' continue;;')
|
|
assert txt.count('[A-Z]*=*') == 1
|
|
writefile(fname, txt)
|
|
|
|
|
|
def winprune() -> None:
|
|
"""Prune unneeded files from windows python dists."""
|
|
for libdir in ('assets/src/windows/Win32/Lib',
|
|
'assets/src/windows/x64/Lib'):
|
|
assert os.path.isdir(libdir)
|
|
assert (' ' not in name for name in PRUNE_LIB_NAMES)
|
|
subprocess.run(f'cd "{libdir}" && rm -rf ' + ' '.join(PRUNE_LIB_NAMES),
|
|
shell=True,
|
|
check=True)
|
|
# Kill python cache dirs.
|
|
subprocess.run(
|
|
f'find "{libdir}" -name __pycache__ -print0 | xargs -0 rm -rf',
|
|
shell=True,
|
|
check=True)
|
|
for dlldir in ('assets/src/windows/Win32/DLLs',
|
|
'assets/src/windows/x64/DLLs'):
|
|
assert os.path.isdir(dlldir)
|
|
assert (' ' not in name for name in PRUNE_DLL_NAMES)
|
|
subprocess.run(f'cd "{dlldir}" && rm -rf ' + ' '.join(PRUNE_DLL_NAMES),
|
|
shell=True,
|
|
check=True)
|
|
|
|
print('Win-prune successful.')
|
|
|
|
|
|
def gather() -> None:
|
|
"""Gather per-platform python headers, libs, and modules together.
|
|
|
|
This assumes all embeddable py builds have been run successfully,
|
|
and that PROJROOT is the cwd.
|
|
"""
|
|
# pylint: disable=too-many-locals
|
|
# pylint: disable=too-many-statements
|
|
|
|
do_android = True
|
|
|
|
# First off, clear out any existing output.
|
|
existing_dirs = [
|
|
os.path.join('src/external', d) for d in os.listdir('src/external')
|
|
if d.startswith('python-') and d != 'python-notes.txt'
|
|
]
|
|
existing_dirs += [
|
|
os.path.join('assets/src', d) for d in os.listdir('assets/src')
|
|
if d.startswith('pylib-')
|
|
]
|
|
if not do_android:
|
|
existing_dirs = [d for d in existing_dirs if 'android' not in d]
|
|
|
|
for existing_dir in existing_dirs:
|
|
subprocess.run(['rm', '-rf', existing_dir], check=True)
|
|
|
|
apost2 = f'src/Python-{PY_VER_EXACT_ANDROID}/Android/sysroot'
|
|
for buildtype in ['debug', 'release']:
|
|
debug = buildtype == 'debug'
|
|
bsuffix = '_debug' if buildtype == 'debug' else ''
|
|
bsuffix2 = '-debug' if buildtype == 'debug' else ''
|
|
libname = 'python' + PY_VER + ('d' if debug else '')
|
|
|
|
bases = {
|
|
'mac': f'build/python_apple_mac{bsuffix}/build/macOS',
|
|
'ios': f'build/python_apple_ios{bsuffix}/build/iOS',
|
|
'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS',
|
|
'android_arm': f'build/python_android_arm{bsuffix}/build',
|
|
'android_arm64': f'build/python_android_arm64{bsuffix}/build',
|
|
'android_x86': f'build/python_android_x86{bsuffix}/build',
|
|
'android_x86_64': f'build/python_android_x86_64{bsuffix}/build'
|
|
}
|
|
bases2 = {
|
|
'android_arm': f'build/python_android_arm{bsuffix}/{apost2}',
|
|
'android_arm64': f'build/python_android_arm64{bsuffix}/{apost2}',
|
|
'android_x86': f'build/python_android_x86{bsuffix}/{apost2}',
|
|
'android_x86_64': f'build/python_android_x86_64{bsuffix}/{apost2}'
|
|
}
|
|
|
|
# Note: only need pylib for the first in each group.
|
|
mac_arch_dir = 'macos-arm64_x86_64'
|
|
ios_arch_dir = 'ios-arm64'
|
|
tvos_arch_dir = 'tvos-arm64'
|
|
builds: list[dict[str, Any]] = [{
|
|
'name':
|
|
'macos',
|
|
'group':
|
|
'apple',
|
|
'headers':
|
|
bases['mac'] +
|
|
f'/Support/Python.xcframework/{mac_arch_dir}/Headers',
|
|
'libs': [
|
|
bases['mac'] +
|
|
f'/Support/Python.xcframework/{mac_arch_dir}/libPython.a',
|
|
bases['mac'] +
|
|
f'/Support/OpenSSL.xcframework/{mac_arch_dir}/libOpenSSL.a',
|
|
bases['mac'] +
|
|
f'/Support/XZ.xcframework/{mac_arch_dir}/libxz.a',
|
|
bases['mac'] +
|
|
f'/Support/BZip2.xcframework/{mac_arch_dir}/libbzip2.a',
|
|
],
|
|
'pylib':
|
|
(bases['mac'] + f'/Python-{PY_VER_EXACT_APPLE}-macOS/lib'),
|
|
}, {
|
|
'name':
|
|
'ios',
|
|
'group':
|
|
'apple',
|
|
'headers':
|
|
bases['ios'] +
|
|
f'/Support/Python.xcframework/{ios_arch_dir}/Headers',
|
|
'libs': [
|
|
bases['ios'] +
|
|
f'/Support/Python.xcframework/{ios_arch_dir}/libPython.a',
|
|
bases['ios'] +
|
|
f'/Support/OpenSSL.xcframework/{ios_arch_dir}/libOpenSSL.a',
|
|
bases['ios'] +
|
|
f'/Support/XZ.xcframework/{ios_arch_dir}/libxz.a',
|
|
bases['ios'] +
|
|
f'/Support/BZip2.xcframework/{ios_arch_dir}/libbzip2.a',
|
|
],
|
|
}, {
|
|
'name':
|
|
'tvos',
|
|
'group':
|
|
'apple',
|
|
'headers':
|
|
bases['tvos'] +
|
|
f'/Support/Python.xcframework/{tvos_arch_dir}/Headers',
|
|
'libs': [
|
|
bases['tvos'] +
|
|
f'/Support/Python.xcframework/{tvos_arch_dir}/libPython.a',
|
|
bases['tvos'] +
|
|
f'/Support/OpenSSL.xcframework/{tvos_arch_dir}/libOpenSSL.a',
|
|
bases['tvos'] +
|
|
f'/Support/XZ.xcframework/{tvos_arch_dir}/libxz.a',
|
|
bases['tvos'] +
|
|
f'/Support/BZip2.xcframework/{tvos_arch_dir}/libbzip2.a',
|
|
],
|
|
}, {
|
|
'name': 'android_arm',
|
|
'group': 'android',
|
|
'headers': bases['android_arm'] + f'/usr/include/{libname}',
|
|
'libs': [
|
|
bases['android_arm'] + f'/usr/lib/lib{libname}.a',
|
|
bases2['android_arm'] + '/usr/lib/libssl.a',
|
|
bases2['android_arm'] + '/usr/lib/libcrypto.a',
|
|
bases2['android_arm'] + '/usr/lib/liblzma.a',
|
|
bases2['android_arm'] + '/usr/lib/libsqlite3.a',
|
|
bases2['android_arm'] + '/usr/lib/libbz2.a',
|
|
bases2['android_arm'] + '/usr/lib/libuuid.a',
|
|
],
|
|
'libinst': 'android_armeabi-v7a',
|
|
'pylib': (bases['android_arm'] + '/usr/lib/python' + PY_VER),
|
|
}, {
|
|
'name': 'android_arm64',
|
|
'group': 'android',
|
|
'headers': bases['android_arm64'] + f'/usr/include/{libname}',
|
|
'libs': [
|
|
bases['android_arm64'] + f'/usr/lib/lib{libname}.a',
|
|
bases2['android_arm64'] + '/usr/lib/libssl.a',
|
|
bases2['android_arm64'] + '/usr/lib/libcrypto.a',
|
|
bases2['android_arm64'] + '/usr/lib/liblzma.a',
|
|
bases2['android_arm64'] + '/usr/lib/libsqlite3.a',
|
|
bases2['android_arm64'] + '/usr/lib/libbz2.a',
|
|
bases2['android_arm64'] + '/usr/lib/libuuid.a',
|
|
],
|
|
'libinst': 'android_arm64-v8a',
|
|
}, {
|
|
'name': 'android_x86',
|
|
'group': 'android',
|
|
'headers': bases['android_x86'] + f'/usr/include/{libname}',
|
|
'libs': [
|
|
bases['android_x86'] + f'/usr/lib/lib{libname}.a',
|
|
bases2['android_x86'] + '/usr/lib/libssl.a',
|
|
bases2['android_x86'] + '/usr/lib/libcrypto.a',
|
|
bases2['android_x86'] + '/usr/lib/liblzma.a',
|
|
bases2['android_x86'] + '/usr/lib/libsqlite3.a',
|
|
bases2['android_x86'] + '/usr/lib/libbz2.a',
|
|
bases2['android_x86'] + '/usr/lib/libuuid.a',
|
|
],
|
|
'libinst': 'android_x86',
|
|
}, {
|
|
'name': 'android_x86_64',
|
|
'group': 'android',
|
|
'headers': bases['android_x86_64'] + f'/usr/include/{libname}',
|
|
'libs': [
|
|
bases['android_x86_64'] + f'/usr/lib/lib{libname}.a',
|
|
bases2['android_x86_64'] + '/usr/lib/libssl.a',
|
|
bases2['android_x86_64'] + '/usr/lib/libcrypto.a',
|
|
bases2['android_x86_64'] + '/usr/lib/liblzma.a',
|
|
bases2['android_x86_64'] + '/usr/lib/libsqlite3.a',
|
|
bases2['android_x86_64'] + '/usr/lib/libbz2.a',
|
|
bases2['android_x86_64'] + '/usr/lib/libuuid.a',
|
|
],
|
|
'libinst': 'android_x86_64',
|
|
}]
|
|
|
|
for build in builds:
|
|
grp = build['group']
|
|
if not do_android and grp == 'android':
|
|
continue
|
|
builddir = f'src/external/python-{grp}{bsuffix2}'
|
|
header_dst = os.path.join(builddir, 'include')
|
|
lib_dst = os.path.join(builddir, 'lib')
|
|
assets_src_dst = f'assets/src/pylib-{grp}'
|
|
|
|
# Do some setup only once per group.
|
|
if not os.path.exists(builddir):
|
|
subprocess.run(['mkdir', '-p', builddir], check=True)
|
|
subprocess.run(['mkdir', '-p', lib_dst], check=True)
|
|
|
|
# Only pull modules into game assets on release pass.
|
|
if not debug:
|
|
# Copy system modules into the src assets
|
|
# dir for this group.
|
|
subprocess.run(['mkdir', '-p', assets_src_dst], check=True)
|
|
subprocess.run(
|
|
[
|
|
'rsync', '--recursive', '--include', '*.py',
|
|
'--exclude', '__pycache__', '--include', '*/',
|
|
'--exclude', '*', build['pylib'] + '/',
|
|
assets_src_dst
|
|
],
|
|
check=True,
|
|
)
|
|
|
|
# Prune a bunch of modules we don't need to cut
|
|
# down on size.
|
|
subprocess.run('cd "' + assets_src_dst + '" && rm -rf ' +
|
|
' '.join(PRUNE_LIB_NAMES),
|
|
shell=True,
|
|
check=True)
|
|
|
|
# Some minor filtering to system scripts:
|
|
# on iOS/tvOS, addusersitepackages() leads to a crash
|
|
# due to _sysconfigdata_dm_ios_darwin module not existing,
|
|
# so let's skip that.
|
|
fname = f'{assets_src_dst}/site.py'
|
|
txt = readfile(fname)
|
|
txt = replace_exact(
|
|
txt,
|
|
' known_paths = addusersitepackages(known_paths)',
|
|
' # efro tweak: this craps out on ios/tvos.\n'
|
|
' # (and we don\'t use it anyway)\n'
|
|
' # known_paths = addusersitepackages(known_paths)')
|
|
writefile(fname, txt)
|
|
|
|
# Copy in a base set of headers (everything in a group should
|
|
# be using the same headers)
|
|
subprocess.run(f'cp -r "{build["headers"]}" "{header_dst}"',
|
|
shell=True,
|
|
check=True)
|
|
|
|
# Clear whatever pyconfigs came across; we'll build our own
|
|
# universal one below.
|
|
subprocess.run('rm ' + header_dst + '/pyconfig*',
|
|
shell=True,
|
|
check=True)
|
|
|
|
# Write a master pyconfig header that reroutes to each
|
|
# platform's actual header.
|
|
with open(header_dst + '/pyconfig.h', 'w',
|
|
encoding='utf-8') as hfile:
|
|
hfile.write(
|
|
'#if BA_OSTYPE_MACOS\n'
|
|
'#include "pyconfig-macos.h"\n\n'
|
|
'#elif BA_OSTYPE_IOS\n'
|
|
'#include "pyconfig-ios.h"\n\n'
|
|
'#elif BA_OSTYPE_TVOS\n'
|
|
'#include "pyconfig-tvos.h"\n\n'
|
|
'#elif BA_OSTYPE_ANDROID and defined(__arm__)\n'
|
|
'#include "pyconfig-android_arm.h"\n\n'
|
|
'#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n'
|
|
'#include "pyconfig-android_arm64.h"\n\n'
|
|
'#elif BA_OSTYPE_ANDROID and defined(__i386__)\n'
|
|
'#include "pyconfig-android_x86.h"\n\n'
|
|
'#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n'
|
|
'#include "pyconfig-android_x86_64.h"\n\n'
|
|
'#else\n'
|
|
'#error unknown platform\n\n'
|
|
'#endif\n')
|
|
|
|
# Now copy each build's config headers in with unique names.
|
|
cfgs = [
|
|
f for f in os.listdir(build['headers'])
|
|
if f.startswith('pyconfig')
|
|
]
|
|
|
|
# Copy config headers to their filtered names.
|
|
for cfg in cfgs:
|
|
out = cfg.replace('pyconfig', 'pyconfig-' + build['name'])
|
|
if cfg == 'pyconfig.h':
|
|
|
|
# For platform's root pyconfig.h we need to filter
|
|
# contents too (those headers can themselves include
|
|
# others; ios for instance points to a arm64 and a
|
|
# x86_64 variant).
|
|
contents = readfile(build['headers'] + '/' + cfg)
|
|
contents = contents.replace('pyconfig',
|
|
'pyconfig-' + build['name'])
|
|
writefile(header_dst + '/' + out, contents)
|
|
else:
|
|
# other configs we just rename
|
|
subprocess.run('cp "' + build['headers'] + '/' + cfg +
|
|
'" "' + header_dst + '/' + out + '"',
|
|
shell=True,
|
|
check=True)
|
|
|
|
# Copy in libs. If the lib gave a specific install name,
|
|
# use that; otherwise use name.
|
|
targetdir = lib_dst + '/' + build.get('libinst', build['name'])
|
|
subprocess.run(['rm', '-rf', targetdir], check=True)
|
|
subprocess.run(['mkdir', '-p', targetdir], check=True)
|
|
for lib in build['libs']:
|
|
subprocess.run(['cp', lib, targetdir], check=True)
|
|
|
|
print('Great success!')
|