mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-22 23:15:49 +08:00
551 lines
21 KiB
Python
551 lines
21 KiB
Python
# Copyright (c) 2011-2020 Eric Froemling
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
# -----------------------------------------------------------------------------
|
|
"""General functionality related to running builds."""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
from enum import Enum
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from typing import List, Sequence
|
|
|
|
CLRBLU = '\033[94m' # Blue.
|
|
CLRHDR = '\033[95m' # Header.
|
|
CLREND = '\033[0m' # End.
|
|
|
|
# Python modules we require for this project.
|
|
# (module name, required version, pip package (if it differs from module name))
|
|
REQUIRED_PYTHON_MODULES = [
|
|
('pylint', [2, 4, 4], None),
|
|
('mypy', [0, 770], None),
|
|
('yapf', [0, 29, 0], None),
|
|
('typing_extensions', None, None),
|
|
('pytz', None, None),
|
|
('yaml', None, 'PyYAML'),
|
|
('requests', None, None),
|
|
('pytest', None, None),
|
|
]
|
|
|
|
# Parts of full-tests suite we only run on particular days.
|
|
# (This runs in listed order so should be randomized by hand to avoid
|
|
# clustering similar tests too much)
|
|
SPARSE_TEST_BUILDS: List[List[str]] = [
|
|
['ios.pylibs.debug', 'android.pylibs.arm'],
|
|
['linux.package', 'android.pylibs.arm64'],
|
|
['windows.package', 'mac.pylibs'],
|
|
['tvos.pylibs', 'android.pylibs.x86'],
|
|
['android.pylibs.arm.debug'],
|
|
['windows.package.server'],
|
|
['ios.pylibs', 'android.pylibs.arm64.debug'],
|
|
['linux.package.server'],
|
|
['android.pylibs.x86.debug', 'mac.package'],
|
|
['mac.package.server', 'android.pylibs.x86_64'],
|
|
['windows.package.oculus'],
|
|
['android.pylibs.x86_64.debug'],
|
|
['mac.pylibs.debug', 'android.package'],
|
|
]
|
|
|
|
# Currently only doing sparse-tests in core; not spinoffs.
|
|
# (whole word will get subbed out in spinoffs so this will be false)
|
|
DO_SPARSE_TEST_BUILDS = 'ballistica' + 'core' == 'ballisticacore'
|
|
|
|
|
|
class SourceCategory(Enum):
|
|
"""Types of sources."""
|
|
RESOURCES = 'resources_src'
|
|
CODE_GEN = 'code_gen_src'
|
|
ASSETS = 'assets_src'
|
|
CMAKE = 'cmake_src'
|
|
WIN = 'win_src'
|
|
|
|
|
|
class PrefabTarget(Enum):
|
|
"""Types of prefab builds able to be run."""
|
|
DEBUG = 'debug'
|
|
SERVER_DEBUG = 'server-debug'
|
|
RELEASE = 'release'
|
|
SERVER_RELEASE = 'server-release'
|
|
|
|
|
|
def _checkpaths(inpaths: List[str], category: SourceCategory,
|
|
target: str) -> bool:
|
|
# pylint: disable=too-many-branches
|
|
|
|
mtime = None if not os.path.exists(target) else os.path.getmtime(target)
|
|
|
|
if target.startswith('.cache/lazybuild/'):
|
|
tnamepretty = target[len('.cache/lazybuild/'):]
|
|
else:
|
|
tnamepretty = target
|
|
|
|
def _testpath(path: str) -> bool:
|
|
# Now see this path is newer than our target..
|
|
if mtime is None or os.path.getmtime(path) >= mtime:
|
|
print(f'{CLRHDR}Build of {tnamepretty} triggered by'
|
|
f' {path}{CLREND}')
|
|
return True
|
|
return False
|
|
|
|
unchanged_count = 0
|
|
for inpath in inpaths:
|
|
# Add files verbatim; recurse through dirs.
|
|
if os.path.isfile(inpath):
|
|
if _testpath(inpath):
|
|
return True
|
|
unchanged_count += 1
|
|
continue
|
|
for root, _dnames, fnames in os.walk(inpath):
|
|
|
|
# Only gen category uses gen src.
|
|
if (root.startswith('src/generated_src')
|
|
and category is not SourceCategory.CODE_GEN):
|
|
continue
|
|
|
|
# None of our targets use tools-src.
|
|
if root.startswith('src/tools'):
|
|
continue
|
|
|
|
# Skip most of external except for key cases.
|
|
if root.startswith('src/external'):
|
|
if category is SourceCategory.WIN and root.startswith(
|
|
'src/external/windows'):
|
|
pass
|
|
else:
|
|
continue
|
|
|
|
# Ignore python cache files.
|
|
if '__pycache__' in root:
|
|
continue
|
|
for fname in fnames:
|
|
# Ignore dot files
|
|
if fname.startswith('.'):
|
|
continue
|
|
fpath = os.path.join(root, fname)
|
|
if ' ' in fpath:
|
|
raise RuntimeError(f'Invalid path with space: {fpath}')
|
|
|
|
if _testpath(fpath):
|
|
return True
|
|
unchanged_count += 1
|
|
print(f'{CLRBLU}Skipping build of {tnamepretty}'
|
|
f' ({unchanged_count} inputs unchanged){CLREND}')
|
|
return False
|
|
|
|
|
|
def lazy_build(target: str, category: SourceCategory, command: str) -> None:
|
|
"""Run a build if anything in category is newer than target.
|
|
|
|
Note that target's mod-time will always be updated when the build happens
|
|
regardless of whether the build itself did so itself.
|
|
"""
|
|
paths: List[str]
|
|
if category is SourceCategory.CODE_GEN:
|
|
# Everything possibly affecting generated code.
|
|
paths = ['Makefile', 'tools/generate_code', 'src/generated_src']
|
|
elif category is SourceCategory.ASSETS:
|
|
paths = ['Makefile', 'tools/convert_util', 'assets/src']
|
|
elif category is SourceCategory.CMAKE:
|
|
# Everything possibly affecting CMake builds.
|
|
paths = ['Makefile', 'src', 'ballisticacore-cmake/CMakeLists.txt']
|
|
elif category is SourceCategory.WIN:
|
|
# Everything possibly affecting Windows binary builds.
|
|
paths = ['Makefile', 'src', 'resources/src']
|
|
elif category is SourceCategory.RESOURCES:
|
|
# Everything possibly affecting resources builds.
|
|
paths = [
|
|
'Makefile', 'tools/snippets', 'resources/src', 'resources/Makefile'
|
|
]
|
|
else:
|
|
raise ValueError(f'Invalid source category: {category}')
|
|
|
|
# Now do the thing if any our our input mod times changed.
|
|
if _checkpaths(paths, category, target):
|
|
|
|
subprocess.run(command, shell=True, check=True)
|
|
|
|
# We also explicitly update the mod-time of the target;
|
|
# the command we (such as a VM build) may not have actually
|
|
# done anything but we still want to update our target to
|
|
# be newer than all the lazy sources.
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
Path(target).touch()
|
|
|
|
|
|
def archive_old_builds(ssh_server: str, builds_dir: str,
|
|
ssh_args: List[str]) -> None:
|
|
"""Stuff our old public builds into the 'old' dir.
|
|
|
|
(called after we push newer ones)
|
|
"""
|
|
|
|
def ssh_run(cmd: str) -> str:
|
|
val: str = subprocess.check_output(['ssh'] + ssh_args +
|
|
[ssh_server, cmd]).decode()
|
|
return val
|
|
|
|
files = ssh_run('ls -1t "' + builds_dir + '"').splitlines()
|
|
|
|
# For every file we find, gather all the ones with the same prefix;
|
|
# we'll want to archive all but the first one.
|
|
files_to_archive = set()
|
|
for fname in files:
|
|
if '_' not in fname:
|
|
continue
|
|
prefix = '_'.join(fname.split('_')[:-1])
|
|
for old_file in [f for f in files if f.startswith(prefix)][1:]:
|
|
files_to_archive.add(old_file)
|
|
|
|
# Would be faster to package this into a single command but
|
|
# this works.
|
|
for fname in sorted(files_to_archive):
|
|
print('Archiving ' + fname, file=sys.stderr)
|
|
ssh_run('mv "' + builds_dir + '/' + fname + '" "' + builds_dir +
|
|
'/old/"')
|
|
|
|
|
|
def gen_fulltest_buildfile_android() -> None:
|
|
"""Generate fulltest command list for jenkins.
|
|
|
|
(so we see nice pretty split-up build trees)
|
|
"""
|
|
# pylint: disable=too-many-branches
|
|
import datetime
|
|
|
|
# Its a pretty big time-suck building all architectures for
|
|
# all of our subplatforms, so lets usually just build a single one.
|
|
# We'll rotate it though and occasionally do all 4 at once just to
|
|
# be safe.
|
|
dayoffset = datetime.datetime.now().timetuple().tm_yday
|
|
|
|
# Let's only do a full 'prod' once every two times through the loop.
|
|
# (it really should never catch anything that individual platforms don't)
|
|
modes = ['arm', 'arm64', 'x86', 'x86_64']
|
|
modes += modes
|
|
modes.append('prod')
|
|
|
|
lines = []
|
|
for i, flavor in enumerate(
|
|
sorted(os.listdir('ballisticacore-android/BallisticaCore/src'))):
|
|
if flavor == 'main' or flavor.startswith('.'):
|
|
continue
|
|
mode = modes[(dayoffset + i) % len(modes)]
|
|
lines.append('ANDROID_PLATFORM=' + flavor + ' ANDROID_MODE=' + mode +
|
|
' nice -n 15 make android-build')
|
|
|
|
# Now add sparse tests that land on today.
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('android.')]
|
|
for extra in extras:
|
|
if extra == 'android.pylibs.arm':
|
|
lines.append('tools/snippets python_build_android arm')
|
|
elif extra == 'android.pylibs.arm.debug':
|
|
lines.append('tools/snippets python_build_android_debug arm')
|
|
elif extra == 'android.pylibs.arm64':
|
|
lines.append('tools/snippets python_build_android arm64')
|
|
elif extra == 'android.pylibs.arm64.debug':
|
|
lines.append('tools/snippets python_build_android_debug arm64')
|
|
elif extra == 'android.pylibs.x86':
|
|
lines.append('tools/snippets python_build_android x86')
|
|
elif extra == 'android.pylibs.x86.debug':
|
|
lines.append('tools/snippets python_build_android_debug x86')
|
|
elif extra == 'android.pylibs.x86_64':
|
|
lines.append('tools/snippets python_build_android x86_64')
|
|
elif extra == 'android.pylibs.x86_64.debug':
|
|
lines.append(
|
|
'tools/snippets python_build_android_debug x86_64')
|
|
elif extra == 'android.package':
|
|
lines.append('make android-package')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
with open('_fulltest_buildfile_android', 'w') as outfile:
|
|
outfile.write('\n'.join(lines))
|
|
|
|
|
|
def gen_fulltest_buildfile_windows() -> None:
|
|
"""Generate fulltest command list for jenkins.
|
|
|
|
(so we see nice pretty split-up build trees)
|
|
"""
|
|
import datetime
|
|
|
|
dayoffset = datetime.datetime.now().timetuple().tm_yday
|
|
|
|
lines: List[str] = []
|
|
|
|
# We want to do one regular, one headless, and one oculus build,
|
|
# but let's switch up 32 or 64 bit based on the day.
|
|
# Also occasionally throw a release build in but stick to
|
|
# mostly debug builds to keep build times speedier.
|
|
pval1 = 'Win32' if dayoffset % 2 == 0 else 'x64'
|
|
pval2 = 'Win32' if (dayoffset + 1) % 2 == 0 else 'x64'
|
|
pval3 = 'Win32' if (dayoffset + 2) % 2 == 0 else 'x64'
|
|
cfg1 = 'Release' if dayoffset % 7 == 0 else 'Debug'
|
|
cfg2 = 'Release' if (dayoffset + 1) % 7 == 0 else 'Debug'
|
|
cfg3 = 'Release' if (dayoffset + 2) % 7 == 0 else 'Debug'
|
|
|
|
lines.append(f'WINDOWS_PROJECT= WINDOWS_PLATFORM={pval1} '
|
|
f'WINDOWS_CONFIGURATION={cfg1} make windows-build')
|
|
lines.append(f'WINDOWS_PROJECT=Headless WINDOWS_PLATFORM={pval2} '
|
|
f'WINDOWS_CONFIGURATION={cfg2} make windows-build')
|
|
lines.append(f'WINDOWS_PROJECT=Oculus WINDOWS_PLATFORM={pval3} '
|
|
f'WINDOWS_CONFIGURATION={cfg3} make windows-build')
|
|
|
|
# Now add sparse tests that land on today.
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('windows.')]
|
|
for extra in extras:
|
|
if extra == 'windows.package':
|
|
lines.append('make windows-package')
|
|
elif extra == 'windows.package.server':
|
|
lines.append('make windows-server-package')
|
|
elif extra == 'windows.package.oculus':
|
|
lines.append('make windows-oculus-package')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
with open('_fulltest_buildfile_windows', 'w') as outfile:
|
|
outfile.write('\n'.join(lines))
|
|
|
|
|
|
def gen_fulltest_buildfile_apple() -> None:
|
|
"""Generate fulltest command list for jenkins.
|
|
|
|
(so we see nice pretty split-up build trees)
|
|
"""
|
|
# pylint: disable=too-many-branches
|
|
import datetime
|
|
|
|
dayoffset = datetime.datetime.now().timetuple().tm_yday
|
|
|
|
# noinspection PyListCreation
|
|
lines = []
|
|
|
|
# iOS stuff
|
|
lines.append('nice -n 18 make ios-build')
|
|
lines.append('nice -n 18 make ios-new-build')
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('ios.')]
|
|
for extra in extras:
|
|
if extra == 'ios.pylibs':
|
|
lines.append('tools/snippets python_build_apple ios')
|
|
elif extra == 'ios.pylibs.debug':
|
|
lines.append('tools/snippets python_build_apple_debug ios')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
# tvOS stuff
|
|
lines.append('nice -n 18 make tvos-build')
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('tvos.')]
|
|
for extra in extras:
|
|
if extra == 'tvos.pylibs':
|
|
lines.append('tools/snippets python_build_apple tvos')
|
|
elif extra == 'tvos.pylibs.debug':
|
|
lines.append('tools/snippets python_build_apple_debug tvos')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
# macOS stuff
|
|
lines.append('nice -n 18 make mac-build')
|
|
# (throw release build in the mix to hopefully catch opt-mode-only errors).
|
|
lines.append('nice -n 18 make mac-appstore-release-build')
|
|
lines.append('nice -n 18 make mac-new-build')
|
|
lines.append('nice -n 18 make mac-server-build')
|
|
lines.append('nice -n 18 make cmake-build')
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('mac.')]
|
|
for extra in extras:
|
|
if extra == 'mac.package':
|
|
lines.append('make mac-package')
|
|
elif extra == 'mac.package.server':
|
|
lines.append('make mac-server-package')
|
|
elif extra == 'mac.pylibs':
|
|
lines.append('tools/snippets python_build_apple mac')
|
|
elif extra == 'mac.pylibs.debug':
|
|
lines.append('tools/snippets python_build_apple_debug mac')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
with open('_fulltest_buildfile_apple', 'w') as outfile:
|
|
outfile.write('\n'.join(lines))
|
|
|
|
|
|
def gen_fulltest_buildfile_linux() -> None:
|
|
"""Generate fulltest command list for jenkins.
|
|
|
|
(so we see nice pretty split-up build trees)
|
|
"""
|
|
import datetime
|
|
|
|
dayoffset = datetime.datetime.now().timetuple().tm_yday
|
|
|
|
targets = ['build', 'server-build']
|
|
linflav = 'LINUX_FLAVOR=u18s'
|
|
lines = []
|
|
for target in targets:
|
|
lines.append(f'{linflav} make linux-{target}')
|
|
|
|
if DO_SPARSE_TEST_BUILDS:
|
|
extras = SPARSE_TEST_BUILDS[dayoffset % len(SPARSE_TEST_BUILDS)]
|
|
extras = [e for e in extras if e.startswith('linux.')]
|
|
for extra in extras:
|
|
if extra == 'linux.package':
|
|
lines.append(f'{linflav} make linux-package')
|
|
elif extra == 'linux.package.server':
|
|
lines.append(f'{linflav} make linux-server-package')
|
|
else:
|
|
raise RuntimeError(f'Unknown extra: {extra}')
|
|
|
|
with open('_fulltest_buildfile_linux', 'w') as outfile:
|
|
outfile.write('\n'.join(lines))
|
|
|
|
|
|
def get_current_prefab_platform() -> str:
|
|
"""Get the name of the running platform.
|
|
|
|
Throws a RuntimeError on unsupported platforms.
|
|
"""
|
|
import platform
|
|
system = platform.system()
|
|
machine = platform.machine()
|
|
if system == 'Darwin':
|
|
# Currently there's just x86_64 on mac;
|
|
# will need to revisit when arm
|
|
# cpus happen.
|
|
return 'mac'
|
|
if system == 'Linux':
|
|
# If it looks like we're in Windows Subsystem for Linux,
|
|
# go with the Windows version.
|
|
if 'microsoft' in platform.uname()[3].lower():
|
|
return 'windows'
|
|
|
|
# We currently only support x86_64 linux.
|
|
if machine == 'x86_64':
|
|
return 'linux'
|
|
raise RuntimeError(f'make_prefab: unsupported linux machine type:'
|
|
f' {machine}.')
|
|
raise RuntimeError(f'make_prefab: unrecognized platform:'
|
|
f' {platform.system()}.')
|
|
|
|
|
|
def _vstr(nums: Sequence[int]) -> str:
|
|
return '.'.join(str(i) for i in nums)
|
|
|
|
|
|
def checkenv() -> None:
|
|
"""Check for tools necessary to build and run the app."""
|
|
from efrotools import PYTHON_BIN
|
|
print('Checking environment...', flush=True)
|
|
|
|
# Make sure they've got curl.
|
|
if subprocess.run(['which', 'curl'], check=False,
|
|
capture_output=True).returncode != 0:
|
|
raise RuntimeError(f'curl is required; please install it.')
|
|
|
|
# Make sure they've got our target python version.
|
|
if subprocess.run(['which', PYTHON_BIN], check=False,
|
|
capture_output=True).returncode != 0:
|
|
raise RuntimeError(f'{PYTHON_BIN} is required; please install it.')
|
|
|
|
# Make sure they've got pip for that python version.
|
|
if subprocess.run(f"{PYTHON_BIN} -m pip --version",
|
|
shell=True,
|
|
check=False,
|
|
capture_output=True).returncode != 0:
|
|
raise RuntimeError(
|
|
'pip (for {PYTHON_BIN}) is required; please install it.')
|
|
|
|
# Check for some required python modules.
|
|
for modname, minver, packagename in REQUIRED_PYTHON_MODULES:
|
|
if packagename is None:
|
|
packagename = modname
|
|
if minver is not None:
|
|
results = subprocess.run(f'{PYTHON_BIN} -m {modname} --version',
|
|
shell=True,
|
|
check=False,
|
|
capture_output=True)
|
|
else:
|
|
results = subprocess.run(f'{PYTHON_BIN} -c "import {modname}"',
|
|
shell=True,
|
|
check=False,
|
|
capture_output=True)
|
|
if results.returncode != 0:
|
|
raise RuntimeError(
|
|
f'{packagename} (for {PYTHON_BIN}) is required.\n'
|
|
f'To install it, try: "{PYTHON_BIN}'
|
|
f' -m pip install {packagename}"')
|
|
if minver is not None:
|
|
ver_line = results.stdout.decode().splitlines()[0]
|
|
assert modname in ver_line
|
|
vnums = [int(x) for x in ver_line.split()[-1].split('.')]
|
|
assert len(vnums) == len(minver)
|
|
if vnums < minver:
|
|
raise RuntimeError(
|
|
f'{packagename} ver. {_vstr(minver)} or newer required;'
|
|
f' found {_vstr(vnums)}')
|
|
|
|
print('Environment ok.', flush=True)
|
|
|
|
|
|
def get_pip_reqs() -> List[str]:
|
|
"""Return the pip requirements needed to build/run stuff."""
|
|
out: List[str] = []
|
|
for module in REQUIRED_PYTHON_MODULES:
|
|
name = module[0] if module[2] is None else module[2]
|
|
assert isinstance(name, str)
|
|
out.append(name)
|
|
return out
|
|
|
|
|
|
def update_makebob() -> None:
|
|
"""Build fresh make_bob binaries for all relevant platforms."""
|
|
print('Building mac_x86_64...', flush=True)
|
|
env = dict(os.environ)
|
|
env['CMAKE_BUILD_TYPE'] = 'Release'
|
|
subprocess.run(['make', 'cmake-build'], check=True, env=env)
|
|
subprocess.run(
|
|
[
|
|
'cp', '-v', 'ballisticacore-cmake/build/release/make_bob',
|
|
'tools/make_bob/mac_x86_64/'
|
|
],
|
|
check=True,
|
|
)
|
|
print('Building linux_x86_64...', flush=True)
|
|
subprocess.run(['make', 'linux-build'], check=True, env=env)
|
|
subprocess.run(
|
|
[
|
|
'cp', '-v', 'build/linux-release/make_bob',
|
|
'tools/make_bob/linux_x86_64/'
|
|
],
|
|
check=True,
|
|
)
|
|
print('All builds complete!', flush=True)
|