ballistica/tools/snippets

#!/usr/bin/env python3.7
# Copyright (c) 2011-2020 Eric Froemling
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# -----------------------------------------------------------------------------
"""Wee little snippets of functionality specific to this project.

All top level functions here can be run by passing them as the first
argument on the command line. (or pass no arguments to get a list of them).

Functions can be placed here when they're not complex enough to warrant
their own files. Often these functions act as user-facing entry points
to functionality contained in efrotools or other standalone tool modules.
"""

from __future__ import annotations

import os
import sys
from typing import TYPE_CHECKING

import efrotools

# Pull in some standard snippets we want to expose.
# noinspection PyUnresolvedReferences
from efrotools.snippets import (  # pylint: disable=unused-import
    PROJROOT, CleanError, snippets_main, formatcode, formatscripts,
    formatmakefile, cpplint, pylint, mypy, dmypy, tool_config_install, sync,
    sync_all, scriptfiles, pycharm, clioncode, androidstudiocode,
    makefile_target_list, spelling, spelling_all, compile_python_files, pytest)

if TYPE_CHECKING:
    from typing import Optional


def archive_old_builds() -> None:
    """Stuff our old public builds into the 'old' dir.

    (called after we push newer ones)
    """
    import batools.build
    if len(sys.argv) < 3:
        raise Exception('invalid arguments')
    ssh_server = sys.argv[2]
    builds_dir = sys.argv[3]
    ssh_args = sys.argv[4:]
    batools.build.archive_old_builds(ssh_server, builds_dir, ssh_args)


def gen_fulltest_buildfile_android() -> None:
    """Generate fulltest command list for jenkins.

    (so we see nice pretty split-up build trees)
    """
    import batools.build
    batools.build.gen_fulltest_buildfile_android()


def gen_fulltest_buildfile_windows() -> None:
    """Generate fulltest command list for jenkins.

    (so we see nice pretty split-up build trees)
    """
    import batools.build
    batools.build.gen_fulltest_buildfile_windows()


def gen_fulltest_buildfile_apple() -> None:
    """Generate fulltest command list for jenkins.

    (so we see nice pretty split-up build trees)
    """
    import batools.build
    batools.build.gen_fulltest_buildfile_apple()


def gen_fulltest_buildfile_linux() -> None:
    """Generate fulltest command list for jenkins.

    (so we see nice pretty split-up build trees)
    """
    import batools.build
    batools.build.gen_fulltest_buildfile_linux()


def resize_image() -> None:
    """Resize an image and save it to a new location.

    args: xres, yres, src, dst
    """
    if len(sys.argv) != 6:
        raise Exception("expected 5 args")
    width = int(sys.argv[2])
    height = int(sys.argv[3])
    src = sys.argv[4]
    dst = sys.argv[5]
    if not dst.endswith('.png'):
        raise Exception("dst must be a png")
    if not src.endswith('.png'):
        raise Exception("src must be a png")
    print('Creating: ' + os.path.basename(dst), file=sys.stderr)

    # Switching to imagemagick from sips - hopefully this goes well
    # so we'll be nice and cross-platform...
    efrotools.run('convert "' + src + '" -resize ' + str(width) + 'x' +
                  str(height) + ' "' + dst + '"')


def check_clean_safety() -> None:
    """Ensure all files are are added to git or in gitignore.

    Use to avoid losing work if we accidentally do a clean without
    adding something.
    """
    from efrotools.snippets import check_clean_safety as std_snippet

    # First do standard checks.
    std_snippet()

    # Then also make sure there are no untracked changes to core files
    # (since we may be blowing core away here).
    spinoff_bin = os.path.join(str(PROJROOT), 'tools', 'spinoff')
    if os.path.exists(spinoff_bin):
        status = os.system(spinoff_bin + ' cleancheck')
        if status != 0:
            sys.exit(255)


def get_master_asset_src_dir() -> None:
    """Print master-asset-source dir for this repo."""
    import subprocess

    # Ok, for now lets simply use our hard-coded master-src
    # path if we're on master in and not otherwise.  Should
    # probably make this configurable.
    output = subprocess.check_output(
        ['git', 'status', '--branch', '--porcelain']).decode()

    # Also compare repo name to split version of itself to
    # see if we're outside of core (filtering will cause mismatch if so).
    if ('origin/master' in output.splitlines()[0]
            and 'ballistica' + 'core' == 'ballisticacore'):

        # We seem to be in master in core repo.. lets do it.
        print('/Users/ericf/Dropbox/ballisticacore_master_assets')
    else:
        # Still need to supply dummy path for makefile if not..
        print('/__DUMMY_MASTER_SRC_DISABLED_PATH__')


def androidaddr() -> None:
    """Return the source file location for an android program-counter.

    command line args: archive_dir architecture addr
    """
    import subprocess
    if len(sys.argv) != 5:
        print('ERROR: expected 4 args; got ' + str(len(sys.argv) - 1) + '.\n' +
              'Usage: "tools/snippets android_addr'
              ' <ARCHIVE-PATH> <ARCH> <ADDR>"')
        sys.exit(255)
    archive_dir = sys.argv[2]
    if not os.path.isdir(archive_dir):
        print('ERROR: invalid archive dir: "' + archive_dir + '"')
        sys.exit(255)
    arch = sys.argv[3]
    archs = {
        'x86': {'prefix': 'x86-', 'libmain': 'libmain_x86.so'},
        'arm': {'prefix': 'arm-', 'libmain': 'libmain_arm.so'},
        'arm64': {'prefix': 'aarch64-', 'libmain': 'libmain_arm64.so'},
        'x86-64': {'prefix': 'x86_64-', 'libmain': 'libmain_x86-64.so'}
    }  # yapf: disable
    if arch not in archs:
        print('ERROR: invalid arch "' + arch + '"; (choices are ' +
              ', '.join(archs.keys()) + ')')
        sys.exit(255)
    addr = sys.argv[4]
    sdkutils = os.path.abspath(
        os.path.join(os.path.dirname(sys.argv[0]), 'android_sdk_utils'))
    rootdir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
    ndkpath = subprocess.check_output([sdkutils,
                                       'get-ndk-path']).decode().strip()
    if not os.path.isdir(ndkpath):
        print("ERROR: ndk-path '" + ndkpath + '" does not exist')
        sys.exit(255)
    lines = subprocess.check_output(
        ['find',
         os.path.join(ndkpath, 'toolchains'), '-name',
         '*addr2line']).decode().strip().splitlines()
    lines = [line for line in lines if archs[arch]['prefix'] in line]
    if len(lines) != 1:
        print("ERROR: couldn't find addr2line binary")
        sys.exit(255)
    addr2line = lines[0]
    efrotools.run('mkdir -p "' + os.path.join(rootdir, 'android_addr_tmp') +
                  '"')
    try:
        efrotools.run('cd "' + os.path.join(rootdir, 'android_addr_tmp') +
                      '" && tar -xf "' +
                      os.path.join(archive_dir, 'unstripped_libs',
                                   archs[arch]['libmain'] + '.tgz') + '"')
        efrotools.run(
            addr2line + ' -e "' +
            os.path.join(rootdir, 'android_addr_tmp', archs[arch]['libmain']) +
            '" ' + addr)
    finally:
        os.system('rm -rf "' + os.path.join(rootdir, 'android_addr_tmp') + '"')


def python_build_apple() -> None:
    """Build an embeddable python for mac/ios/tvos."""
    _python_build_apple(debug=False)


def python_build_apple_debug() -> None:
    """Build embeddable python for mac/ios/tvos (dbg ver)."""
    _python_build_apple(debug=True)


def _python_build_apple(debug: bool) -> None:
    """Build an embeddable python for macOS/iOS/tvOS."""
    from efrotools import pybuild
    os.chdir(PROJROOT)
    archs = ('mac', 'ios', 'tvos')
    if len(sys.argv) != 3:
        print("ERROR: expected one <ARCH> arg: " + ', '.join(archs))
        sys.exit(255)
    arch = sys.argv[2]
    if arch not in archs:
        print('ERROR: invalid arch. valid values are: ' + ', '.join(archs))
        sys.exit(255)
    pybuild.build_apple(arch, debug=debug)


def python_build_android() -> None:
    """Build an embeddable Python lib for Android."""
    _python_build_android(debug=False)


def python_build_android_debug() -> None:
    """Build embeddable Android Python lib (debug ver)."""
    _python_build_android(debug=True)


def _python_build_android(debug: bool) -> None:
    from efrotools import pybuild
    os.chdir(PROJROOT)
    archs = ('arm', 'arm64', 'x86', 'x86_64')
    if len(sys.argv) != 3:
        print("ERROR: expected one <ARCH> arg: " + ', '.join(archs))
        sys.exit(255)
    arch = sys.argv[2]
    if arch not in archs:
        print('ERROR: invalid arch. valid values are: ' + ', '.join(archs))
        sys.exit(255)
    pybuild.build_android(str(PROJROOT), arch, debug=debug)


def python_android_patch() -> None:
    """Patches Python to prep for building for Android."""
    from efrotools import pybuild
    os.chdir(sys.argv[2])
    pybuild.android_patch()


def python_gather() -> None:
    """Gather build python components into the project.

    This assumes all embeddable py builds have been run successfully.
    """
    from efrotools import pybuild
    os.chdir(PROJROOT)
    pybuild.gather()


def clean_orphaned_assets() -> None:
    """Remove asset files that are no longer part of the build."""
    import json

    # Operate from dist root..
    os.chdir(PROJROOT)

    # Our manifest is split into 2 files (public and non-public)
    with open('assets/.asset_manifest_1.json') as infile:
        manifest = set(json.loads(infile.read()))
    with open('assets/.asset_manifest_2.json') as infile:
        manifest.update(set(json.loads(infile.read())))
    for root, _dirs, fnames in os.walk('assets/build'):
        for fname in fnames:
            fpath = os.path.join(root, fname)
            fpathrel = fpath[13:]  # paths are relative to assets/build
            if fpathrel not in manifest:
                print(f"Removing orphaned asset file: {fpath}")
                os.unlink(fpath)

    # Lastly, clear empty dirs.
    efrotools.run('find assets/build -depth -empty -type d -delete')


def py_examine() -> None:
    """Run a python examination at a given point in a given file."""
    from pathlib import Path
    if len(sys.argv) != 7:
        print('ERROR: expected 7 args')
        sys.exit(255)
    filename = Path(sys.argv[2])
    line = int(sys.argv[3])
    column = int(sys.argv[4])
    selection: Optional[str] = (None if sys.argv[5] == '' else sys.argv[5])
    operation = sys.argv[6]

    # This stuff assumes it is being run from project root.
    os.chdir(PROJROOT)

    # Set up pypaths so our main distro stuff works.
    scriptsdir = os.path.abspath(
        os.path.join(os.path.dirname(sys.argv[0]),
                     '../assets/src/ba_data/python'))
    toolsdir = os.path.abspath(
        os.path.join(os.path.dirname(sys.argv[0]), '../tools'))
    if scriptsdir not in sys.path:
        sys.path.append(scriptsdir)
    if toolsdir not in sys.path:
        sys.path.append(toolsdir)
    efrotools.py_examine(filename, line, column, selection, operation)


def push_ipa() -> None:
    """Construct and push ios IPA for testing."""
    from pathlib import Path
    from efrotools import ios
    root = Path(sys.argv[0], '../..').resolve()
    if len(sys.argv) != 3:
        raise Exception('expected 1 arg (debug or release)')
    modename = sys.argv[2]
    ios.push_ipa(root, modename)


def fix_mac_ssh() -> None:
    """Turn off mac ssh password access.

    (This totally doesn't belong in this project btw..)
    """
    configpath = '/etc/ssh/sshd_config'
    with open(configpath) as infile:
        lines = infile.readlines()
    index = lines.index('#PasswordAuthentication yes\n')
    lines[index] = 'PasswordAuthentication no\n'
    index = lines.index('#ChallengeResponseAuthentication yes\n')
    lines[index] = 'ChallengeResponseAuthentication no\n'
    index = lines.index('UsePAM yes\n')
    lines[index] = 'UsePAM no\n'
    with open(configpath, 'w') as outfile:
        outfile.write(''.join(lines))
    print('SSH config updated successfully!')


def check_mac_ssh() -> None:
    """Make sure ssh password access is turned off.

    (This totally doesn't belong here, but I use it it to remind myself to
    fix mac ssh after system updates which blow away ssh customizations).
    """
    with open('/etc/ssh/sshd_config') as infile:
        lines = infile.read().splitlines()
    if ('UsePAM yes' in lines or '#PasswordAuthentication yes' in lines
            or '#ChallengeResponseAuthentication yes' in lines):
        print('ERROR: ssh config is allowing password access.\n'
              'To fix: sudo tools/snippets fix_mac_ssh')
        sys.exit(255)
    print('password ssh auth seems disabled; hooray!')


def capitalize() -> None:
    """Print args capitalized."""
    print(' '.join(w.capitalize() for w in sys.argv[2:]))


def efrocache_update() -> None:
    """Build & push files to efrocache for public access."""
    from efrotools.efrocache import update_cache
    makefile_dirs = ['', 'assets']
    update_cache(makefile_dirs)


def efrocache_get() -> None:
    """Get a file from efrocache."""
    from efrotools.efrocache import get_target
    if len(sys.argv) != 3:
        raise RuntimeError('Expected exactly 1 arg')
    get_target(sys.argv[2])


def get_modern_make() -> None:
    """Print name of a modern make command."""
    import platform
    import subprocess

    # Mac gnu make is outdated (due to newer versions using GPL3 I believe).
    # so let's return 'gmake' there which will point to homebrew make which
    # should be up to date.
    if platform.system() == 'Darwin':
        if subprocess.run(['which', 'gmake'], check=False,
                          capture_output=True).returncode != 0:
            print(
                "WARNING: this requires gmake (mac system make is too old)."
                " Install it with 'brew install make'",
                file=sys.stderr,
                flush=True)
        print('gmake')
    else:
        print('make')


def warm_start_asset_build() -> None:
    """Prep asset builds to run faster."""
    import subprocess
    from pathlib import Path
    from efrotools import get_config
    public: bool = get_config(PROJROOT)['public']

    if public:
        from efrotools.efrocache import warm_start_cache
        os.chdir(PROJROOT)
        warm_start_cache()
    else:
        # For internal builds we don't use efrocache but we do use an
        # internal build cache. Download an initial cache/etc. if need be.
        subprocess.run(
            [str(Path(PROJROOT, 'tools/convert_util')), '--init-asset-cache'],
            check=True)


def update_docs_md() -> None:
    """Updates docs markdown files if necessary."""
    # pylint: disable=too-many-locals
    from efrotools import get_files_hash, run

    check = ('--check' in sys.argv)

    docs_path = 'docs/ba_module.md'

    # We store the hash in a separate file that only exists on private
    # so public isn't full of constant hash change commits.
    # (don't care so much on private)
    docs_hash_path = 'docs/ba_module_hash'

    # Generate a hash from all c/c++ sources under the python subdir
    # as well as all python scripts.
    pysources = []
    exts = ['.cc', '.c', '.h', '.py']
    for basedir in [
            'src/ballistica/python',
            'assets/src/ba_data/python/efro',
            'assets/src/ba_data/python/bacommon',
            'assets/src/ba_data/python/ba',
    ]:
        assert os.path.isdir(basedir)
        for root, _dirs, files in os.walk(basedir):
            for fname in files:
                if any(fname.endswith(ext) for ext in exts):
                    pysources.append(os.path.join(root, fname))
    curhash = get_files_hash(pysources)

    # Extract the current embedded hash.
    with open(docs_hash_path) as infile:
        storedhash = infile.read()

    if curhash != storedhash or not os.path.exists(docs_path):
        if check:
            raise CleanError('Docs markdown is out of date.')

        print(f'Updating {docs_path}...', flush=True)
        run('make docs')

        # Our docs markdown is just the docs html with a few added
        # bits at the top.
        with open('build/docs.html') as infile:
            docs = infile.read()
        docs = ('<!-- THIS FILE IS AUTO GENERATED; DO NOT EDIT BY HAND -->\n'
                ) + docs
        with open(docs_path, 'w') as outfile:
            outfile.write(docs)
        with open(docs_hash_path, 'w') as outfile:
            outfile.write(curhash)
    print(f'{docs_path} is up to date.')


def list_pip_reqs() -> None:
    """List Python Pip packages needed for this project."""
    from batools.build import get_pip_reqs
    print(' '.join(get_pip_reqs()))


def install_pip_reqs() -> None:
    """Install Python Pip packages needed for this project."""
    import subprocess
    from efrotools import PYTHON_BIN
    from batools.build import get_pip_reqs
    subprocess.run([PYTHON_BIN, '-m', 'pip', 'install', '--upgrade'] +
                   get_pip_reqs(),
                   check=True)
    print(f'All pip requirements installed!')


def checkenv() -> None:
    """Check for tools necessary to build and run the app."""
    import batools.build
    try:
        batools.build.checkenv()
    except RuntimeError as exc:
        raise CleanError(exc)


def ensure_prefab_platform() -> None:
    """Ensure we are running on a particular prefab platform."""
    import batools.build
    if len(sys.argv) != 3:
        raise CleanError('Expected 1 platform name arg.')
    needed = sys.argv[2]
    current = batools.build.get_current_prefab_platform()
    if current != needed:
        raise CleanError(
            f'Incorrect platform: we are {current}, this requires {needed}.')


def prefab_run_var() -> None:
    """Print a variable for running a prefab run for the current platform.

    We use this mechanism instead of just having a command recursively run
    a make target so that ctrl-c can be handled cleanly and directly by the
    command getting run instead of generating extra errors in the recursive
    processes.
    """
    import batools.build
    if len(sys.argv) != 3:
        raise RuntimeError('Expected 1 arg.')
    base = sys.argv[2].replace('-', '_').upper()
    platform = batools.build.get_current_prefab_platform().upper()
    print(f'RUN_PREFAB_{platform}_{base}', end='')


def make_prefab() -> None:
    """Run prefab builds for the current platform."""
    import subprocess
    import batools.build
    if len(sys.argv) != 3:
        raise RuntimeError('Expected one argument')
    target = batools.build.PrefabTarget(sys.argv[2])
    platform = batools.build.get_current_prefab_platform()
    try:
        subprocess.run(['make', f'prefab-{platform}-{target.value}-build'],
                       check=True)
    except (Exception, KeyboardInterrupt) as exc:
        if str(exc):
            print(f'make_prefab failed with error: {exc}')
        sys.exit(-1)


def update_makebob() -> None:
    """Build fresh make_bob binaries for all relevant platforms."""
    import batools.build
    batools.build.update_makebob()


def lazybuild() -> None:
    """Run a build command only if an input has changed."""
    import subprocess
    import batools.build
    if len(sys.argv) < 5:
        raise CleanError('Expected at least 3 args')
    try:
        category = batools.build.SourceCategory(sys.argv[2])
    except ValueError as exc:
        raise CleanError(exc)
    target = sys.argv[3]
    command = ' '.join(sys.argv[4:])
    try:
        batools.build.lazy_build(target, category, command)
    except subprocess.CalledProcessError as exc:
        raise CleanError(exc)


if __name__ == '__main__':
    snippets_main(globals())