ballistica/tools/update_project

#!/usr/bin/env python3.7
"""
This script acts as a 'meta' Makefile for the project. It is in charge
of generating Makefiles, IDE project files, procedurally generated source
files, etc. based on the current structure of the project.
It can also perform sanity checks or cleanup tasks.

Updating should be explicitly run by the user through commands such as
'make update', 'make check' or 'make preflight'. Other make targets should
avoid running this script as it can modify the project structure
arbitrarily which is not a good idea in the middle of a build.

If the script is invoked with a --check argument, it should not modify any
files but instead fail if any modifications *would* have been made.
(used in CI builds to make sure things are kosher).
"""

from __future__ import annotations

import os
import sys
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import Optional, Tuple, List, Sequence, Dict, Set

CLRHDR = '\033[95m'  # Header.
CLRGRN = '\033[92m'  # Green.
CLRBLU = '\033[94m'  # Glue.
CLRRED = '\033[91m'  # Red.
CLREND = '\033[0m'  # End.


def _find_files(scan_dir: str) -> Tuple[List[str], List[str]]:
    src_files = set()
    header_files = set()
    exts = ['.c', '.cc', '.cpp', '.cxx', '.m', '.mm']
    header_exts = ['.h']

    # Gather all sources and headers.
    for root, _dirs, files in os.walk(scan_dir):
        for ftst in files:
            if any(ftst.endswith(ext) for ext in exts):
                src_files.add(os.path.join(root, ftst)[len(scan_dir):])
            if any(ftst.endswith(ext) for ext in header_exts):
                header_files.add(os.path.join(root, ftst)[len(scan_dir):])
    return sorted(src_files), sorted(header_files)


def _check_files(src_files: Sequence[str]) -> None:

    # A bit of sanity/lint-testing while we're here.
    for fsrc in src_files:
        if fsrc.endswith('.cpp') or fsrc.endswith('.cxx'):
            raise Exception('please use .cc for c++ files; found ' + fsrc)

        # Watch out for in-progress emacs edits.
        # Could just ignore these but it probably means I intended
        # to save something and forgot.
        if '/.#' in fsrc:
            print(f'{CLRRED}'
                  f'ERROR: Found an unsaved emacs file: "{fsrc}"'
                  f'{CLREND}')
            sys.exit(255)


def _check_headers(header_files: Sequence[str], fixable_header_errors: Dict,
                   fix: bool) -> None:
    for header_file_raw in header_files:
        assert header_file_raw[0] == '/'
        header_file = 'src/ballistica' + header_file_raw

        # If its a header-file, lets make sure its define guard is correct.
        if header_file.endswith('.h'):
            guard = (
                header_file[4:].upper().replace('/', '_').replace('.', '_') +
                '_')
            with open(header_file) as fhdr:
                lines = fhdr.read().splitlines()
            expected_lines = [(0, '// Copyright 2019 Eric Froemling'),
                              (2, '#ifndef ' + guard), (3, '#define ' + guard),
                              (-1, '#endif  // ' + guard)]
            errors_found = False
            can_fix = True
            for line, expected in expected_lines:
                if lines[line] != expected:
                    errors_found = True
                    print("Incorrect line " + str(line) + " in " +
                          header_file + ":\n"
                          "Expected: " + expected + "\n"
                          "Found:    " + lines[line])

                    # If the beginning of the line differs,
                    # don't attempt auto-fix.
                    if lines[line][:16] != expected[:16]:
                        can_fix = False
                if errors_found:
                    if can_fix:
                        fixable_header_errors.setdefault(header_file, [])\
                                      .append((line, expected))
                    else:
                        print(CLRRED + "Error found in '" + header_file +
                              "'. Not auto-fixable; please correct manually." +
                              CLREND)
                        sys.exit(255)
    if fixable_header_errors and not fix:
        print(CLRRED +
              'Fixable header error(s) found; pass --fix to correct.' + CLREND)
        sys.exit(255)


def _update_cmake_file(fname: str, src_files: List[str],
                       header_files: List[str],
                       files_to_write: Dict[str, str]) -> None:

    with open(fname) as infile:
        lines = infile.read().splitlines()
    auto_start = lines.index('        #AUTOGENERATED_BEGIN (this section'
                             ' is managed by the "update_project" tool)')
    auto_end = lines.index('        #AUTOGENERATED_END')
    our_lines = [
        '        ${BA_SRC_ROOT}/ballistica' + f
        for f in sorted(src_files + header_files)
        if not f.endswith('.mm') and not f.endswith('.m')
    ]
    filtered = lines[:auto_start + 1] + our_lines + lines[auto_end:]
    files_to_write[fname] = '\n'.join(filtered) + '\n'


def _update_visual_studio_project(fname: str, src_root: str,
                                  src_files: List[str],
                                  header_files: List[str],
                                  files_to_write: Dict[str, str]) -> None:
    # pylint: disable=too-many-locals
    with open(fname) as infile:
        lines = infile.read().splitlines()

    # Hmm can we include headers in the project for easy access?
    # Seems VS attempts to compile them if we do so here.
    # all_files = sorted(src_files + header_files)
    # del header_files  # Unused.
    all_files = sorted([
        f for f in (src_files + header_files) if not f.endswith('.m')
        and not f.endswith('.mm') and not f.endswith('.c')
    ])

    # Find the ItemGroup containing stdafx.cpp. This is where we'll dump
    # our stuff.
    index = lines.index('    <ClCompile Include="stdafx.cpp">')
    begin_index = end_index = index
    while lines[begin_index] != '  <ItemGroup>':
        begin_index -= 1
    while lines[end_index] != '  </ItemGroup>':
        end_index += 1
    group_lines = lines[begin_index + 1:end_index]

    # Strip out any existing files from src/ballistica.
    group_lines = [
        l for l in group_lines if src_root + '\\ballistica\\' not in l
    ]

    # Now add in our own.
    # Note: we can't use C files in this build at the moment; breaks
    # precompiled header stuff. (shouldn't be a problem though).
    group_lines = [
        '    <' +
        ('ClInclude' if src.endswith('.h') else 'ClCompile') + ' Include="' +
        src_root + '\\ballistica' + src.replace('/', '\\') + '" />'
        for src in all_files
    ] + group_lines
    filtered = lines[:begin_index + 1] + group_lines + lines[end_index:]
    files_to_write[fname] = '\r\n'.join(filtered) + '\r\n'

    filterpaths: Set[str] = set()
    filterlines: List[str] = [
        '<?xml version="1.0" encoding="utf-8"?>',
        '<Project ToolsVersion="4.0"'
        ' xmlns="http://schemas.microsoft.com/developer/msbuild/2003">',
        '  <ItemGroup>',
    ]
    sourcelines = [l for l in filtered if 'Include="' + src_root in l]
    for line in sourcelines:
        entrytype = line.strip().split()[0][1:]
        path = line.split('"')[1]
        filterlines.append('    <' + entrytype + ' Include="' + path + '">')

        # If we have a dir foo/bar/eep we need to create filters for
        # each of foo, foo/bar, and foo/bar/eep
        splits = path[len(src_root):].split('\\')
        splits = [s for s in splits if s != '']
        splits = splits[:-1]
        for i in range(len(splits)):
            filterpaths.add('\\'.join(splits[:(i + 1)]))
        filterlines.append('      <Filter>' + '\\'.join(splits) + '</Filter>')
        filterlines.append('    </' + entrytype + '>')
    filterlines += [
        '  </ItemGroup>',
        '  <ItemGroup>',
    ]
    for filterpath in sorted(filterpaths):
        filterlines.append('    <Filter Include="' + filterpath + '" />')
    filterlines += [
        '  </ItemGroup>',
        '</Project>',
    ]

    files_to_write[fname + '.filters'] = '\r\n'.join(filterlines) + '\r\n'


def main() -> None:
    """Main script entry point."""

    # pylint: disable=too-many-locals
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-statements

    # Make sure we're operating from dist root.
    os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..'))

    check = ('--check' in sys.argv)
    fix = ('--fix' in sys.argv)

    checkarg = ' --check' if check else ''

    # Update our python enums module.
    # Should do this before updating asset deps since this is an asset.
    if os.path.exists('tools/update_python_enums_module'):
        if os.system('tools/update_python_enums_module' + checkarg) != 0:
            print(CLRRED + 'Error checking/updating python enums module' +
                  CLREND)
            sys.exit(255)

    # Update our resources Makefile.
    if os.path.exists('tools/update_resources_makefile'):
        if os.system('tools/update_resources_makefile' + checkarg) != 0:
            print(CLRRED + 'Error checking/updating resources Makefile' +
                  CLREND)
            sys.exit(255)

    # Update our generated-code Makefile.
    if os.path.exists('tools/update_generated_code_makefile'):
        if os.system('tools/update_generated_code_makefile' + checkarg) != 0:
            print(CLRRED + 'Error checking/updating generated-code Makefile' +
                  CLREND)
            sys.exit(255)

    # Update our assets Makefile.
    if os.path.exists('tools/update_assets_makefile'):
        if os.system('tools/update_assets_makefile' + checkarg) != 0:
            print(CLRRED + 'Error checking/updating assets Makefile' + CLREND)
            sys.exit(255)

    # Make sure all module dirs in python scripts contain an __init__.py
    scripts_dir = 'assets/src/data/scripts'
    for root, _dirs, files in os.walk(scripts_dir):
        if (root != scripts_dir and '__pycache__' not in root
                and os.path.basename(root) != '.vscode'):
            if '__init__.py' not in files:
                print(CLRRED + 'Error: no __init__.py in package dir: ' +
                      root + CLREND)
                sys.exit(255)

    # Make sure none of our sync targets have been mucked with since
    # their last sync.
    if os.system('tools/snippets sync check') != 0:
        print(CLRRED + 'Sync check failed; you may need to run "sync".' +
              CLREND)
        sys.exit(255)

    # Now go through and update our various build files
    # (MSVC, CMake, Android NDK, etc) as well as sanity-testing/fixing
    # various other files such as headers while we're at it.

    files_to_write: Dict[str, str] = {}

    # Grab sources/headers.
    src_files, header_files = _find_files('src/ballistica')

    # Run some checks on them.
    _check_files(src_files)
    fixable_header_errors: Dict[str, List[Tuple[int, str]]] = {}
    _check_headers(header_files, fixable_header_errors, fix)

    # Now update various builds.

    # CMake
    fname = 'ballisticacore-cmake/CMakeLists.txt'
    if os.path.exists(fname):
        _update_cmake_file(
            fname,
            src_files,
            header_files,
            files_to_write,
        )
    fname = 'ballisticacore-android/BallisticaCore/src/main/cpp/CMakeLists.txt'
    if os.path.exists(fname):
        _update_cmake_file(
            fname,
            src_files,
            header_files,
            files_to_write,
        )

    # Visual Studio
    fname = 'ballisticacore-windows/BallisticaCore/BallisticaCore.vcxproj'
    if os.path.exists(fname):
        _update_visual_studio_project(
            fname,
            '..\\..\\src',
            src_files,
            header_files,
            files_to_write,
        )
    fname = ('ballisticacore-windows/BallisticaCoreHeadless/'
             'BallisticaCoreHeadless.vcxproj')
    if os.path.exists(fname):
        _update_visual_studio_project(
            fname,
            '..\\..\\src',
            src_files,
            header_files,
            files_to_write,
        )
    fname = ('ballisticacore-windows/BallisticaCoreOculus'
             '/BallisticaCoreOculus.vcxproj')
    if os.path.exists(fname):
        _update_visual_studio_project(
            fname,
            '..\\..\\src',
            src_files,
            header_files,
            files_to_write,
        )

    # If we're all good to here, do the actual writes.

    # First, write any header fixes.
    if fixable_header_errors:
        for filename, fixes in fixable_header_errors.items():
            with open(filename, 'r') as infile:
                lines = infile.read().splitlines()
            for fix_line, fix_str in fixes:
                lines[fix_line] = fix_str
            with open(filename, 'w') as outfile:
                outfile.write('\n'.join(lines) + '\n')
                print(CLRBLU + 'Writing header: ' + filename + CLREND)
    else:
        print(f'All {len(header_files)} headers up to date.')

    # Now write out any project files that have changed
    # (or error if we're in check mode).
    unchanged_project_count = 0
    for fname, fcode in files_to_write.items():
        f_orig: Optional[str]
        if os.path.exists(fname):
            with open(fname, 'r') as infile:
                f_orig = infile.read()
        else:
            f_orig = None
        if f_orig == fcode.replace('\r\n', '\n'):
            unchanged_project_count += 1
        else:
            if check:
                print(f'{CLRRED}ERROR: found out-of-date'
                      f' project file: {fname}{CLREND}')
                sys.exit(255)

            print(f'{CLRBLU}Writing project file: {fname}{CLREND}')
            with open(fname, 'w') as outfile:
                outfile.write(fcode)
    if unchanged_project_count > 0:
        print(f'All {unchanged_project_count} project files up to date.')

    # Update our local compile-commands file based on any changes to
    # our cmake stuff. Do this at end so cmake changes already happened.
    if not check and os.path.exists('ballisticacore-cmake'):
        if os.system('make .irony/compile_commands.json') != 0:
            print(CLRRED + 'Error updating compile-commands.' + CLREND)
            sys.exit(255)

    # Lastly update our dummy _ba module.
    # We need to do this very last because it may run the cmake build
    # so its success may depend on the cmake build files having already
    # been updated.
    if os.path.exists('tools/gendummymodule.py'):
        if os.system('tools/gendummymodule.py' + checkarg) != 0:
            print(CLRRED + 'Error checking/updating dummy module' + CLREND)
            sys.exit(255)

    if check:
        print('Check-Builds: Everything up to date.')
    else:
        print('Update-Builds: SUCCESS!')


if __name__ == '__main__':
    main()