ballistica/tools/update_project

#!/usr/bin/env python3.7
# Copyright (c) 2011-2019 Eric Froemling
"""
This script acts as a 'meta' Makefile for the project. It is in charge
of generating Makefiles, IDE project files, procedurally generated source
files, etc. based on the current structure of the project.
It can also perform sanity checks or cleanup tasks.

Updating should be explicitly run by the user through commands such as
'make update', 'make check' or 'make preflight'. Other make targets should
avoid running this script as it can modify the project structure
arbitrarily which is not a good idea in the middle of a build.

If the script is invoked with a --check argument, it should not modify any
files but instead fail if any modifications *would* have been made.
(used in CI builds to make sure things are kosher).
"""

from __future__ import annotations

import os
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import Optional, Tuple, List, Dict, Set

LEGAL_NOTICE_PRIVATE = 'Copyright (c) 2011-2019 Eric Froemling'

CLRHDR = '\033[95m'  # Header.
CLRGRN = '\033[92m'  # Green.
CLRBLU = '\033[94m'  # Glue.
CLRRED = '\033[91m'  # Red.
CLREND = '\033[0m'  # End.


@dataclass
class LineChange:
    """A change applying to a particular line in a file."""
    line_number: int
    expected: str
    can_auto_update: bool


class App:
    """Context for an app run."""

    def __init__(self) -> None:
        from efrotools import get_config
        from pathlib import Path
        self._check = ('--check' in sys.argv)
        self._fix = ('--fix' in sys.argv)
        self._checkarg = ' --check' if self._check else ''

        # We behave differently in the public repo
        self._public = get_config(Path('.'))['public']
        assert isinstance(self._public, bool)

        self._source_files: List[str] = []
        self._header_files: List[str] = []

        self._line_checks: Dict[str, List[LineChange]] = {}
        self._file_changes: Dict[str, str] = {}

    def run(self) -> None:
        """Do the thing."""

        # Make sure we're operating from a project root.
        if not os.path.isdir('config') or not os.path.isdir('tools'):
            raise Exception('This must be run from a project root.')

        # NOTE: Do py-enums before updating asset deps since it is an asset.
        self._update_python_enums_module()
        self._update_resources_makefile()
        self._update_generated_code_makefile()
        self._update_assets_makefile()

        self._check_python_files()
        self._check_sync_states()

        self._find_sources_and_headers('src/ballistica')
        self._check_source_files()
        self._check_headers()

        self._update_cmake_files()
        self._update_visual_studio_projects()

        # If we're all good to here, do actual writes set up
        # by the above stuff.
        self._apply_line_changes()
        self._apply_file_changes()
        self._update_compile_commands_file()
        self._update_dummy_module()

        if self._check:
            print('Check-Builds: Everything up to date.')
        else:
            print('Update-Builds: SUCCESS!')

    def _update_dummy_module(self) -> None:
        # Lastly update our dummy _ba module.
        # We need to do this very last because it may run the cmake build
        # so its success may depend on the cmake build files having already
        # been updated.
        if os.path.exists('tools/gendummymodule.py'):
            if os.system('tools/gendummymodule.py' + self._checkarg) != 0:
                print(CLRRED + 'Error checking/updating dummy module' + CLREND)
                sys.exit(255)

    def _update_compile_commands_file(self) -> None:
        # Update our local compile-commands file based on any changes to
        # our cmake stuff. Do this at end so cmake changes already happened.
        if not self._check and os.path.exists('ballisticacore-cmake'):
            if os.system('make .irony/compile_commands.json') != 0:
                print(CLRRED + 'Error updating compile-commands.' + CLREND)
                sys.exit(255)

    def _apply_file_changes(self) -> None:
        # Now write out any project files that have changed
        # (or error if we're in check mode).
        unchanged_project_count = 0
        for fname, fcode in self._file_changes.items():
            f_orig: Optional[str]
            if os.path.exists(fname):
                with open(fname, 'r') as infile:
                    f_orig = infile.read()
            else:
                f_orig = None
            if f_orig == fcode.replace('\r\n', '\n'):
                unchanged_project_count += 1
            else:
                if self._check:
                    print(f'{CLRRED}ERROR: found out-of-date'
                          f' project file: {fname}{CLREND}')
                    sys.exit(255)

                print(f'{CLRBLU}Writing project file: {fname}{CLREND}')
                with open(fname, 'w') as outfile:
                    outfile.write(fcode)
        if unchanged_project_count > 0:
            print(f'All {unchanged_project_count} project files up to date.')

    def _apply_line_changes(self) -> None:

        # Build a flat list of entries that can and can-not be auto applied.
        manual_changes: List[Tuple[str, LineChange]] = []
        auto_changes: List[Tuple[str, LineChange]] = []
        for fname, entries in self._line_checks.items():
            for entry in entries:
                if entry.can_auto_update:
                    auto_changes.append((fname, entry))
                else:
                    manual_changes.append((fname, entry))

        # If there are any manual-only entries, list then and bail.
        # (Don't wanna allow auto-apply unless it fixes everything)
        if manual_changes:
            print(f"{CLRRED}Found erroneous lines "
                  f"requiring manual correction:{CLREND}")
            for change in manual_changes:
                print(f'{CLRRED}{change[0]}:{change[1].line_number + 1}:'
                      f' Expected line to be:\n  {change[1].expected}{CLREND}')
            sys.exit(-1)

        # Now, if we've got auto entries, either list or auto-correct them.
        if auto_changes:
            if not self._fix:
                for i, change in enumerate(auto_changes):
                    print(f'{CLRRED}#{i}: {change[0]}:{CLREND}')
                    print(f'{CLRRED}  Expected "{change[1].expected}"{CLREND}')
                    with open(change[0]) as infile:
                        lines = infile.read().splitlines()
                    line = lines[change[1].line_number]
                    print(f'{CLRRED}  Found "{line}"{CLREND}')
                print(CLRRED + 'All errors are auto-fixable;'
                      ' pass --fix to apply corrections.' + CLREND)
                sys.exit(255)
            else:
                for i, change in enumerate(auto_changes):
                    print(f'{CLRBLU}Correcting file: {change[0]}{CLREND}')
                    with open(change[0]) as infile:
                        lines = infile.read().splitlines()
                    lines[change[1].line_number] = change[1].expected
                    with open(change[0], 'w') as outfile:
                        outfile.write('\n'.join(lines) + '\n')

        # If there were no issues whatsoever, note that.
        if not manual_changes and not auto_changes:
            fcount = len(self._header_files) + len(self._source_files)
            print(f'No issues found in {fcount} source files.')

    def _check_source_files(self) -> None:

        for fsrc in self._source_files:
            if fsrc.endswith('.cpp') or fsrc.endswith('.cxx'):
                raise Exception('please use .cc for c++ files; found ' + fsrc)

            # Watch out for in-progress emacs edits.
            # Could just ignore these but it probably means I intended
            # to save something and forgot.
            if '/.#' in fsrc:
                print(f'{CLRRED}'
                      f'ERROR: Found an unsaved emacs file: "{fsrc}"'
                      f'{CLREND}')
                sys.exit(255)

            fname = 'src/ballistica' + fsrc
            self._check_source_file(fname)

    def _check_source_file(self, fname: str) -> None:
        with open(fname) as infile:
            lines = infile.read().splitlines()

        # Look for copyright/legal-notice line(s)
        legal_notice = '// ' + LEGAL_NOTICE_PRIVATE
        lnum = 0
        if lines[lnum] != legal_notice:
            # Allow auto-correcting if it looks close already
            # (don't want to blow away an unrelated line)
            allow_auto = 'Copyright' in lines[
                lnum] and 'Eric Froemling' in lines[lnum]
            self._add_line_check(fname,
                                 line_number=lnum,
                                 expected=legal_notice,
                                 can_auto_update=allow_auto)

    def _check_headers(self) -> None:
        for header_file_raw in self._header_files:
            assert header_file_raw[0] == '/'
            header_file = 'src/ballistica' + header_file_raw

            if header_file.endswith('.h'):
                self._check_header(header_file)

    def _add_line_check(self, filename: str, line_number: int, expected: str,
                        can_auto_update: bool) -> None:
        self._line_checks.setdefault(filename, []).append(
            LineChange(line_number=line_number,
                       expected=expected,
                       can_auto_update=can_auto_update))

    def _check_header(self, fname: str) -> None:

        # Make sure its define guard is correct.
        guard = (fname[4:].upper().replace('/', '_').replace('.', '_') + '_')
        with open(fname) as fhdr:
            lines = fhdr.read().splitlines()

        if self._public:
            raise RuntimeError('FIXME: Check for full license.')

        # Look for copyright/legal-notice line(s)
        line = '// ' + LEGAL_NOTICE_PRIVATE
        lnum = 0
        if lines[lnum] != line:
            # Allow auto-correcting if it looks close already
            # (don't want to blow away an unrelated line)
            allow_auto = 'Copyright' in lines[
                lnum] and 'Eric Froemling' in lines[lnum]
            self._add_line_check(fname,
                                 line_number=lnum,
                                 expected=line,
                                 can_auto_update=allow_auto)

        # Check for header guard at top
        line = '#ifndef ' + guard
        lnum = 2
        if lines[lnum] != line:
            # Allow auto-correcting if it looks close already
            # (don't want to blow away an unrelated line)
            allow_auto = lines[lnum].startswith('#ifndef BALLISTICA_')
            self._add_line_check(fname,
                                 line_number=lnum,
                                 expected=line,
                                 can_auto_update=allow_auto)

        # Check for header guard at bottom
        line = '#endif  // ' + guard
        lnum = -1
        if lines[lnum] != line:
            # Allow auto-correcting if it looks close already
            # (don't want to blow away an unrelated line)
            allow_auto = lines[lnum].startswith('#endif  // BALLISTICA_')
            self._add_line_check(fname,
                                 line_number=lnum,
                                 expected=line,
                                 can_auto_update=allow_auto)

    def _check_python_file(self, fname: str) -> None:
        with open(fname) as infile:
            contents = infile.read()
            lines = contents.splitlines()

        # Make sure all standalone scripts are pointing to the right
        # version of python (with a few exceptions where it needs to
        # differ)
        if contents.startswith('#!/'):
            copyrightline = 1
            if fname not in [
                    'tools/devtool', 'tools/version_utils', 'tools/vmshell'
            ]:
                if not contents.startswith('#!/usr/bin/env python3.7'):
                    print(f"{CLRRED}Incorrect shebang (first line) for "
                          f"{fname}.{CLREND}")
                    sys.exit(255)
        else:
            copyrightline = 0

        # Look for copyright/legal-notice line(s)
        line = '# ' + LEGAL_NOTICE_PRIVATE
        lnum = copyrightline
        if len(lines) < lnum + 1:
            raise RuntimeError('Not enough lines in file:', fname)

        if lines[lnum] != line:
            # Allow auto-correcting if it looks close already
            # (don't want to blow away an unrelated line)
            allow_auto = 'Copyright' in lines[
                lnum] and 'Eric Froemling' in lines[lnum]
            self._add_line_check(fname,
                                 line_number=lnum,
                                 expected=line,
                                 can_auto_update=allow_auto)

    def _check_python_files(self) -> None:
        from pathlib import Path
        from efrotools.code import get_script_filenames

        scriptfiles = get_script_filenames(Path('.'))
        for fname in scriptfiles:
            self._check_python_file(fname)

        # Check our packages and make sure all subdirs contain and __init__.py
        # (I tend to forget this sometimes)
        packagedirs = ['tools/efrotools']
        script_asset_dir = 'assets/src/data/scripts'
        for name in os.listdir(script_asset_dir):
            # (Assume all dirs under our script assets dir are packages)
            if os.path.isdir(os.path.join(script_asset_dir)):
                packagedirs.append(os.path.join(script_asset_dir, name))

        for packagedir in packagedirs:
            for root, _dirs, files in os.walk(packagedir):
                if ('__pycache__' not in root
                        and os.path.basename(root) != '.vscode'):
                    if '__init__.py' not in files:
                        print(CLRRED +
                              'Error: no __init__.py in package dir: ' + root +
                              CLREND)
                        sys.exit(255)

    def _update_visual_studio_project(self, fname: str, src_root: str) -> None:
        with open(fname) as infile:
            lines = infile.read().splitlines()

        # Hmm can we include headers in the project for easy access?
        # Seems VS attempts to compile them if we do so here.
        # all_files = sorted(src_files + header_files)
        # del header_files  # Unused.
        all_files = sorted([
            f for f in (self._source_files + self._header_files)
            if not f.endswith('.m') and not f.endswith('.mm')
            and not f.endswith('.c')
        ])

        # Find the ItemGroup containing stdafx.cpp. This is where we'll dump
        # our stuff.
        index = lines.index('    <ClCompile Include="stdafx.cpp">')
        begin_index = end_index = index
        while lines[begin_index] != '  <ItemGroup>':
            begin_index -= 1
        while lines[end_index] != '  </ItemGroup>':
            end_index += 1
        group_lines = lines[begin_index + 1:end_index]

        # Strip out any existing files from src/ballistica.
        group_lines = [
            l for l in group_lines if src_root + '\\ballistica\\' not in l
        ]

        # Now add in our own.
        # Note: we can't use C files in this build at the moment; breaks
        # precompiled header stuff. (shouldn't be a problem though).
        group_lines = [
            '    <' +
            ('ClInclude' if src.endswith('.h') else 'ClCompile') + ' Include="'
            + src_root + '\\ballistica' + src.replace('/', '\\') + '" />'
            for src in all_files
        ] + group_lines
        filtered = lines[:begin_index + 1] + group_lines + lines[end_index:]
        self._file_changes[fname] = '\r\n'.join(filtered) + '\r\n'

        self._update_visual_studio_project_filters(filtered, fname, src_root)

    def _update_visual_studio_project_filters(self, lines_in: List[str],
                                              fname: str,
                                              src_root: str) -> None:
        filterpaths: Set[str] = set()
        filterlines: List[str] = [
            '<?xml version="1.0" encoding="utf-8"?>',
            '<Project ToolsVersion="4.0"'
            ' xmlns="http://schemas.microsoft.com/developer/msbuild/2003">',
            '  <ItemGroup>',
        ]
        sourcelines = [l for l in lines_in if 'Include="' + src_root in l]
        for line in sourcelines:
            entrytype = line.strip().split()[0][1:]
            path = line.split('"')[1]
            filterlines.append('    <' + entrytype + ' Include="' + path +
                               '">')

            # If we have a dir foo/bar/eep we need to create filters for
            # each of foo, foo/bar, and foo/bar/eep
            splits = path[len(src_root):].split('\\')
            splits = [s for s in splits if s != '']
            splits = splits[:-1]
            for i in range(len(splits)):
                filterpaths.add('\\'.join(splits[:(i + 1)]))
            filterlines.append('      <Filter>' + '\\'.join(splits) +
                               '</Filter>')
            filterlines.append('    </' + entrytype + '>')
        filterlines += [
            '  </ItemGroup>',
            '  <ItemGroup>',
        ]
        for filterpath in sorted(filterpaths):
            filterlines.append('    <Filter Include="' + filterpath + '" />')
        filterlines += [
            '  </ItemGroup>',
            '</Project>',
        ]
        self._file_changes[fname +
                           '.filters'] = '\r\n'.join(filterlines) + '\r\n'

    def _update_visual_studio_projects(self) -> None:
        fname = 'ballisticacore-windows/BallisticaCore/BallisticaCore.vcxproj'
        if os.path.exists(fname):
            self._update_visual_studio_project(fname, '..\\..\\src')
        fname = ('ballisticacore-windows/BallisticaCoreHeadless/'
                 'BallisticaCoreHeadless.vcxproj')
        if os.path.exists(fname):
            self._update_visual_studio_project(fname, '..\\..\\src')
        fname = ('ballisticacore-windows/BallisticaCoreOculus'
                 '/BallisticaCoreOculus.vcxproj')
        if os.path.exists(fname):
            self._update_visual_studio_project(fname, '..\\..\\src')

    def _update_cmake_file(self, fname: str) -> None:

        with open(fname) as infile:
            lines = infile.read().splitlines()
        auto_start = lines.index('        #AUTOGENERATED_BEGIN (this section'
                                 ' is managed by the "update_project" tool)')
        auto_end = lines.index('        #AUTOGENERATED_END')
        our_lines = [
            '        ${BA_SRC_ROOT}/ballistica' + f
            for f in sorted(self._source_files + self._header_files)
            if not f.endswith('.mm') and not f.endswith('.m')
        ]
        filtered = lines[:auto_start + 1] + our_lines + lines[auto_end:]
        self._file_changes[fname] = '\n'.join(filtered) + '\n'

    def _update_cmake_files(self) -> None:
        fname = 'ballisticacore-cmake/CMakeLists.txt'
        if os.path.exists(fname):
            self._update_cmake_file(fname)
        fname = ('ballisticacore-android/BallisticaCore'
                 '/src/main/cpp/CMakeLists.txt')
        if os.path.exists(fname):
            self._update_cmake_file(fname)

    def _find_sources_and_headers(self, scan_dir: str) -> None:
        src_files = set()
        header_files = set()
        exts = ['.c', '.cc', '.cpp', '.cxx', '.m', '.mm']
        header_exts = ['.h']

        # Gather all sources and headers.
        # HMMM: Ideally we should use efrotools.code.get_code_filenames() here.
        # (though we return things relative to the scan-dir which could
        # throw things off)
        for root, _dirs, files in os.walk(scan_dir):
            for ftst in files:
                if any(ftst.endswith(ext) for ext in exts):
                    src_files.add(os.path.join(root, ftst)[len(scan_dir):])
                if any(ftst.endswith(ext) for ext in header_exts):
                    header_files.add(os.path.join(root, ftst)[len(scan_dir):])
        self._source_files = sorted(src_files)
        self._header_files = sorted(header_files)

    def _check_sync_states(self) -> None:
        # Make sure none of our sync targets have been mucked with since
        # their last sync.
        if os.system('tools/snippets sync check') != 0:
            print(CLRRED + 'Sync check failed; you may need to run "sync".' +
                  CLREND)
            sys.exit(255)

    def _update_assets_makefile(self) -> None:
        if os.path.exists('tools/update_assets_makefile'):
            if os.system('tools/update_assets_makefile' + self._checkarg) != 0:
                print(CLRRED + 'Error checking/updating assets Makefile' +
                      CLREND)
                sys.exit(255)

    def _update_generated_code_makefile(self) -> None:
        if os.path.exists('tools/update_generated_code_makefile'):
            if os.system('tools/update_generated_code_makefile' +
                         self._checkarg) != 0:
                print(CLRRED +
                      'Error checking/updating generated-code Makefile' +
                      CLREND)
                sys.exit(255)

    def _update_resources_makefile(self) -> None:
        if os.path.exists('tools/update_resources_makefile'):
            if os.system('tools/update_resources_makefile' +
                         self._checkarg) != 0:
                print(CLRRED + 'Error checking/updating resources Makefile' +
                      CLREND)
                sys.exit(255)

    def _update_python_enums_module(self) -> None:
        if os.path.exists('tools/update_python_enums_module'):
            if os.system('tools/update_python_enums_module' +
                         self._checkarg) != 0:
                print(CLRRED + 'Error checking/updating python enums module' +
                      CLREND)
                sys.exit(255)


if __name__ == '__main__':
    App().run()