#!/usr/bin/env python3.7 """ This script acts as a 'meta' Makefile for the project. It is in charge of generating Makefiles, IDE project files, procedurally generated source files, etc. based on the current structure of the project. It can also perform sanity checks or cleanup tasks. Updating should be explicitly run by the user through commands such as 'make update', 'make check' or 'make preflight'. Other make targets should avoid running this script as it can modify the project structure arbitrarily which is not a good idea in the middle of a build. If the script is invoked with a --check argument, it should not modify any files but instead fail if any modifications *would* have been made. (used in CI builds to make sure things are kosher). """ from __future__ import annotations import os import sys from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: from typing import Optional, Tuple, List, Dict, Set CLRHDR = '\033[95m' # Header. CLRGRN = '\033[92m' # Green. CLRBLU = '\033[94m' # Glue. CLRRED = '\033[91m' # Red. CLREND = '\033[0m' # End. @dataclass class LineChange: """A change applying to a particular line in a file.""" line_number: int expected: str can_auto_update: bool class App: """Context for an app run.""" def __init__(self) -> None: self._check = ('--check' in sys.argv) self._fix = ('--fix' in sys.argv) self._checkarg = ' --check' if self._check else '' self._src_files: List[str] = [] self._header_files: List[str] = [] self._line_changes: Dict[str, List[LineChange]] = {} self._file_changes: Dict[str, str] = {} # KILL ME self._fixable_header_errors: Dict[str, List[Tuple[int, str]]] = {} def run(self) -> None: """Do the thing.""" # Make sure we're operating from a project root. if not os.path.isdir('config') or not os.path.isdir('tools'): raise Exception('This must be run from a project root.') # NOTE: Do py-enums before updating asset deps since it is an asset. self._update_python_enums_module() self._update_resources_makefile() self._update_generated_code_makefile() self._update_assets_makefile() self._check_python_files() self._check_sync_states() self._find_sources_and_headers('src/ballistica') self._check_files() self._check_headers() self._update_cmake_files() self._update_visual_studio_projects() # If we're all good to here, do actual writes set up # by the above stuff. self._apply_line_changes() self._apply_file_changes() self._update_compile_commands_file() self._update_dummy_module() if self._check: print('Check-Builds: Everything up to date.') else: print('Update-Builds: SUCCESS!') def _update_dummy_module(self) -> None: # Lastly update our dummy _ba module. # We need to do this very last because it may run the cmake build # so its success may depend on the cmake build files having already # been updated. if os.path.exists('tools/gendummymodule.py'): if os.system('tools/gendummymodule.py' + self._checkarg) != 0: print(CLRRED + 'Error checking/updating dummy module' + CLREND) sys.exit(255) def _update_compile_commands_file(self) -> None: # Update our local compile-commands file based on any changes to # our cmake stuff. Do this at end so cmake changes already happened. if not self._check and os.path.exists('ballisticacore-cmake'): if os.system('make .irony/compile_commands.json') != 0: print(CLRRED + 'Error updating compile-commands.' + CLREND) sys.exit(255) def _apply_file_changes(self) -> None: # Now write out any project files that have changed # (or error if we're in check mode). unchanged_project_count = 0 for fname, fcode in self._file_changes.items(): f_orig: Optional[str] if os.path.exists(fname): with open(fname, 'r') as infile: f_orig = infile.read() else: f_orig = None if f_orig == fcode.replace('\r\n', '\n'): unchanged_project_count += 1 else: if self._check: print(f'{CLRRED}ERROR: found out-of-date' f' project file: {fname}{CLREND}') sys.exit(255) print(f'{CLRBLU}Writing project file: {fname}{CLREND}') with open(fname, 'w') as outfile: outfile.write(fcode) if unchanged_project_count > 0: print(f'All {unchanged_project_count} project files up to date.') def _apply_line_changes(self) -> None: print("LOOKING AT", len(self._line_changes), 'CHANGES') # Build a flat list of entries needing to be manually applied. manual_changes: List[Tuple[str, LineChange]] = [] for fname, entries in self._line_changes.items(): for entry in entries: if not entry.can_auto_update: manual_changes.append((fname, entry)) # If there are any said entries, list then and bail. # (Don't wanna allow auto-apply unless it fixes everything) if manual_changes: print(f"{CLRRED}Found incorrect lines (cannot auto-update;" f" please correct manually):{CLREND}") for change in manual_changes: print(f'{CLRRED}{change}{CLREND}') sys.exit(-1) if self._fixable_header_errors: for filename, fixes in self._fixable_header_errors.items(): with open(filename, 'r') as infile: lines = infile.read().splitlines() for fix_line, fix_str in fixes: lines[fix_line] = fix_str with open(filename, 'w') as outfile: outfile.write('\n'.join(lines) + '\n') print(CLRBLU + 'Writing header: ' + filename + CLREND) else: print(f'No issues found in {len(self._header_files)} headers.') def _check_files(self) -> None: # A bit of sanity/lint-testing while we're here. for fsrc in self._src_files: if fsrc.endswith('.cpp') or fsrc.endswith('.cxx'): raise Exception('please use .cc for c++ files; found ' + fsrc) # Watch out for in-progress emacs edits. # Could just ignore these but it probably means I intended # to save something and forgot. if '/.#' in fsrc: print(f'{CLRRED}' f'ERROR: Found an unsaved emacs file: "{fsrc}"' f'{CLREND}') sys.exit(255) def _check_headers(self) -> None: for header_file_raw in self._header_files: assert header_file_raw[0] == '/' header_file = 'src/ballistica' + header_file_raw if header_file.endswith('.h'): self._check_header(header_file) if self._fixable_header_errors and not self._fix: print(CLRRED + 'Fixable header error(s) found; pass --fix to correct.' + CLREND) sys.exit(255) def _check_header(self, header_file: str) -> None: # Make sure its define guard is correct. guard = (header_file[4:].upper().replace('/', '_').replace('.', '_') + '_') with open(header_file) as fhdr: lines = fhdr.read().splitlines() expected_lines = [(0, '// Copyright 2019 Eric Froemling'), (2, '#ifndef ' + guard), (3, '#define ' + guard), (-1, '#endif // ' + guard)] errors_found = False can_fix = True for line, expected in expected_lines: if lines[line] != expected: errors_found = True print("Incorrect line " + str(line) + " in " + header_file + ":\n" "Expected: " + expected + "\n" "Found: " + lines[line]) # If the beginning of the line differs, don't attempt auto-fix. if lines[line][:16] != expected[:16]: can_fix = False if errors_found: if can_fix: self._fixable_header_errors.setdefault(header_file, []).append( (line, expected)) else: print(CLRRED + "Error found in '" + header_file + "'. Not auto-fixable; please correct manually." + CLREND) sys.exit(255) def _update_visual_studio_project(self, fname: str, src_root: str) -> None: with open(fname) as infile: lines = infile.read().splitlines() # Hmm can we include headers in the project for easy access? # Seems VS attempts to compile them if we do so here. # all_files = sorted(src_files + header_files) # del header_files # Unused. all_files = sorted([ f for f in (self._src_files + self._header_files) if not f.endswith('.m') and not f.endswith('.mm') and not f.endswith('.c') ]) # Find the ItemGroup containing stdafx.cpp. This is where we'll dump # our stuff. index = lines.index(' ') begin_index = end_index = index while lines[begin_index] != ' ': begin_index -= 1 while lines[end_index] != ' ': end_index += 1 group_lines = lines[begin_index + 1:end_index] # Strip out any existing files from src/ballistica. group_lines = [ l for l in group_lines if src_root + '\\ballistica\\' not in l ] # Now add in our own. # Note: we can't use C files in this build at the moment; breaks # precompiled header stuff. (shouldn't be a problem though). group_lines = [ ' <' + ('ClInclude' if src.endswith('.h') else 'ClCompile') + ' Include="' + src_root + '\\ballistica' + src.replace('/', '\\') + '" />' for src in all_files ] + group_lines filtered = lines[:begin_index + 1] + group_lines + lines[end_index:] self._file_changes[fname] = '\r\n'.join(filtered) + '\r\n' self._update_visual_studio_project_filters(filtered, fname, src_root) def _update_visual_studio_project_filters(self, lines_in: List[str], fname: str, src_root: str) -> None: filterpaths: Set[str] = set() filterlines: List[str] = [ '', '', ' ', ] sourcelines = [l for l in lines_in if 'Include="' + src_root in l] for line in sourcelines: entrytype = line.strip().split()[0][1:] path = line.split('"')[1] filterlines.append(' <' + entrytype + ' Include="' + path + '">') # If we have a dir foo/bar/eep we need to create filters for # each of foo, foo/bar, and foo/bar/eep splits = path[len(src_root):].split('\\') splits = [s for s in splits if s != ''] splits = splits[:-1] for i in range(len(splits)): filterpaths.add('\\'.join(splits[:(i + 1)])) filterlines.append(' ' + '\\'.join(splits) + '') filterlines.append(' ') filterlines += [ ' ', ' ', ] for filterpath in sorted(filterpaths): filterlines.append(' ') filterlines += [ ' ', '', ] self._file_changes[fname + '.filters'] = '\r\n'.join(filterlines) + '\r\n' def _update_visual_studio_projects(self) -> None: fname = 'ballisticacore-windows/BallisticaCore/BallisticaCore.vcxproj' if os.path.exists(fname): self._update_visual_studio_project(fname, '..\\..\\src') fname = ('ballisticacore-windows/BallisticaCoreHeadless/' 'BallisticaCoreHeadless.vcxproj') if os.path.exists(fname): self._update_visual_studio_project(fname, '..\\..\\src') fname = ('ballisticacore-windows/BallisticaCoreOculus' '/BallisticaCoreOculus.vcxproj') if os.path.exists(fname): self._update_visual_studio_project(fname, '..\\..\\src') def _update_cmake_file(self, fname: str) -> None: with open(fname) as infile: lines = infile.read().splitlines() auto_start = lines.index(' #AUTOGENERATED_BEGIN (this section' ' is managed by the "update_project" tool)') auto_end = lines.index(' #AUTOGENERATED_END') our_lines = [ ' ${BA_SRC_ROOT}/ballistica' + f for f in sorted(self._src_files + self._header_files) if not f.endswith('.mm') and not f.endswith('.m') ] filtered = lines[:auto_start + 1] + our_lines + lines[auto_end:] self._file_changes[fname] = '\n'.join(filtered) + '\n' def _update_cmake_files(self) -> None: fname = 'ballisticacore-cmake/CMakeLists.txt' if os.path.exists(fname): self._update_cmake_file(fname) fname = ('ballisticacore-android/BallisticaCore' '/src/main/cpp/CMakeLists.txt') if os.path.exists(fname): self._update_cmake_file(fname) def _find_sources_and_headers(self, scan_dir: str) -> None: src_files = set() header_files = set() exts = ['.c', '.cc', '.cpp', '.cxx', '.m', '.mm'] header_exts = ['.h'] # Gather all sources and headers. for root, _dirs, files in os.walk(scan_dir): for ftst in files: if any(ftst.endswith(ext) for ext in exts): src_files.add(os.path.join(root, ftst)[len(scan_dir):]) if any(ftst.endswith(ext) for ext in header_exts): header_files.add(os.path.join(root, ftst)[len(scan_dir):]) self._src_files = sorted(src_files) self._header_files = sorted(header_files) def _check_python_files(self) -> None: # Make sure all module dirs in python scripts contain an __init__.py scripts_dir = 'assets/src/data/scripts' for root, _dirs, files in os.walk(scripts_dir): if (root != scripts_dir and '__pycache__' not in root and os.path.basename(root) != '.vscode'): if '__init__.py' not in files: print(CLRRED + 'Error: no __init__.py in package dir: ' + root + CLREND) sys.exit(255) def _check_sync_states(self) -> None: # Make sure none of our sync targets have been mucked with since # their last sync. if os.system('tools/snippets sync check') != 0: print(CLRRED + 'Sync check failed; you may need to run "sync".' + CLREND) sys.exit(255) def _update_assets_makefile(self) -> None: if os.path.exists('tools/update_assets_makefile'): if os.system('tools/update_assets_makefile' + self._checkarg) != 0: print(CLRRED + 'Error checking/updating assets Makefile' + CLREND) sys.exit(255) def _update_generated_code_makefile(self) -> None: if os.path.exists('tools/update_generated_code_makefile'): if os.system('tools/update_generated_code_makefile' + self._checkarg) != 0: print(CLRRED + 'Error checking/updating generated-code Makefile' + CLREND) sys.exit(255) def _update_resources_makefile(self) -> None: if os.path.exists('tools/update_resources_makefile'): if os.system('tools/update_resources_makefile' + self._checkarg) != 0: print(CLRRED + 'Error checking/updating resources Makefile' + CLREND) sys.exit(255) def _update_python_enums_module(self) -> None: if os.path.exists('tools/update_python_enums_module'): if os.system('tools/update_python_enums_module' + self._checkarg) != 0: print(CLRRED + 'Error checking/updating python enums module' + CLREND) sys.exit(255) if __name__ == '__main__': App().run()