cleaned up the project-update script a bit

This commit is contained in:
Eric Froemling 2019-10-07 05:36:48 -07:00
parent 7d800fd5a6
commit b22c29bd43

View File

@ -31,22 +31,6 @@ CLRRED = '\033[91m' # Red.
CLREND = '\033[0m' # End.
def _find_files(scan_dir: str) -> Tuple[List[str], List[str]]:
src_files = set()
header_files = set()
exts = ['.c', '.cc', '.cpp', '.cxx', '.m', '.mm']
header_exts = ['.h']
# Gather all sources and headers.
for root, _dirs, files in os.walk(scan_dir):
for ftst in files:
if any(ftst.endswith(ext) for ext in exts):
src_files.add(os.path.join(root, ftst)[len(scan_dir):])
if any(ftst.endswith(ext) for ext in header_exts):
header_files.add(os.path.join(root, ftst)[len(scan_dir):])
return sorted(src_files), sorted(header_files)
def _check_files(src_files: Sequence[str]) -> None:
# A bit of sanity/lint-testing while we're here.
@ -109,289 +93,297 @@ def _check_headers(header_files: Sequence[str], fixable_header_errors: Dict,
sys.exit(255)
def _update_cmake_file(fname: str, src_files: List[str],
header_files: List[str],
files_to_write: Dict[str, str]) -> None:
class App:
"""Context for an app run."""
with open(fname) as infile:
lines = infile.read().splitlines()
auto_start = lines.index(' #AUTOGENERATED_BEGIN (this section'
' is managed by the "update_project" tool)')
auto_end = lines.index(' #AUTOGENERATED_END')
our_lines = [
' ${BA_SRC_ROOT}/ballistica' + f
for f in sorted(src_files + header_files)
if not f.endswith('.mm') and not f.endswith('.m')
]
filtered = lines[:auto_start + 1] + our_lines + lines[auto_end:]
files_to_write[fname] = '\n'.join(filtered) + '\n'
def __init__(self) -> None:
self._check = ('--check' in sys.argv)
self._fix = ('--fix' in sys.argv)
self._checkarg = ' --check' if self._check else ''
self._files_to_write: Dict[str, str] = {}
self._src_files: List[str] = []
self._header_files: List[str] = []
self._fixable_header_errors: Dict[str, List[Tuple[int, str]]] = {}
def _update_visual_studio_project(fname: str, src_root: str,
src_files: List[str],
header_files: List[str],
files_to_write: Dict[str, str]) -> None:
# pylint: disable=too-many-locals
with open(fname) as infile:
lines = infile.read().splitlines()
def run(self) -> None:
"""Do the thing."""
# Hmm can we include headers in the project for easy access?
# Seems VS attempts to compile them if we do so here.
# all_files = sorted(src_files + header_files)
# del header_files # Unused.
all_files = sorted([
f for f in (src_files + header_files) if not f.endswith('.m')
and not f.endswith('.mm') and not f.endswith('.c')
])
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# Find the ItemGroup containing stdafx.cpp. This is where we'll dump
# our stuff.
index = lines.index(' <ClCompile Include="stdafx.cpp">')
begin_index = end_index = index
while lines[begin_index] != ' <ItemGroup>':
begin_index -= 1
while lines[end_index] != ' </ItemGroup>':
end_index += 1
group_lines = lines[begin_index + 1:end_index]
# Make sure we're operating from a project root.
if not os.path.isdir('config') or not os.path.isdir('tools'):
raise Exception('This must be run from a project root.')
# Strip out any existing files from src/ballistica.
group_lines = [
l for l in group_lines if src_root + '\\ballistica\\' not in l
]
# NOTE: Do py-enums before updating asset deps since it is an asset.
self._update_python_enums_module()
self._update_resources_makefile()
self._update_generated_code_makefile()
self._update_assets_makefile()
# Now add in our own.
# Note: we can't use C files in this build at the moment; breaks
# precompiled header stuff. (shouldn't be a problem though).
group_lines = [
' <' +
('ClInclude' if src.endswith('.h') else 'ClCompile') + ' Include="' +
src_root + '\\ballistica' + src.replace('/', '\\') + '" />'
for src in all_files
] + group_lines
filtered = lines[:begin_index + 1] + group_lines + lines[end_index:]
files_to_write[fname] = '\r\n'.join(filtered) + '\r\n'
self._check_python_files()
self._check_sync_states()
filterpaths: Set[str] = set()
filterlines: List[str] = [
'<?xml version="1.0" encoding="utf-8"?>',
'<Project ToolsVersion="4.0"'
' xmlns="http://schemas.microsoft.com/developer/msbuild/2003">',
' <ItemGroup>',
]
sourcelines = [l for l in filtered if 'Include="' + src_root in l]
for line in sourcelines:
entrytype = line.strip().split()[0][1:]
path = line.split('"')[1]
filterlines.append(' <' + entrytype + ' Include="' + path + '">')
# Now go through and update our various build files
# (MSVC, CMake, Android NDK, etc) as well as sanity-testing/fixing
# various other files such as headers while we're at it.
# If we have a dir foo/bar/eep we need to create filters for
# each of foo, foo/bar, and foo/bar/eep
splits = path[len(src_root):].split('\\')
splits = [s for s in splits if s != '']
splits = splits[:-1]
for i in range(len(splits)):
filterpaths.add('\\'.join(splits[:(i + 1)]))
filterlines.append(' <Filter>' + '\\'.join(splits) + '</Filter>')
filterlines.append(' </' + entrytype + '>')
filterlines += [
' </ItemGroup>',
' <ItemGroup>',
]
for filterpath in sorted(filterpaths):
filterlines.append(' <Filter Include="' + filterpath + '" />')
filterlines += [
' </ItemGroup>',
'</Project>',
]
# Grab sources/headers.
self._find_sources_and_headers('src/ballistica')
files_to_write[fname + '.filters'] = '\r\n'.join(filterlines) + '\r\n'
# Run some checks on them.
_check_files(self._src_files)
_check_headers(self._header_files, self._fixable_header_errors,
self._fix)
self._update_cmake_files()
self._update_visual_studio_projects()
def main() -> None:
"""Main script entry point."""
# If we're all good to here, do the actual writes.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# First, write any header fixes.
if self._fixable_header_errors:
for filename, fixes in self._fixable_header_errors.items():
with open(filename, 'r') as infile:
lines = infile.read().splitlines()
for fix_line, fix_str in fixes:
lines[fix_line] = fix_str
with open(filename, 'w') as outfile:
outfile.write('\n'.join(lines) + '\n')
print(CLRBLU + 'Writing header: ' + filename + CLREND)
else:
print(f'No issues found in {len(self._header_files)} headers.')
# Make sure we're operating from dist root.
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..'))
# Now write out any project files that have changed
# (or error if we're in check mode).
unchanged_project_count = 0
for fname, fcode in self._files_to_write.items():
f_orig: Optional[str]
if os.path.exists(fname):
with open(fname, 'r') as infile:
f_orig = infile.read()
else:
f_orig = None
if f_orig == fcode.replace('\r\n', '\n'):
unchanged_project_count += 1
else:
if self._check:
print(f'{CLRRED}ERROR: found out-of-date'
f' project file: {fname}{CLREND}')
sys.exit(255)
check = ('--check' in sys.argv)
fix = ('--fix' in sys.argv)
print(f'{CLRBLU}Writing project file: {fname}{CLREND}')
with open(fname, 'w') as outfile:
outfile.write(fcode)
if unchanged_project_count > 0:
print(f'All {unchanged_project_count} project files up to date.')
checkarg = ' --check' if check else ''
# Update our python enums module.
# Should do this before updating asset deps since this is an asset.
if os.path.exists('tools/update_python_enums_module'):
if os.system('tools/update_python_enums_module' + checkarg) != 0:
print(CLRRED + 'Error checking/updating python enums module' +
CLREND)
sys.exit(255)
# Update our resources Makefile.
if os.path.exists('tools/update_resources_makefile'):
if os.system('tools/update_resources_makefile' + checkarg) != 0:
print(CLRRED + 'Error checking/updating resources Makefile' +
CLREND)
sys.exit(255)
# Update our generated-code Makefile.
if os.path.exists('tools/update_generated_code_makefile'):
if os.system('tools/update_generated_code_makefile' + checkarg) != 0:
print(CLRRED + 'Error checking/updating generated-code Makefile' +
CLREND)
sys.exit(255)
# Update our assets Makefile.
if os.path.exists('tools/update_assets_makefile'):
if os.system('tools/update_assets_makefile' + checkarg) != 0:
print(CLRRED + 'Error checking/updating assets Makefile' + CLREND)
sys.exit(255)
# Make sure all module dirs in python scripts contain an __init__.py
scripts_dir = 'assets/src/data/scripts'
for root, _dirs, files in os.walk(scripts_dir):
if (root != scripts_dir and '__pycache__' not in root
and os.path.basename(root) != '.vscode'):
if '__init__.py' not in files:
print(CLRRED + 'Error: no __init__.py in package dir: ' +
root + CLREND)
# Update our local compile-commands file based on any changes to
# our cmake stuff. Do this at end so cmake changes already happened.
if not self._check and os.path.exists('ballisticacore-cmake'):
if os.system('make .irony/compile_commands.json') != 0:
print(CLRRED + 'Error updating compile-commands.' + CLREND)
sys.exit(255)
# Make sure none of our sync targets have been mucked with since
# their last sync.
if os.system('tools/snippets sync check') != 0:
print(CLRRED + 'Sync check failed; you may need to run "sync".' +
CLREND)
sys.exit(255)
# Lastly update our dummy _ba module.
# We need to do this very last because it may run the cmake build
# so its success may depend on the cmake build files having already
# been updated.
if os.path.exists('tools/gendummymodule.py'):
if os.system('tools/gendummymodule.py' + self._checkarg) != 0:
print(CLRRED + 'Error checking/updating dummy module' + CLREND)
sys.exit(255)
# Now go through and update our various build files
# (MSVC, CMake, Android NDK, etc) as well as sanity-testing/fixing
# various other files such as headers while we're at it.
if self._check:
print('Check-Builds: Everything up to date.')
else:
print('Update-Builds: SUCCESS!')
files_to_write: Dict[str, str] = {}
def _update_visual_studio_project(self, fname: str, src_root: str) -> None:
# pylint: disable=too-many-locals
with open(fname) as infile:
lines = infile.read().splitlines()
# Grab sources/headers.
src_files, header_files = _find_files('src/ballistica')
# Hmm can we include headers in the project for easy access?
# Seems VS attempts to compile them if we do so here.
# all_files = sorted(src_files + header_files)
# del header_files # Unused.
all_files = sorted([
f for f in (self._src_files + self._header_files)
if not f.endswith('.m') and not f.endswith('.mm')
and not f.endswith('.c')
])
# Run some checks on them.
_check_files(src_files)
fixable_header_errors: Dict[str, List[Tuple[int, str]]] = {}
_check_headers(header_files, fixable_header_errors, fix)
# Find the ItemGroup containing stdafx.cpp. This is where we'll dump
# our stuff.
index = lines.index(' <ClCompile Include="stdafx.cpp">')
begin_index = end_index = index
while lines[begin_index] != ' <ItemGroup>':
begin_index -= 1
while lines[end_index] != ' </ItemGroup>':
end_index += 1
group_lines = lines[begin_index + 1:end_index]
# Now update various builds.
# Strip out any existing files from src/ballistica.
group_lines = [
l for l in group_lines if src_root + '\\ballistica\\' not in l
]
# CMake
fname = 'ballisticacore-cmake/CMakeLists.txt'
if os.path.exists(fname):
_update_cmake_file(
fname,
src_files,
header_files,
files_to_write,
)
fname = 'ballisticacore-android/BallisticaCore/src/main/cpp/CMakeLists.txt'
if os.path.exists(fname):
_update_cmake_file(
fname,
src_files,
header_files,
files_to_write,
)
# Now add in our own.
# Note: we can't use C files in this build at the moment; breaks
# precompiled header stuff. (shouldn't be a problem though).
group_lines = [
' <' +
('ClInclude' if src.endswith('.h') else 'ClCompile') + ' Include="'
+ src_root + '\\ballistica' + src.replace('/', '\\') + '" />'
for src in all_files
] + group_lines
filtered = lines[:begin_index + 1] + group_lines + lines[end_index:]
self._files_to_write[fname] = '\r\n'.join(filtered) + '\r\n'
# Visual Studio
fname = 'ballisticacore-windows/BallisticaCore/BallisticaCore.vcxproj'
if os.path.exists(fname):
_update_visual_studio_project(
fname,
'..\\..\\src',
src_files,
header_files,
files_to_write,
)
fname = ('ballisticacore-windows/BallisticaCoreHeadless/'
'BallisticaCoreHeadless.vcxproj')
if os.path.exists(fname):
_update_visual_studio_project(
fname,
'..\\..\\src',
src_files,
header_files,
files_to_write,
)
fname = ('ballisticacore-windows/BallisticaCoreOculus'
'/BallisticaCoreOculus.vcxproj')
if os.path.exists(fname):
_update_visual_studio_project(
fname,
'..\\..\\src',
src_files,
header_files,
files_to_write,
)
filterpaths: Set[str] = set()
filterlines: List[str] = [
'<?xml version="1.0" encoding="utf-8"?>',
'<Project ToolsVersion="4.0"'
' xmlns="http://schemas.microsoft.com/developer/msbuild/2003">',
' <ItemGroup>',
]
sourcelines = [l for l in filtered if 'Include="' + src_root in l]
for line in sourcelines:
entrytype = line.strip().split()[0][1:]
path = line.split('"')[1]
filterlines.append(' <' + entrytype + ' Include="' + path +
'">')
# If we're all good to here, do the actual writes.
# If we have a dir foo/bar/eep we need to create filters for
# each of foo, foo/bar, and foo/bar/eep
splits = path[len(src_root):].split('\\')
splits = [s for s in splits if s != '']
splits = splits[:-1]
for i in range(len(splits)):
filterpaths.add('\\'.join(splits[:(i + 1)]))
filterlines.append(' <Filter>' + '\\'.join(splits) +
'</Filter>')
filterlines.append(' </' + entrytype + '>')
filterlines += [
' </ItemGroup>',
' <ItemGroup>',
]
for filterpath in sorted(filterpaths):
filterlines.append(' <Filter Include="' + filterpath + '" />')
filterlines += [
' </ItemGroup>',
'</Project>',
]
# First, write any header fixes.
if fixable_header_errors:
for filename, fixes in fixable_header_errors.items():
with open(filename, 'r') as infile:
lines = infile.read().splitlines()
for fix_line, fix_str in fixes:
lines[fix_line] = fix_str
with open(filename, 'w') as outfile:
outfile.write('\n'.join(lines) + '\n')
print(CLRBLU + 'Writing header: ' + filename + CLREND)
else:
print(f'No issues found in {len(header_files)} headers.')
self._files_to_write[fname +
'.filters'] = '\r\n'.join(filterlines) + '\r\n'
# Now write out any project files that have changed
# (or error if we're in check mode).
unchanged_project_count = 0
for fname, fcode in files_to_write.items():
f_orig: Optional[str]
def _update_visual_studio_projects(self) -> None:
fname = 'ballisticacore-windows/BallisticaCore/BallisticaCore.vcxproj'
if os.path.exists(fname):
with open(fname, 'r') as infile:
f_orig = infile.read()
else:
f_orig = None
if f_orig == fcode.replace('\r\n', '\n'):
unchanged_project_count += 1
else:
if check:
print(f'{CLRRED}ERROR: found out-of-date'
f' project file: {fname}{CLREND}')
self._update_visual_studio_project(fname, '..\\..\\src')
fname = ('ballisticacore-windows/BallisticaCoreHeadless/'
'BallisticaCoreHeadless.vcxproj')
if os.path.exists(fname):
self._update_visual_studio_project(fname, '..\\..\\src')
fname = ('ballisticacore-windows/BallisticaCoreOculus'
'/BallisticaCoreOculus.vcxproj')
if os.path.exists(fname):
self._update_visual_studio_project(fname, '..\\..\\src')
def _update_cmake_file(self, fname: str) -> None:
with open(fname) as infile:
lines = infile.read().splitlines()
auto_start = lines.index(' #AUTOGENERATED_BEGIN (this section'
' is managed by the "update_project" tool)')
auto_end = lines.index(' #AUTOGENERATED_END')
our_lines = [
' ${BA_SRC_ROOT}/ballistica' + f
for f in sorted(self._src_files + self._header_files)
if not f.endswith('.mm') and not f.endswith('.m')
]
filtered = lines[:auto_start + 1] + our_lines + lines[auto_end:]
self._files_to_write[fname] = '\n'.join(filtered) + '\n'
def _update_cmake_files(self) -> None:
fname = 'ballisticacore-cmake/CMakeLists.txt'
if os.path.exists(fname):
self._update_cmake_file(fname)
fname = ('ballisticacore-android/BallisticaCore'
'/src/main/cpp/CMakeLists.txt')
if os.path.exists(fname):
self._update_cmake_file(fname)
def _find_sources_and_headers(self, scan_dir: str) -> None:
src_files = set()
header_files = set()
exts = ['.c', '.cc', '.cpp', '.cxx', '.m', '.mm']
header_exts = ['.h']
# Gather all sources and headers.
for root, _dirs, files in os.walk(scan_dir):
for ftst in files:
if any(ftst.endswith(ext) for ext in exts):
src_files.add(os.path.join(root, ftst)[len(scan_dir):])
if any(ftst.endswith(ext) for ext in header_exts):
header_files.add(os.path.join(root, ftst)[len(scan_dir):])
self._src_files = sorted(src_files)
self._header_files = sorted(header_files)
def _check_python_files(self) -> None:
# Make sure all module dirs in python scripts contain an __init__.py
scripts_dir = 'assets/src/data/scripts'
for root, _dirs, files in os.walk(scripts_dir):
if (root != scripts_dir and '__pycache__' not in root
and os.path.basename(root) != '.vscode'):
if '__init__.py' not in files:
print(CLRRED + 'Error: no __init__.py in package dir: ' +
root + CLREND)
sys.exit(255)
def _check_sync_states(self) -> None:
# Make sure none of our sync targets have been mucked with since
# their last sync.
if os.system('tools/snippets sync check') != 0:
print(CLRRED + 'Sync check failed; you may need to run "sync".' +
CLREND)
sys.exit(255)
def _update_assets_makefile(self) -> None:
if os.path.exists('tools/update_assets_makefile'):
if os.system('tools/update_assets_makefile' + self._checkarg) != 0:
print(CLRRED + 'Error checking/updating assets Makefile' +
CLREND)
sys.exit(255)
print(f'{CLRBLU}Writing project file: {fname}{CLREND}')
with open(fname, 'w') as outfile:
outfile.write(fcode)
if unchanged_project_count > 0:
print(f'All {unchanged_project_count} project files up to date.')
def _update_generated_code_makefile(self) -> None:
if os.path.exists('tools/update_generated_code_makefile'):
if os.system('tools/update_generated_code_makefile' +
self._checkarg) != 0:
print(CLRRED +
'Error checking/updating generated-code Makefile' +
CLREND)
sys.exit(255)
# Update our local compile-commands file based on any changes to
# our cmake stuff. Do this at end so cmake changes already happened.
if not check and os.path.exists('ballisticacore-cmake'):
if os.system('make .irony/compile_commands.json') != 0:
print(CLRRED + 'Error updating compile-commands.' + CLREND)
sys.exit(255)
def _update_resources_makefile(self) -> None:
if os.path.exists('tools/update_resources_makefile'):
if os.system('tools/update_resources_makefile' +
self._checkarg) != 0:
print(CLRRED + 'Error checking/updating resources Makefile' +
CLREND)
sys.exit(255)
# Lastly update our dummy _ba module.
# We need to do this very last because it may run the cmake build
# so its success may depend on the cmake build files having already
# been updated.
if os.path.exists('tools/gendummymodule.py'):
if os.system('tools/gendummymodule.py' + checkarg) != 0:
print(CLRRED + 'Error checking/updating dummy module' + CLREND)
sys.exit(255)
if check:
print('Check-Builds: Everything up to date.')
else:
print('Update-Builds: SUCCESS!')
def _update_python_enums_module(self) -> None:
if os.path.exists('tools/update_python_enums_module'):
if os.system('tools/update_python_enums_module' +
self._checkarg) != 0:
print(CLRRED + 'Error checking/updating python enums module' +
CLREND)
sys.exit(255)
if __name__ == '__main__':
main()
App().run()