Latest binaries and build system optimizations

.efrocachemap

@@ -4117,8 +4117,8 @@
   "assets/build/windows/x64/pythonw.exe": "https://files.ballistica.net/cache/ba1/6c/bb/b6f52c306aa4e88061510e96cefe",
   "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/8c/ae/8d2561ca2c4bb1bb033560866410",
   "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/74/2b/eea5b942b0cead421529d09039cd",
-  "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/41/22/06f954698bd228a23c0423830236",
+  "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/75/f5/740ce4ae3fc843c4cec96967f75a",
   "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/d0/1f/573bbb85fbb6a3bf8a056caeeaf9",
-  "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/5b/5b/1dcc8ebb16f82a08531c945909fb",
+  "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/35/6d/c5e69424a36f80c0fac65dc684ca",
-  "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/f8/d7/dc541bfca363ead79e0c3edf6d61"
+  "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/a6/83/e3ed78e10bf251b44168ddd73600"
 }

.idea/dictionaries/ericf.xml

@@ -151,8 +151,11 @@
       <w>bbot</w>
       <w>bbtn</w>
       <w>bcppcompiler</w>
+      <w>bdfl</w>
       <w>belarussian</w>
       <w>benboncan</w>
+      <w>bfiledir</w>
+      <w>bfiles</w>
       <w>bgmodel</w>
       <w>bgterrain</w>
       <w>bgtex</w>
@@ -191,6 +194,7 @@
       <w>bsfoundation</w>
       <w>bsmaster</w>
       <w>bsmusic</w>
+      <w>bsources</w>
       <w>bsplaylist</w>
       <w>bsremote</w>
       <w>bsstd</w>
@@ -290,6 +294,7 @@
       <w>clrhdr</w>
       <w>clrred</w>
       <w>cmathmodule</w>
+      <w>cmds</w>
       <w>cmembers</w>
       <w>cmodel</w>
       <w>cnode</w>
@@ -845,6 +850,7 @@
       <w>inits</w>
       <w>inmobi</w>
       <w>inpath</w>
+      <w>inpaths</w>
       <w>inputdevice</w>
       <w>inputfiles</w>
       <w>inputhash</w>
@@ -1322,6 +1328,7 @@
       <w>priceraw</w>
       <w>printnodes</w>
       <w>printobjects</w>
+      <w>printpaths</w>
       <w>priv</w>
       <w>proactor</w>
       <w>proc</w>
@@ -1675,6 +1682,7 @@
       <w>taobaomascot</w>
       <w>targ</w>
       <w>targetdir</w>
+      <w>targetname</w>
       <w>targetpath</w>
       <w>targetpractice</w>
       <w>tbtcolor</w>

Makefile

@@ -28,6 +28,23 @@
 # Prefix used for output of docs/changelogs/etc targets for use in webpages.
 DOCPREFIX = "ballisticacore_"
 
+# This setup lets us set up files "bfiles" for expensive dummy targets
+# to avoid re-running them every time. An good use case is VM build targets
+# where just spinning up the VM to confirm that nothing needs rebuilding is
+# time-consuming. To use these, do the following:
+# - create a physical file for the target: ${BFILEDIR}/targetname
+#   (targets that are already physical files work too)
+# - add this dependency to it: ${shell ${BSOURCES} <category>}
+#   (where <category> covers all files that could affect the target)
+# - always touch the target file as the last build step:
+#   mkdir -p `dirname ${@}` && touch ${@}
+#   (even if the build step usually does; the build may not actually run
+#    which could leave one of the overly-broad dep files newer than it)
+# Note that this mechanism slows builds a bit if category contains a lot of
+# files, so is not always a win.
+BFILEDIR = .cache/bfile
+BSOURCES = tools/snippets sources
+
 
 ################################################################################
 #                                                                              #
@@ -86,6 +103,18 @@ assets-android: prereqs
 assets-clean:
 	@cd assets && $(MAKE) clean
 
+# A bfile for the resources target so we don't always have to run it.
+RESOURCES_F = ${BFILEDIR}/resources
+${RESOURCES_F}: ${PREREQS} resources/Makefile ${shell ${BSOURCES} resources}
+	@cd resources && $(MAKE) -j${CPUS} resources
+	@mkdir -p `dirname ${@}` && touch ${@}
+
+# A bfile for the code target so we don't always have to run it.
+CODE_F = ${BFILEDIR}/code
+${CODE_F}: ${PREREQS} ${shell ${BSOURCES} gen}
+	@cd src/generated_src && $(MAKE) -j${CPUS} generated_code
+	@mkdir -p `dirname ${@}` && touch ${@}
+
 # Remove *ALL* files and directories that aren't managed by git
 # (except for a few things such as localconfig.json).
 clean:

tools/efrotools/efrocache.py

@@ -202,6 +202,12 @@ def update_cache(makefile_dirs: List[str]) -> None:
             else:
                 fnames2.append(fullpath)
 
+    # if bool(True):
+    #     print("1", fnames1)
+    #     print("2", fnames2)
+    #     print('SO FAR SO GOOD')
+    #     sys.exit(0)
+
     staging_dir = 'build/efrocache'
     mapping_file = 'build/efrocachemap'
     run(f'rm -rf {staging_dir}')
@@ -224,32 +230,6 @@ def update_cache(makefile_dirs: List[str]) -> None:
     print(f'Cache update successful!')
 
 
-def _write_cache_file(staging_dir: str, fname: str) -> Tuple[str, str]:
-    import hashlib
-    from efrotools import run
-    print(f'Caching {fname}')
-    if ' ' in fname:
-        raise RuntimeError('Spaces in paths not supported.')
-
-    # Just going with ol' md5 here; we're the only ones creating these so
-    # security isn't a concern.
-    md5 = hashlib.md5()
-    with open(fname, 'rb') as infile:
-        md5.update(infile.read())
-    md5.update(fname.encode())
-    finalhash = md5.hexdigest()
-    hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
-    path = os.path.join(staging_dir, hashpath)
-    os.makedirs(os.path.dirname(path), exist_ok=True)
-
-    # Fancy pipe stuff which will give us deterministic
-    # tar.gz files (no embedded timestamps)
-    # Note: The 'COPYFILE_DISABLE' prevents mac tar from adding
-    # file attributes/resource-forks to the archive as as ._filename.
-    run(f'COPYFILE_DISABLE=1 tar cf - {fname} | gzip -n > {path}')
-    return fname, hashpath
-
-
 def _write_cache_files(fnames1: List[str], fnames2: List[str],
                        staging_dir: str, mapping_file: str) -> None:
     fhashes1: Set[str] = set()
@@ -301,6 +281,32 @@ def _write_cache_files(fnames1: List[str], fnames2: List[str],
         outfile.write(json.dumps(mapping, indent=2, sort_keys=True))
 
 
+def _write_cache_file(staging_dir: str, fname: str) -> Tuple[str, str]:
+    import hashlib
+    from efrotools import run
+    print(f'Caching {fname}')
+    if ' ' in fname:
+        raise RuntimeError('Spaces in paths not supported.')
+
+    # Just going with ol' md5 here; we're the only ones creating these so
+    # security isn't a concern.
+    md5 = hashlib.md5()
+    with open(fname, 'rb') as infile:
+        md5.update(infile.read())
+    md5.update(fname.encode())
+    finalhash = md5.hexdigest()
+    hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
+    path = os.path.join(staging_dir, hashpath)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+
+    # Fancy pipe stuff which will give us deterministic
+    # tar.gz files (no embedded timestamps)
+    # Note: The 'COPYFILE_DISABLE' prevents mac tar from adding
+    # file attributes/resource-forks to the archive as as ._filename.
+    run(f'COPYFILE_DISABLE=1 tar cf - {fname} | gzip -n > {path}')
+    return fname, hashpath
+
+
 def _check_warm_start_entry(entry: Tuple[str, str]) -> None:
     import hashlib
     fname, filehash = entry

tools/snippets

@@ -876,5 +876,68 @@ def update_makebob() -> None:
     print('All builds complete!', flush=True)
 
 
+def _printpaths(inpaths: List[str], category: str) -> None:
+    paths: List[str] = []
+    for inpath in inpaths:
+        # Add files verbatim; recurse through dirs.
+        if os.path.isfile(inpath):
+            paths.append(inpath)
+            continue
+        for root, _dnames, fnames in os.walk(inpath):
+            # Always skip these..
+            if (root.startswith('src/generated_src')
+                    or root.startswith('src/tools')):
+                continue
+            # Skip some of these...
+            if root.startswith('src/external'):
+                if category == 'win' and root.startswith(
+                        'src/external/windows'):
+                    pass
+                else:
+                    continue
+            # Ignore python cache files.
+            if '__pycache__' in root:
+                continue
+            for fname in fnames:
+                # Ignore dot files
+                if fname.startswith('.'):
+                    continue
+                path = os.path.join(root, fname)
+                if ' ' in path:
+                    raise RuntimeError(f'Invalid path with space: {path}')
+                paths.append(path)
+    print(' '.join(paths))
+
+
+def sources() -> None:
+    """Print source files of different categories.
+
+    These are used as broad, redundant filters for expensive build ops.
+    For instance, when running a build through a VM we might want to skip
+    even spinning up the VM if absolutely no source files have changed.
+    """
+    try:
+        if len(sys.argv) != 3:
+            raise CleanError('Expected one argument.')
+        category = sys.argv[2]
+        if category == 'gen':
+            _printpaths(['tools', 'src/generated_src'], category)
+        elif category == 'assets':
+            _printpaths(['tools', 'assets/src'], category)
+        elif category in ('cmake', 'win'):
+            _printpaths(['tools', 'src'], category)
+        elif category == 'resources':
+            _printpaths(['tools', 'resources/src', 'resources/Makefile'],
+                        category)
+        else:
+            raise ValueError(f'Invalid source category: {category}')
+
+    except Exception as exc:
+        # We're used by ${shell} cmds in Makefiles so need to fail in a
+        # fashion that is noticeable there:
+        print(f'Error in sources snippet: {exc}', file=sys.stderr)
+        print('__nonexistent_error_output__')
+
+
 if __name__ == '__main__':
     snippets_main(globals())