testing efrocache system

.efrocachemap

@@ -0,0 +1,4 @@
+{
+  "assets/build/testfile_a": "https://files.ballistica.net/cache/ba1/c1/4d/25cb4790afaadf2ebf6dd185553a",
+  "build/testfile": "https://files.ballistica.net/cache/ba1/01/ae/0fb710f674cf11761de33b2dfef7"
+}

.idea/dictionaries/ericf.xml

@@ -1,10 +1,6 @@
 <component name="ProjectDictionaryState">
   <dictionary name="ericf">
     <words>
-      <w>maxdepth</w>
-      <w>efrocache</w>
-      <w>packagedir</w>
-      <w>priv</w>
       <w>aaaa</w>
       <w>aaab</w>
       <w>aaac</w>
@@ -117,6 +113,7 @@
       <w>basemult</w>
       <w>basepos</w>
       <w>basespaz</w>
+      <w>baseurl</w>
       <w>basew</w>
       <w>bastd</w>
       <w>bbot</w>
@@ -422,6 +419,8 @@
       <w>editorconfig</w>
       <w>efile</w>
       <w>efro</w>
+      <w>efrocache</w>
+      <w>efrocachemap</w>
       <w>efroemling</w>
       <w>efrotool</w>
       <w>efrotools</w>
@@ -504,6 +503,7 @@
       <w>filterlines</w>
       <w>filterpath</w>
       <w>filterpaths</w>
+      <w>finalhash</w>
       <w>finalmaterials</w>
       <w>firebase</w>
       <w>firestore</w>
@@ -662,6 +662,7 @@
       <w>halign</w>
       <w>handlemessage</w>
       <w>hashopenssl</w>
+      <w>hashpath</w>
       <w>hashstr</w>
       <w>hashval</w>
       <w>hasstarted</w>
@@ -891,6 +892,7 @@
       <w>mathmodule</w>
       <w>mathnode</w>
       <w>mathutils</w>
+      <w>maxdepth</w>
       <w>maxlinks</w>
       <w>maxval</w>
       <w>maxw</w>
@@ -1060,6 +1062,7 @@
       <w>outname</w>
       <w>outpath</w>
       <w>ouya</w>
+      <w>packagedir</w>
       <w>packagedirs</w>
       <w>painttxtattr</w>
       <w>palmos</w>
@@ -1158,6 +1161,7 @@
       <w>priceraw</w>
       <w>printnodes</w>
       <w>printobjects</w>
+      <w>priv</w>
       <w>proactor</w>
       <w>proc</w>
       <w>procs</w>

tools/efrotools/efrocache.py

@@ -21,3 +21,77 @@
 """A simple cloud caching system for making built binaries/assets available."""
 
 from __future__ import annotations
+
+import os
+import subprocess
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import List, Dict
+
+
+def update_cache(makefile_dirs: List[str]) -> None:
+    """Given a list of directories containing makefiles, update caches."""
+
+    import multiprocessing
+    from efrotools import run
+    cpus = multiprocessing.cpu_count()
+    fnames: List[str] = []
+    for path in makefile_dirs:
+        # First, update things..
+        cdp = f'cd {path} && ' if path else ''
+        subprocess.run(f'{cdp}make -j{cpus} efrocache_build',
+                       shell=True,
+                       check=True)
+        # Now get the list of them.
+        fnames += [
+            os.path.join(path, s) for s in subprocess.run(
+                f'{cdp}make efrocache_list',
+                shell=True,
+                check=True,
+                capture_output=True).stdout.decode().split()
+        ]
+    staging_dir = 'build/efrocache'
+    mapping_file = 'build/efrocachemap'
+    run(f'rm -rf {staging_dir}')
+    run(f'mkdir -p {staging_dir}')
+
+    _write_cache_files(fnames, staging_dir, mapping_file)
+
+    # Push what we just wrote to the staging server
+    print('Pushing cache to staging...', flush=True)
+    run('rsync --recursive build/efrocache/'
+        ' ubuntu@ballistica.net:files.ballistica.net/cache/ba1/')
+
+    print(f'Cache update successful!')
+
+
+def _write_cache_files(fnames: List[str], staging_dir: str,
+                       mapping_file: str) -> None:
+    from efrotools import run
+    import hashlib
+    import json
+    mapping: Dict[str, str] = {}
+    baseurl = 'https://files.ballistica.net/cache/ba1/'
+    for fname in fnames:
+
+        if ' ' in fname:
+            raise RuntimeError('Spaces in paths not supported.')
+
+        # Just going with ol' md5 here; we're the only ones creating these so
+        # security isn't a concern.
+        md5 = hashlib.md5()
+        with open(fname, 'rb') as infile:
+            md5.update(infile.read())
+        md5.update(fname.encode())
+        finalhash = md5.hexdigest()
+        hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
+        path = os.path.join(staging_dir, hashpath)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+
+        # Fancy pipe stuff which will give us deterministic
+        # tar.gz files (no embedded timestamps)
+        run(f'tar cf - {fname} | gzip -n > {path}')
+        mapping[fname] = baseurl + hashpath
+    with open(mapping_file, 'w') as outfile:
+        outfile.write(json.dumps(mapping, indent=2, sort_keys=True))

tools/efrotools/snippets.py

@@ -132,11 +132,18 @@ def spelling() -> None:
     if lines[2] != '    <words>':
         raise RuntimeError('Unexpected dictionary format.')
     words = sys.argv[2:]
+    added_count = 0
     for word in words:
-        lines.insert(3, f'      <w>{word.lower()}</w>')
+        line = f'      <w>{word.lower()}</w>'
+        if line not in lines:
+            lines.insert(3, line)
+            added_count += 1
+
     with open(fname, 'w') as outfile:
-        outfile.write('\n'.join(lines))
+        # Sort lines in the words section.
-    print('Added', len(words), 'words to the dictionary.')
+        assert all(l.startswith('      <w>') for l in lines[3:-3])
+        outfile.write('\n'.join(lines[:3] + sorted(lines[3:-3]) + lines[-3:]))
+    print('Added', added_count, 'words to the dictionary.')
 
 
 def check_clean_safety() -> None:

tools/snippets

@@ -599,7 +599,9 @@ def capitalize() -> None:
 
 def efrocache_update() -> None:
     """Build & push files to efrocache for public access."""
-    print('WOULD UPDATE EFROCACHE')
+    from efrotools.efrocache import update_cache
+    makefile_dirs = ['', 'assets']
+    update_cache(makefile_dirs)
 
 
 def efrocache_get() -> None: