efrocache improvements work in progress

.idea/dictionaries/ericf.xml

@@ -3,6 +3,7 @@
     <words>
       <w>_babase</w>
       <w>aaaa</w>
+      <w>aaaand</w>
       <w>aaab</w>
       <w>aaac</w>
       <w>aaad</w>
@@ -1234,6 +1235,7 @@
       <w>getobjs</w>
       <w>getopt</w>
       <w>getplayer</w>
+      <w>getprojectconfig</w>
       <w>getpt</w>
       <w>getr</w>
       <w>getrefs</w>
@@ -2443,6 +2445,7 @@
       <w>reimported</w>
       <w>relfut</w>
       <w>relpath</w>
+      <w>relurl</w>
       <w>relwithdebinfo</w>
       <w>remainingchecks</w>
       <w>remoteapp</w>
@@ -2623,6 +2626,7 @@
       <w>setmusic</w>
       <w>setname</w>
       <w>setnode</w>
+      <w>setprojectconfig</w>
       <w>setsticky</w>
       <w>settingname</w>
       <w>setversion</w>

CHANGELOG.md

@@ -1,5 +1,16 @@
-### 1.7.25 (build 21202, api 8, 2023-07-28)
+### 1.7.25 (build 21204, api 8, 2023-07-28)
 
+- `getconfig` and `setconfig` in `efrotools` are now `getprojectconfig` and
+  `setprojectconfig` (to reflect the file name changes that happened in 1.7.20).
+- The efrocache system (how assets and prebuilt binaries are downloaded during
+  builds) now uses a `efrocache_repository_url` value in
+  `config/projectconfig.json` instead of being hard-coded to my server. This
+  makes it possible to theoretically set up mirror servers. I currently keep the
+  cache pruned to the last few months worth of files but theoretically someone
+  could set up a server that never gets pruned and contains all history from now
+  until forever. Efrocache is basically just a big pile of files organized by
+  their hashes (see `tools/efrotools/efrocache.py` for details).
+  
 ### 1.7.24 (build 21199, api 8, 2023-07-27)
 
 - Fixed an issue where respawn icons could disappear in epic mode (Thanks for

ballisticakit-cmake/.idea/dictionaries/ericf.xml

@@ -3,6 +3,7 @@
     <words>
       <w>NOMINMAX</w>
       <w>_babase</w>
+      <w>aaaand</w>
       <w>aabb</w>
       <w>aate</w>
       <w>abcdefghijklmnopqrstuvwxyz</w>
@@ -745,6 +746,7 @@
       <w>getpackagemodel</w>
       <w>getpackagesound</w>
       <w>getpackagetexture</w>
+      <w>getprojectconfig</w>
       <w>getpublicpartyenabled</w>
       <w>getpublicpartymaxsize</w>
       <w>getqrcodetexture</w>
@@ -1444,6 +1446,7 @@
       <w>reimported</w>
       <w>relfut</w>
       <w>reloadmedia</w>
+      <w>relurl</w>
       <w>relwithdebinfo</w>
       <w>rendererdata</w>
       <w>rendertarget</w>
@@ -1538,6 +1541,7 @@
       <w>setdata</w>
       <w>setname</w>
       <w>setnode</w>
+      <w>setprojectconfig</w>
       <w>setpublicpartyenabled</w>
       <w>setpublicpartymaxsize</w>
       <w>setpublicpartyname</w>

config/projectconfig.json

@@ -18,6 +18,7 @@
     "src/ballistica/base/platform/oculus/main_rift.cc",
     "src/ballistica/core/platform/android/android_gl3.c"
   ],
+  "efrocache_repository_url": "https://files.ballistica.net/cache/ba1",
   "name": "BallisticaKit",
   "public": true,
   "pylint_ignored_untracked_deps": [

src/assets/ba_data/python/baenv.py

@@ -50,7 +50,7 @@ if TYPE_CHECKING:
 
 # Build number and version of the ballistica binary we expect to be
 # using.
-TARGET_BALLISTICA_BUILD = 21202
+TARGET_BALLISTICA_BUILD = 21204
 TARGET_BALLISTICA_VERSION = '1.7.25'
 
 

src/ballistica/shared/ballistica.cc

@@ -39,7 +39,7 @@ auto main(int argc, char** argv) -> int {
 namespace ballistica {
 
 // These are set automatically via script; don't modify them here.
-const int kEngineBuildNumber = 21202;
+const int kEngineBuildNumber = 21204;
 const char* kEngineVersion = "1.7.25";
 
 #if BA_MONOLITHIC_BUILD

tools/batools/assetsmakefile.py

@@ -404,10 +404,10 @@ def generate_assets_makefile(
 ) -> dict[str, str]:
     """Main script entry point."""
     # pylint: disable=too-many-locals
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
     from pathlib import Path
 
-    public = getconfig(Path(projroot))['public']
+    public = getprojectconfig(Path(projroot))['public']
     assert isinstance(public, bool)
 
     original = existing_data

tools/batools/metamakefile.py

@@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
 from dataclasses import dataclass
 
 from efro.error import CleanError
-from efrotools import getconfig
+from efrotools import getprojectconfig
 
 if TYPE_CHECKING:
     pass
@@ -81,7 +81,7 @@ class MetaMakefileGenerator:
         """Do the thing."""
         # pylint: disable=too-many-locals
 
-        public = getconfig(Path(self._projroot))['public']
+        public = getprojectconfig(Path(self._projroot))['public']
         assert isinstance(public, bool)
 
         fname = 'src/meta/Makefile'

tools/batools/pcommand.py

@@ -446,9 +446,9 @@ def warm_start_asset_build() -> None:
     import os
     import subprocess
     from pathlib import Path
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
 
-    public: bool = getconfig(PROJROOT)['public']
+    public: bool = getprojectconfig(PROJROOT)['public']
 
     if public:
         from efrotools.efrocache import warm_start_cache

tools/batools/project/_updater.py

@@ -9,7 +9,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 from dataclasses import dataclass
 
-from efrotools import getconfig, getlocalconfig
+from efrotools import getprojectconfig, getlocalconfig
 from efro.error import CleanError
 from efro.terminal import Clr
 
@@ -60,7 +60,9 @@ class ProjectUpdater:
             raise RuntimeError('fix and check cannot both be enabled')
 
         # We behave a bit differently in the public repo.
-        self.public: bool = getconfig(Path(projroot)).get('public', False)
+        self.public: bool = getprojectconfig(Path(projroot)).get(
+            'public', False
+        )
         assert isinstance(self.public, bool)
 
         self._source_files: list[str] | None = None
@@ -191,7 +193,7 @@ class ProjectUpdater:
             if self.public:
                 sources = []
             else:
-                sources = getconfig(Path(self.projroot)).get(
+                sources = getprojectconfig(Path(self.projroot)).get(
                     'internal_source_files', []
                 )
             if not isinstance(sources, list):
@@ -209,7 +211,7 @@ class ProjectUpdater:
             if self.public:
                 sources = []
             else:
-                sources = getconfig(Path(self.projroot)).get(
+                sources = getprojectconfig(Path(self.projroot)).get(
                     'internal_source_dirs', []
                 )
             if not isinstance(sources, list):

tools/batools/resourcesmakefile.py

@@ -57,9 +57,9 @@ class ResourcesMakefileGenerator:
         existing_data: str,
         projname: str,
     ) -> None:
-        from efrotools import getconfig
+        from efrotools import getprojectconfig
 
-        self.public = getconfig(Path(projroot))['public']
+        self.public = getprojectconfig(Path(projroot))['public']
         assert isinstance(self.public, bool)
         self.existing_data = existing_data
         self.projroot = projroot

tools/batools/spinoff/_context.py

@@ -15,7 +15,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, assert_never
 
 from efrotools.code import format_python_str, format_cpp_str
-from efrotools import getconfig, replace_exact
+from efrotools import getprojectconfig, replace_exact
 from efro.error import CleanError
 from efro.terminal import Clr
 from efro.util import timedelta_str
@@ -113,7 +113,7 @@ class SpinoffContext:
 
         self._src_name = 'BallisticaKit'
 
-        self._public: bool = getconfig(Path(self._src_root))['public']
+        self._public: bool = getprojectconfig(Path(self._src_root))['public']
         assert isinstance(self._public, bool)
 
         self._src_all_feature_sets = {

tools/batools/spinoff/_main.py

@@ -97,9 +97,9 @@ def _main() -> None:
     elif cmd is Command.CREATE:
         _do_create(src_root, dst_root)
     elif cmd is Command.ADD_SUBMODULE_PARENT:
-        from efrotools import getconfig
+        from efrotools import getprojectconfig
 
-        public = getconfig(Path(dst_root))['public']
+        public = getprojectconfig(Path(dst_root))['public']
         _do_add_submodule_parent(dst_root, is_new=False, public=public)
     else:
         assert_never(cmd)
@@ -130,7 +130,7 @@ def _do_create(src_root: str | None, dst_root: str) -> None:
     # pylint: disable=too-many-locals, cyclic-import
     from efrotools import extract_arg, extract_flag
     from efrotools.code import format_python_str
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
     import batools.spinoff
 
     # Note: in our case dst_root is actually what becomes the src project
@@ -204,7 +204,7 @@ def _do_create(src_root: str | None, dst_root: str) -> None:
     # on git so its best to always do this.
     subprocess.run(['git', 'init'], cwd=path, check=True, capture_output=True)
 
-    public = getconfig(Path(dst_root))['public']
+    public = getprojectconfig(Path(dst_root))['public']
 
     if submodule_parent:
         _do_add_submodule_parent(path, is_new=True, public=public)

tools/batools/spinoff/_test.py

@@ -20,7 +20,7 @@ def spinoff_test(args: list[str]) -> None:
     import subprocess
 
     from batools.featureset import FeatureSet
-    from efrotools import extract_flag, getconfig
+    from efrotools import extract_flag, getprojectconfig
     from efro.terminal import Clr
     from efro.error import CleanError
 
@@ -31,7 +31,7 @@ def spinoff_test(args: list[str]) -> None:
             "spinoff-test: can't pass both submodule parent"
             ' and shared test parent.'
         )
-    public = getconfig(Path('.'))['public']
+    public = getprojectconfig(Path('.'))['public']
     if shared_test_parent and public:
         raise CleanError('--shared-test-parent not available in public repo.')
 

tools/batools/toplevelmakefile.py

@@ -14,10 +14,10 @@ if TYPE_CHECKING:
 
 def generate_top_level_makefile(projroot: str, existing_data: str) -> str:
     """Main script entry point."""
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
     from pathlib import Path
 
-    public = getconfig(Path(projroot))['public']
+    public = getprojectconfig(Path(projroot))['public']
     assert isinstance(public, bool)
 
     original = existing_data

tools/efrotools/__init__.py

@@ -34,7 +34,7 @@ def explicit_bool(value: bool) -> bool:
     return value
 
 
-def getlocalconfig(projroot: Path) -> dict[str, Any]:
+def getlocalconfig(projroot: Path | str) -> dict[str, Any]:
     """Return a project's localconfig contents (or default if missing)."""
     localconfig: dict[str, Any]
 
@@ -51,7 +51,7 @@ def getlocalconfig(projroot: Path) -> dict[str, Any]:
     return localconfig
 
 
-def getconfig(projroot: Path) -> dict[str, Any]:
+def getprojectconfig(projroot: Path | str) -> dict[str, Any]:
     """Return a project's projectconfig contents (or default if missing)."""
     config: dict[str, Any]
     try:
@@ -64,7 +64,7 @@ def getconfig(projroot: Path) -> dict[str, Any]:
     return config
 
 
-def setconfig(projroot: Path, config: dict[str, Any]) -> None:
+def setprojectconfig(projroot: Path | str, config: dict[str, Any]) -> None:
     """Set the project config contents."""
     os.makedirs(Path(projroot, 'config'), exist_ok=True)
     with Path(projroot, 'config/projectconfig.json').open(

tools/efrotools/code.py

@@ -138,7 +138,7 @@ def check_cpplint(projroot: Path, full: bool) -> None:
     from concurrent.futures import ThreadPoolExecutor
     from multiprocessing import cpu_count
 
-    from efrotools import getconfig, PYVER
+    from efrotools import getprojectconfig, PYVER
     from efro.terminal import Clr
 
     os.chdir(projroot)
@@ -148,7 +148,9 @@ def check_cpplint(projroot: Path, full: bool) -> None:
             raise RuntimeError(f'Found space in path {fpath}; unexpected.')
 
     # Check the config for a list of ones to ignore.
-    code_blacklist: list[str] = getconfig(projroot).get('cpplint_blacklist', [])
+    code_blacklist: list[str] = getprojectconfig(projroot).get(
+        'cpplint_blacklist', []
+    )
 
     # Just pretend blacklisted ones don't exist.
     filenames = [f for f in filenames if f not in code_blacklist]
@@ -220,10 +222,10 @@ def get_code_filenames(projroot: Path, include_generated: bool) -> list[str]:
     could cause dirty generated files to not get updated properly when
     their sources change).
     """
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
 
     exts = ('.h', '.c', '.cc', '.cpp', '.cxx', '.m', '.mm')
-    places = getconfig(projroot).get('code_source_dirs', None)
+    places = getprojectconfig(projroot).get('code_source_dirs', None)
     if places is None:
         raise RuntimeError('code_source_dirs not declared in config')
     codefilenames = []
@@ -337,12 +339,12 @@ def _should_include_script(fnamefull: str) -> bool:
 
 def get_script_filenames(projroot: Path) -> list[str]:
     """Return the Python filenames to lint-check or auto-format."""
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
 
     proot = f'{projroot}/'
 
     filenames = set()
-    places = getconfig(projroot).get('python_source_dirs', None)
+    places = getprojectconfig(projroot).get('python_source_dirs', None)
     if places is None:
         raise RuntimeError('python_source_dirs not declared in config')
     for place in places:
@@ -561,7 +563,7 @@ def _apply_pylint_run_to_cache(
 
     from astroid import modutils
 
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
 
     # First off, build a map of dirtyfiles to module names
     # (and the corresponding reverse map).
@@ -609,7 +611,7 @@ def _apply_pylint_run_to_cache(
                 untracked_deps.add(mname)
 
     ignored_untracked_deps: set[str] = set(
-        getconfig(projroot).get('pylint_ignored_untracked_deps', [])
+        getprojectconfig(projroot).get('pylint_ignored_untracked_deps', [])
     )
 
     # Add a few that this package itself triggers.

tools/efrotools/efrocache.py

@@ -31,8 +31,6 @@ from efro.terminal import Clr
 if TYPE_CHECKING:
     pass
 
-BASE_URL = 'https://files.ballistica.net/cache/ba1/'
-
 TARGET_TAG = '# __EFROCACHE_TARGET__'
 
 CACHE_DIR_NAME = '.efrocache'
@@ -57,6 +55,21 @@ g_cache_prefix_noexec: bytes | None = None
 g_cache_prefix_exec: bytes | None = None
 
 
+def get_repository_base_url() -> str:
+    """Return the base repository url (assumes cwd is project root)."""
+    # from efrotools import getprojectconfig
+    import efrotools
+
+    pconfig = efrotools.getprojectconfig('.')
+    name = 'efrocache_repository_url'
+    val = pconfig.get(name)
+    if not isinstance(val, str):
+        raise RuntimeError(f"'{name}' was not found in projectconfig.")
+    if val.endswith('/'):
+        raise RuntimeError('Repository string should not end in a slash.')
+    return val
+
+
 def get_existing_file_hash(path: str) -> str:
     """Return the hash used for caching."""
     import hashlib
@@ -95,7 +108,15 @@ def get_target(path: str) -> None:
         efrocachemap = json.loads(infile.read())
     if path not in efrocachemap:
         raise RuntimeError(f'Path not found in efrocache: {path}')
-    url = efrocachemap[path]
+    relurl = efrocachemap[path]
+
+    # These used to be abs paths but are now relative.
+    assert not relurl.startswith('https:')
+    assert not relurl.startswith('/')
+
+    repo = get_repository_base_url()
+    url = f'{repo}/{relurl}'
+
     subpath = '/'.join(url.split('/')[-3:])
     local_cache_path = os.path.join(CACHE_DIR_NAME, subpath)
     local_cache_path_dl = local_cache_path + '.download'
@@ -198,7 +219,7 @@ def filter_makefile(makefile_dir: str, contents: str) -> str:
     lines = contents.splitlines()
     pcommand = 'tools/pcommand'
 
-    # Replace cachable targets with cache lookups
+    # Replace cachable targets with cache lookups.
     while TARGET_TAG in lines:
         index = lines.index(TARGET_TAG)
         endindex = index
@@ -378,14 +399,16 @@ def _write_cache_files(
     with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
         results = executor.map(call, fnames1)
     for result in results:
-        mapping[result[0]] = BASE_URL + result[1]
+        # mapping[result[0]] = f'{base_url}/{result[1]}'
+        mapping[result[0]] = result[1]
         fhashes1.add(result[1])
 
     # Now finish up with the second set.
     with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
         results = executor.map(call, fnames2)
     for result in results:
-        mapping[result[0]] = BASE_URL + result[1]
+        # mapping[result[0]] = f'{base_url}/result[1]'
+        mapping[result[0]] = result[1]
         fhashes2.add(result[1])
 
     # We want the server to have a startercache.tar.xz file which
@@ -395,8 +418,8 @@ def _write_cache_files(
     # upload that.
 
     # Also let's have the script touch both sets of files so we can use
-    # mod-times to prune older files. (otherwise files that never change
-    # might have very old mod times)
+    # mod-times to prune older files. Otherwise files that never change
+    # might have very old mod times.
     script = (
         'import os\n'
         'import pathlib\n'
@@ -508,6 +531,8 @@ def _check_warm_start_entries(entries: list[tuple[str, str]]) -> None:
 def warm_start_cache() -> None:
     """Run a pre-pass on the efrocache to improve efficiency."""
 
+    base_url = get_repository_base_url()
+
     # We maintain a starter-cache on the staging server, which is simply
     # the latest set of cache entries compressed into a single
     # compressed archive. If we have no local cache yet we can download
@@ -518,7 +543,7 @@ def warm_start_cache() -> None:
     if not os.path.exists(CACHE_DIR_NAME):
         print('Downloading asset starter-cache...', flush=True)
         subprocess.run(
-            f'curl --fail {BASE_URL}startercache.tar.xz'
+            f'curl --fail {base_url}/startercache.tar.xz'
             f' --output startercache.tar.xz',
             shell=True,
             check=True,

tools/efrotools/ios.py

@@ -9,7 +9,7 @@ import subprocess
 import sys
 from dataclasses import dataclass
 
-from efrotools import getlocalconfig, getconfig
+from efrotools import getprojectconfig, getlocalconfig
 
 MODES = {
     'debug': {'configuration': 'Debug'},
@@ -59,7 +59,8 @@ def push_ipa(
     from efrotools.xcodebuild import project_build_path
 
     # Load both the local and project config data.
-    cfg = Config(**getconfig(root)['push_ipa_config'])
+    # FIXME: switch this to use dataclassio.
+    cfg = Config(**getprojectconfig(root)['push_ipa_config'])
     lcfg = LocalConfig(**getlocalconfig(root)['push_ipa_local_config'])
 
     if modename not in MODES:

tools/efrotools/pcommand.py

@@ -491,7 +491,7 @@ def tool_config_install() -> None:
 def _filter_tool_config(cfg: str) -> str:
     import textwrap
 
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
 
     # Stick project-root wherever they want.
     cfg = cfg.replace('__EFRO_PROJECT_ROOT__', str(PROJROOT))
@@ -499,7 +499,7 @@ def _filter_tool_config(cfg: str) -> str:
     # Stick a colon-separated list of project Python paths wherever they want.
     name = '__EFRO_PYTHON_PATHS__'
     if name in cfg:
-        pypaths = getconfig(PROJROOT).get('python_paths')
+        pypaths = getprojectconfig(PROJROOT).get('python_paths')
         if pypaths is None:
             raise RuntimeError('python_paths not set in project config')
         assert not any(' ' in p for p in pypaths)
@@ -560,7 +560,7 @@ truthy-function, unused-awaitable
     # Gen a pylint init hook which sets up our python paths.
     pylint_init_tag = '__EFRO_PYLINT_INIT__'
     if pylint_init_tag in cfg:
-        pypaths = getconfig(PROJROOT).get('python_paths')
+        pypaths = getprojectconfig(PROJROOT).get('python_paths')
         if pypaths is None:
             raise RuntimeError('python_paths not set in project config')
         cstr = 'init-hook=import sys;'
@@ -633,14 +633,14 @@ def sync_all() -> None:
 
 def sync() -> None:
     """Runs standard syncs between this project and others."""
-    from efrotools import getconfig
+    from efrotools import getprojectconfig
     from efrotools.sync import Mode, SyncItem, run_standard_syncs
 
     mode = Mode(sys.argv[2]) if len(sys.argv) > 2 else Mode.PULL
 
     # Load sync-items from project config and run them
     sync_items = [
-        SyncItem(**i) for i in getconfig(PROJROOT).get('sync_items', [])
+        SyncItem(**i) for i in getprojectconfig(PROJROOT).get('sync_items', [])
     ]
     run_standard_syncs(PROJROOT, mode, sync_items)
 
@@ -674,11 +674,11 @@ def pytest() -> None:
     import os
     import platform
     import subprocess
-    from efrotools import getconfig, PYTHON_BIN
+    from efrotools import getprojectconfig, PYTHON_BIN
     from efro.error import CleanError
 
     # Grab our python paths for the project and stuff them in PYTHONPATH.
-    pypaths = getconfig(PROJROOT).get('python_paths')
+    pypaths = getprojectconfig(PROJROOT).get('python_paths')
     if pypaths is None:
         raise CleanError('python_paths not found in project config.')