diff --git a/.efrocachemap b/.efrocachemap index 57ab8ece..db8cd596 100644 --- a/.efrocachemap +++ b/.efrocachemap @@ -4132,16 +4132,16 @@ "assets/build/windows/x64/python.exe": "https://files.ballistica.net/cache/ba1/25/a7/dc87c1be41605eb6fefd0145144c", "assets/build/windows/x64/python37.dll": "https://files.ballistica.net/cache/ba1/b9/e4/d912f56e42e9991bcbb4c804cfcb", "assets/build/windows/x64/pythonw.exe": "https://files.ballistica.net/cache/ba1/6c/bb/b6f52c306aa4e88061510e96cefe", - "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/6a/56/e48e9306428b4ae7b58225fce76c", - "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/dd/2d/bdd22cbd39a9bb27a75302ef837a", - "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/61/f2/a679c5a3c9113c7b7599a59415fb", - "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/6c/21/c80e03031e6c5959140588d439bf", - "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/42/ff/6d9056500a768dae79ed44180796", - "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/58/d4/3f6d3cb75988db046f22bfb5c2a5", - "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/a8/07/edf6360c8e5b824633bf55c97ed9", - "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/27/cc/0ec7cab492f111923320dc165427", - "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/b2/b4/f68901c3dae4267a5640bf702d86", - "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/ae/b7/0400bd1021d0fc0c40d1c2149364", - "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/ee/6e/8f88fef729d85a07910cbedbac31", - "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/e1/90/4877825fb2d779bd7ac9704a1a4c" + "build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/8b/e2/8ab3f02c813d2e43df787d478023", + "build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ca/46/ae21c3d6765f6625ac06b9215f3d", + "build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/21/f2/079c550e8ba0e255c6255dc0737a", + "build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/19/f3/1121125a99e5efb3577a491799e9", + "build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/87/b1/4d076556aac39f44ccb03278f584", + "build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/84/f6/607fd2ab859f610fdd91a9a857db", + "build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/40/f3/00df4a55512b824534bda6945204", + "build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/f8/2f/09295de6690e6df09c0003b4d85b", + "build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/61/62/2e8398b3dcf1ae702ba591aa947e", + "build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/e1/db/1acefefb21e4fd0fe8e337647592", + "build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/ef/85/b8235ca82f2e3319850535558f0f", + "build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/26/ba/1f40445d853e6be53beacd7a3ca3" } \ No newline at end of file diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index b847cba0..7dea5109 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -17,6 +17,7 @@ abeb abot abtn + accesstime accountname accountui accum @@ -687,6 +688,7 @@ fsplit fsrc fstab + fstat fstrs ftime ftmp diff --git a/tools/efrotools/efrocache.py b/tools/efrotools/efrocache.py index 73ef3dd2..5f8b8542 100644 --- a/tools/efrotools/efrocache.py +++ b/tools/efrotools/efrocache.py @@ -284,8 +284,9 @@ def _gen_hashes(fnames: List[str]) -> str: def _write_cache_files(fnames1: List[str], fnames2: List[str], staging_dir: str, mapping_file: str) -> None: - fhashes1: Set[str] = set() import functools + fhashes1: Set[str] = set() + fhashes2: Set[str] = set() mapping: Dict[str, str] = {} call = functools.partial(_write_cache_file, staging_dir) @@ -296,21 +297,38 @@ def _write_cache_files(fnames1: List[str], fnames2: List[str], mapping[result[0]] = BASE_URL + result[1] fhashes1.add(result[1]) + # Now finish up with the second set. + with ThreadPoolExecutor(max_workers=cpu_count()) as executor: + results = executor.map(call, fnames2) + for result in results: + mapping[result[0]] = BASE_URL + result[1] + fhashes2.add(result[1]) + # We want the server to have a startercache.tar.xz file which contains - # this entire first set. It is much more efficient to build that file + # the entire first set. It is much more efficient to build that file # on the server than it is to build it here and upload the whole thing. # ...so let's simply write a script to generate it and upload that. + # Also let's have the script touch both sets of files so we can use + # mod-times to prune older files. (otherwise files that never change + # might have very old mod times) script = ( 'import os\n' + 'import pathlib\n' 'import subprocess\n' 'fnames = ' + repr(fhashes1) + '\n' + 'fnames2 = ' + repr(fhashes2) + '\n' 'subprocess.run(["rm", "-rf", "efrocache"], check=True)\n' 'print("Copying starter cache files...", flush=True)\n' 'for fname in fnames:\n' ' dst = os.path.join("efrocache", fname)\n' ' os.makedirs(os.path.dirname(dst), exist_ok=True)\n' ' subprocess.run(["cp", fname, dst], check=True)\n' + 'print("Touching full file set...", flush=True)\n' + 'for fname in list(fnames) + list(fnames2):\n' + ' fpath = pathlib.Path(fname)\n' + ' assert fpath.exists()\n' + ' fpath.touch()\n' 'print("Compressing starter cache archive...", flush=True)\n' 'subprocess.run(["tar", "-Jcf", "tmp.tar.xz", "efrocache"],' ' check=True)\n' @@ -322,12 +340,6 @@ def _write_cache_files(fnames1: List[str], fnames2: List[str], with open('build/efrocache/genstartercache.py', 'w') as outfile: outfile.write(script) - # Now finish up with the second set. - with ThreadPoolExecutor(max_workers=cpu_count()) as executor: - results = executor.map(call, fnames2) - for result in results: - mapping[result[0]] = BASE_URL + result[1] - with open(mapping_file, 'w') as outfile: outfile.write(json.dumps(mapping, indent=2, sort_keys=True)) diff --git a/tools/staging_server_upkeep b/tools/staging_server_upkeep new file mode 100755 index 00000000..8ab000da --- /dev/null +++ b/tools/staging_server_upkeep @@ -0,0 +1,82 @@ +#!/usr/bin/env python3.7 +# Copyright (c) 2011-2020 Eric Froemling +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ----------------------------------------------------------------------------- +"""Perform various tidying on the staging server.""" + +from __future__ import annotations + +import sys +import os +import subprocess +import stat +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + pass + + +def run() -> None: + """Do the thing.""" + rootdir = 'files.ballistica.net/cache/ba1' + assert os.path.isdir(rootdir) + + # Just make a temp file to get 'now'. + # (ugly but effective) + tmppath = Path('_tmp_time_test') + tmppath.touch() + curtime = tmppath.stat()[stat.ST_ATIME] + tmppath.unlink() + + oldest_age = 0 + + for root, _dirnames, fnames in os.walk(rootdir): + for fname in fnames: + # print('doing', os.path.join(root, fname)) + fstat = os.stat(os.path.join(root, fname)) + accesstime = fstat[stat.ST_MTIME] + age = curtime - accesstime + oldest_age = max(oldest_age, age) + # print('was', (curtime - accesstime)/60, 'min') + # print('TEMP BREW') + # break + print('oldest is', oldest_age / 60 / 60 // 24, 'day') + + +def remote_run() -> None: + """Upload and execute ourself.""" + + print(f'Uploading {__file__}...') + subprocess.run(['rsync', __file__, 'ubuntu@ballistica.net:'], check=True) + + subprocess.run([ + 'ssh', 'ubuntu@ballistica.net', + './staging_server_upkeep -run && rm staging_server_upkeep' + ], + check=True) + print('SUCCESS') + + +if __name__ == '__main__': + if '-run' in sys.argv: + run() + else: + remote_run()