mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-01-22 14:59:25 +08:00
Cache tool work
This commit is contained in:
Eric Froemling
parent
c95357e3d0
commit
ce258cd00b
@ -4132,16 +4132,16 @@
|
||||
"assets/build/windows/x64/python.exe": "https://files.ballistica.net/cache/ba1/25/a7/dc87c1be41605eb6fefd0145144c",
|
||||
"assets/build/windows/x64/python37.dll": "https://files.ballistica.net/cache/ba1/b9/e4/d912f56e42e9991bcbb4c804cfcb",
|
||||
"assets/build/windows/x64/pythonw.exe": "https://files.ballistica.net/cache/ba1/6c/bb/b6f52c306aa4e88061510e96cefe",
|
||||
"build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/6a/56/e48e9306428b4ae7b58225fce76c",
|
||||
"build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/dd/2d/bdd22cbd39a9bb27a75302ef837a",
|
||||
"build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/61/f2/a679c5a3c9113c7b7599a59415fb",
|
||||
"build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/6c/21/c80e03031e6c5959140588d439bf",
|
||||
"build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/42/ff/6d9056500a768dae79ed44180796",
|
||||
"build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/58/d4/3f6d3cb75988db046f22bfb5c2a5",
|
||||
"build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/a8/07/edf6360c8e5b824633bf55c97ed9",
|
||||
"build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/27/cc/0ec7cab492f111923320dc165427",
|
||||
"build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/b2/b4/f68901c3dae4267a5640bf702d86",
|
||||
"build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/ae/b7/0400bd1021d0fc0c40d1c2149364",
|
||||
"build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/ee/6e/8f88fef729d85a07910cbedbac31",
|
||||
"build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/e1/90/4877825fb2d779bd7ac9704a1a4c"
|
||||
"build/prefab/linux-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/8b/e2/8ab3f02c813d2e43df787d478023",
|
||||
"build/prefab/linux-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/ca/46/ae21c3d6765f6625ac06b9215f3d",
|
||||
"build/prefab/linux/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/21/f2/079c550e8ba0e255c6255dc0737a",
|
||||
"build/prefab/linux/release/ballisticacore": "https://files.ballistica.net/cache/ba1/19/f3/1121125a99e5efb3577a491799e9",
|
||||
"build/prefab/mac-server/debug/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/87/b1/4d076556aac39f44ccb03278f584",
|
||||
"build/prefab/mac-server/release/dist/ballisticacore_headless": "https://files.ballistica.net/cache/ba1/84/f6/607fd2ab859f610fdd91a9a857db",
|
||||
"build/prefab/mac/debug/ballisticacore": "https://files.ballistica.net/cache/ba1/40/f3/00df4a55512b824534bda6945204",
|
||||
"build/prefab/mac/release/ballisticacore": "https://files.ballistica.net/cache/ba1/f8/2f/09295de6690e6df09c0003b4d85b",
|
||||
"build/prefab/windows-server/debug/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/61/62/2e8398b3dcf1ae702ba591aa947e",
|
||||
"build/prefab/windows-server/release/dist/ballisticacore_headless.exe": "https://files.ballistica.net/cache/ba1/e1/db/1acefefb21e4fd0fe8e337647592",
|
||||
"build/prefab/windows/debug/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/ef/85/b8235ca82f2e3319850535558f0f",
|
||||
"build/prefab/windows/release/BallisticaCore.exe": "https://files.ballistica.net/cache/ba1/26/ba/1f40445d853e6be53beacd7a3ca3"
|
||||
}
|
||||
2
.idea/dictionaries/ericf.xml
generated
2
.idea/dictionaries/ericf.xml
generated
@ -17,6 +17,7 @@
|
||||
<w>abeb</w>
|
||||
<w>abot</w>
|
||||
<w>abtn</w>
|
||||
<w>accesstime</w>
|
||||
<w>accountname</w>
|
||||
<w>accountui</w>
|
||||
<w>accum</w>
|
||||
@ -687,6 +688,7 @@
|
||||
<w>fsplit</w>
|
||||
<w>fsrc</w>
|
||||
<w>fstab</w>
|
||||
<w>fstat</w>
|
||||
<w>fstrs</w>
|
||||
<w>ftime</w>
|
||||
<w>ftmp</w>
|
||||
|
||||
@ -284,8 +284,9 @@ def _gen_hashes(fnames: List[str]) -> str:
|
||||
|
||||
def _write_cache_files(fnames1: List[str], fnames2: List[str],
|
||||
staging_dir: str, mapping_file: str) -> None:
|
||||
fhashes1: Set[str] = set()
|
||||
import functools
|
||||
fhashes1: Set[str] = set()
|
||||
fhashes2: Set[str] = set()
|
||||
mapping: Dict[str, str] = {}
|
||||
call = functools.partial(_write_cache_file, staging_dir)
|
||||
|
||||
@ -296,21 +297,38 @@ def _write_cache_files(fnames1: List[str], fnames2: List[str],
|
||||
mapping[result[0]] = BASE_URL + result[1]
|
||||
fhashes1.add(result[1])
|
||||
|
||||
# Now finish up with the second set.
|
||||
with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
|
||||
results = executor.map(call, fnames2)
|
||||
for result in results:
|
||||
mapping[result[0]] = BASE_URL + result[1]
|
||||
fhashes2.add(result[1])
|
||||
|
||||
# We want the server to have a startercache.tar.xz file which contains
|
||||
# this entire first set. It is much more efficient to build that file
|
||||
# the entire first set. It is much more efficient to build that file
|
||||
# on the server than it is to build it here and upload the whole thing.
|
||||
# ...so let's simply write a script to generate it and upload that.
|
||||
|
||||
# Also let's have the script touch both sets of files so we can use
|
||||
# mod-times to prune older files. (otherwise files that never change
|
||||
# might have very old mod times)
|
||||
script = (
|
||||
'import os\n'
|
||||
'import pathlib\n'
|
||||
'import subprocess\n'
|
||||
'fnames = ' + repr(fhashes1) + '\n'
|
||||
'fnames2 = ' + repr(fhashes2) + '\n'
|
||||
'subprocess.run(["rm", "-rf", "efrocache"], check=True)\n'
|
||||
'print("Copying starter cache files...", flush=True)\n'
|
||||
'for fname in fnames:\n'
|
||||
' dst = os.path.join("efrocache", fname)\n'
|
||||
' os.makedirs(os.path.dirname(dst), exist_ok=True)\n'
|
||||
' subprocess.run(["cp", fname, dst], check=True)\n'
|
||||
'print("Touching full file set...", flush=True)\n'
|
||||
'for fname in list(fnames) + list(fnames2):\n'
|
||||
' fpath = pathlib.Path(fname)\n'
|
||||
' assert fpath.exists()\n'
|
||||
' fpath.touch()\n'
|
||||
'print("Compressing starter cache archive...", flush=True)\n'
|
||||
'subprocess.run(["tar", "-Jcf", "tmp.tar.xz", "efrocache"],'
|
||||
' check=True)\n'
|
||||
@ -322,12 +340,6 @@ def _write_cache_files(fnames1: List[str], fnames2: List[str],
|
||||
with open('build/efrocache/genstartercache.py', 'w') as outfile:
|
||||
outfile.write(script)
|
||||
|
||||
# Now finish up with the second set.
|
||||
with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
|
||||
results = executor.map(call, fnames2)
|
||||
for result in results:
|
||||
mapping[result[0]] = BASE_URL + result[1]
|
||||
|
||||
with open(mapping_file, 'w') as outfile:
|
||||
outfile.write(json.dumps(mapping, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
82
tools/staging_server_upkeep
Executable file
82
tools/staging_server_upkeep
Executable file
@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3.7
|
||||
# Copyright (c) 2011-2020 Eric Froemling
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
# -----------------------------------------------------------------------------
|
||||
"""Perform various tidying on the staging server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
def run() -> None:
|
||||
"""Do the thing."""
|
||||
rootdir = 'files.ballistica.net/cache/ba1'
|
||||
assert os.path.isdir(rootdir)
|
||||
|
||||
# Just make a temp file to get 'now'.
|
||||
# (ugly but effective)
|
||||
tmppath = Path('_tmp_time_test')
|
||||
tmppath.touch()
|
||||
curtime = tmppath.stat()[stat.ST_ATIME]
|
||||
tmppath.unlink()
|
||||
|
||||
oldest_age = 0
|
||||
|
||||
for root, _dirnames, fnames in os.walk(rootdir):
|
||||
for fname in fnames:
|
||||
# print('doing', os.path.join(root, fname))
|
||||
fstat = os.stat(os.path.join(root, fname))
|
||||
accesstime = fstat[stat.ST_MTIME]
|
||||
age = curtime - accesstime
|
||||
oldest_age = max(oldest_age, age)
|
||||
# print('was', (curtime - accesstime)/60, 'min')
|
||||
# print('TEMP BREW')
|
||||
# break
|
||||
print('oldest is', oldest_age / 60 / 60 // 24, 'day')
|
||||
|
||||
|
||||
def remote_run() -> None:
|
||||
"""Upload and execute ourself."""
|
||||
|
||||
print(f'Uploading {__file__}...')
|
||||
subprocess.run(['rsync', __file__, 'ubuntu@ballistica.net:'], check=True)
|
||||
|
||||
subprocess.run([
|
||||
'ssh', 'ubuntu@ballistica.net',
|
||||
'./staging_server_upkeep -run && rm staging_server_upkeep'
|
||||
],
|
||||
check=True)
|
||||
print('SUCCESS')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if '-run' in sys.argv:
|
||||
run()
|
||||
else:
|
||||
remote_run()
|
||||
Loading…
x
Reference in New Issue
Block a user