mirror of
https://github.com/RYDE-WORK/ballistica.git
synced 2026-02-03 14:03:18 +08:00
98 lines
3.7 KiB
Python
98 lines
3.7 KiB
Python
# Copyright (c) 2011-2019 Eric Froemling
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
# -----------------------------------------------------------------------------
|
|
"""A simple cloud caching system for making built binaries/assets available."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from typing import List, Dict
|
|
|
|
|
|
def update_cache(makefile_dirs: List[str]) -> None:
|
|
"""Given a list of directories containing makefiles, update caches."""
|
|
|
|
import multiprocessing
|
|
from efrotools import run
|
|
cpus = multiprocessing.cpu_count()
|
|
fnames: List[str] = []
|
|
for path in makefile_dirs:
|
|
# First, update things..
|
|
cdp = f'cd {path} && ' if path else ''
|
|
subprocess.run(f'{cdp}make -j{cpus} efrocache_build',
|
|
shell=True,
|
|
check=True)
|
|
# Now get the list of them.
|
|
fnames += [
|
|
os.path.join(path, s) for s in subprocess.run(
|
|
f'{cdp}make efrocache_list',
|
|
shell=True,
|
|
check=True,
|
|
capture_output=True).stdout.decode().split()
|
|
]
|
|
staging_dir = 'build/efrocache'
|
|
mapping_file = 'build/efrocachemap'
|
|
run(f'rm -rf {staging_dir}')
|
|
run(f'mkdir -p {staging_dir}')
|
|
|
|
_write_cache_files(fnames, staging_dir, mapping_file)
|
|
|
|
# Push what we just wrote to the staging server
|
|
print('Pushing cache to staging...', flush=True)
|
|
run('rsync --recursive build/efrocache/'
|
|
' ubuntu@ballistica.net:files.ballistica.net/cache/ba1/')
|
|
|
|
print(f'Cache update successful!')
|
|
|
|
|
|
def _write_cache_files(fnames: List[str], staging_dir: str,
|
|
mapping_file: str) -> None:
|
|
from efrotools import run
|
|
import hashlib
|
|
import json
|
|
mapping: Dict[str, str] = {}
|
|
baseurl = 'https://files.ballistica.net/cache/ba1/'
|
|
for fname in fnames:
|
|
|
|
if ' ' in fname:
|
|
raise RuntimeError('Spaces in paths not supported.')
|
|
|
|
# Just going with ol' md5 here; we're the only ones creating these so
|
|
# security isn't a concern.
|
|
md5 = hashlib.md5()
|
|
with open(fname, 'rb') as infile:
|
|
md5.update(infile.read())
|
|
md5.update(fname.encode())
|
|
finalhash = md5.hexdigest()
|
|
hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
|
|
path = os.path.join(staging_dir, hashpath)
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
|
|
# Fancy pipe stuff which will give us deterministic
|
|
# tar.gz files (no embedded timestamps)
|
|
run(f'tar cf - {fname} | gzip -n > {path}')
|
|
mapping[fname] = baseurl + hashpath
|
|
with open(mapping_file, 'w') as outfile:
|
|
outfile.write(json.dumps(mapping, indent=2, sort_keys=True))
|