RYDE-WORK/ballistica
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/ballistica.git synced 2026-01-26 00:47:10 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

moving bacloud guts to batools package

Browse Source
This commit is contained in:
Eric 2023-06-30 14:02:16 -07:00
parent f011110967
commit f25dd7bcc5
No known key found for this signature in database
GPG Key ID: 89C93F0F8D6D5A98
3 changed files with 409 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
.efrocachemap generated
View File

@ -4076,18 +4076,18 @@
"build/prefab/full/linux_x86_64_gui/release/ballisticakit": "https://files.ballistica.net/cache/ba1/5f/b0/3fce5a36c251f4090efa8d5c17f6",
"build/prefab/full/linux_x86_64_server/debug/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/41/76/508f80aa2cad4248be0f644ac577",
"build/prefab/full/linux_x86_64_server/release/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/e2/8c/71a04c5829575d085aef4e36a77f",
"build/prefab/full/mac_arm64_gui/debug/ballisticakit": "https://files.ballistica.net/cache/ba1/50/ea/3425b68d9cbd018b591620ea21c8",
"build/prefab/full/mac_arm64_gui/debug/ballisticakit": "https://files.ballistica.net/cache/ba1/a4/90/eb9eebb974e9d40476e9d42709ef",
"build/prefab/full/mac_arm64_gui/release/ballisticakit": "https://files.ballistica.net/cache/ba1/ab/c8/e638cf97e0f18c247c87c453d017",
"build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/3a/16/39bbdf1fd716309eb9bc00a3914c",
"build/prefab/full/mac_arm64_server/debug/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/9c/05/179138bdfd06d1fb54828acdaed6",
"build/prefab/full/mac_arm64_server/release/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/60/ff/8ad1713b4fbd20d43b3a791095b3",
"build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "https://files.ballistica.net/cache/ba1/81/38/d5beff6112f26a82271e6f8430ba",
"build/prefab/full/mac_x86_64_gui/debug/ballisticakit": "https://files.ballistica.net/cache/ba1/5c/d8/29ef01a424be79e7b5d7de8fdcf2",
"build/prefab/full/mac_x86_64_gui/release/ballisticakit": "https://files.ballistica.net/cache/ba1/c2/ed/95e885a5bc785a1e595c74ff76dc",
"build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/e2/1a/a14431e08d923eb01f6a06132b30",
"build/prefab/full/mac_x86_64_server/debug/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/f1/f9/97a7f6b6cd9b460f25dbb5cb5a96",
"build/prefab/full/mac_x86_64_server/release/dist/ballisticakit_headless": "https://files.ballistica.net/cache/ba1/6d/ab/bb8b72e8764292d17803fbecfcb0",
"build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "https://files.ballistica.net/cache/ba1/3f/7a/37310338d005ba12cab72fdd990d",
"build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "https://files.ballistica.net/cache/ba1/de/09/c0e6a848c89c20279a8d562821f7",
"build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "https://files.ballistica.net/cache/ba1/59/3a/97b3e97889a8111b1d4fa54e28e9",
"build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "https://files.ballistica.net/cache/ba1/9e/e5/ec1673fdc92998ee8952225b46ec",
"build/prefab/full/windows_x86_gui/debug/BallisticaKit.exe": "https://files.ballistica.net/cache/ba1/ec/f1/74692bee69a1c8dd0e58058d838c",
"build/prefab/full/windows_x86_gui/release/BallisticaKit.exe": "https://files.ballistica.net/cache/ba1/73/92/ba936c09947ada91109477075f59",
"build/prefab/full/windows_x86_server/debug/dist/BallisticaKitHeadless.exe": "https://files.ballistica.net/cache/ba1/a8/18/110496e9d31cc4810ef19e377a61",
"build/prefab/full/windows_x86_server/release/dist/BallisticaKitHeadless.exe": "https://files.ballistica.net/cache/ba1/e0/bf/c93d5851379821f4d043819b11ef",
"build/prefab/lib/linux_arm64_gui/debug/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/be/19/b5458933dfc7371d91ecfcd2e06f",
"build/prefab/lib/linux_arm64_gui/release/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/4e/48/123b806cbe6ddb3d9a8368bbb4f8",
"build/prefab/lib/linux_arm64_server/debug/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/be/19/b5458933dfc7371d91ecfcd2e06f",
@ -4104,14 +4104,14 @@
"build/prefab/lib/mac_x86_64_gui/release/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/7e/fa/291fd7e935502ced7e99b8c8f7f0",
"build/prefab/lib/mac_x86_64_server/debug/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/e1/cb/7e8440699e59e8646da25aa5782b",
"build/prefab/lib/mac_x86_64_server/release/libballistica_plus.a": "https://files.ballistica.net/cache/ba1/7e/fa/291fd7e935502ced7e99b8c8f7f0",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "https://files.ballistica.net/cache/ba1/5d/1c/956723a220dbd69a6fccbdec29df",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "https://files.ballistica.net/cache/ba1/e5/5c/c7737071206b53292cdc954bfde5",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "https://files.ballistica.net/cache/ba1/50/78/552f6c71715a4939b0ca67468efc",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "https://files.ballistica.net/cache/ba1/71/5f/bcbdc8e71a29515ed95ec3c33fae",
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "https://files.ballistica.net/cache/ba1/de/64/60396bbaba483ff09a421998372a",
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "https://files.ballistica.net/cache/ba1/8b/92/725858ebddf7f1d35562ce516e59",
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "https://files.ballistica.net/cache/ba1/15/08/f62cc35c4746f7767059e0e358b5",
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "https://files.ballistica.net/cache/ba1/e4/39/8964fd8d1781fe1d5b09bce43d98",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.lib": "https://files.ballistica.net/cache/ba1/5c/96/b8a42d96d3af5ffff38c57efbd24",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitGenericPlus.pdb": "https://files.ballistica.net/cache/ba1/4d/f3/1a23c221637c7b7d477b44e4fd81",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.lib": "https://files.ballistica.net/cache/ba1/e6/dd/ec3db84a6f9c3a94f924dbbaf88a",
"build/prefab/lib/windows/Debug_Win32/BallisticaKitHeadlessPlus.pdb": "https://files.ballistica.net/cache/ba1/c3/dd/6767011fcea297758ee6f05f1a2a",
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.lib": "https://files.ballistica.net/cache/ba1/fd/ce/93577e4d73568f2a866f2da07ffd",
"build/prefab/lib/windows/Release_Win32/BallisticaKitGenericPlus.pdb": "https://files.ballistica.net/cache/ba1/f0/03/af61323f340c42b2be9ebd7f8364",
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.lib": "https://files.ballistica.net/cache/ba1/97/55/985b572fd6cf68c782a93fcd891d",
"build/prefab/lib/windows/Release_Win32/BallisticaKitHeadlessPlus.pdb": "https://files.ballistica.net/cache/ba1/7f/ac/e2870829d60fe499298bfec614bc",
"src/assets/ba_data/python/babase/_mgen/__init__.py": "https://files.ballistica.net/cache/ba1/f8/85/fed7f2ed98ff2ba271f9dbe3391c",
"src/assets/ba_data/python/babase/_mgen/enums.py": "https://files.ballistica.net/cache/ba1/48/4b/e6974f0a4d14be8213dc00d971c3",
"src/ballistica/base/mgen/pyembed/binding_base.inc": "https://files.ballistica.net/cache/ba1/3e/7a/203e2a5d2b5bb42cfe3fd2fe16c2",

380
tools/bacloud
View File

@ -7,377 +7,17 @@ This facilitates workflows such as creating asset-packages, etc.
from __future__ import annotations
import os
import sys
import zlib
import time
import datetime
import subprocess
from pathlib import Path
from typing import TYPE_CHECKING
from dataclasses import dataclass
import requests
from efro.terminal import Clr
from batools.bacloud import App
from efro.error import CleanError
from efro.dataclassio import (
dataclass_from_json,
dataclass_to_dict,
dataclass_to_json,
ioprepped,
)
from bacommon.bacloud import RequestData, ResponseData, BACLOUD_VERSION
if TYPE_CHECKING:
from typing import IO
TOOL_NAME = 'bacloud'
TIMEOUT_SECONDS = 60 * 5
# Server we talk to (can override via env var).
BACLOUD_SERVER_URL = os.getenv('BACLOUD_SERVER_URL', 'https://ballistica.net')
@ioprepped
@dataclass
class StateData:
"""Persistent state data stored to disk."""
login_token: str | None = None
def get_tz_offset_seconds() -> float:
"""Return the offset between utc and local time in seconds."""
tval = time.time()
utc_offset = (
datetime.datetime.fromtimestamp(tval)
- datetime.datetime.utcfromtimestamp(tval)
).total_seconds()
return utc_offset
class App:
"""Context for a run of the tool."""
def __init__(self) -> None:
self._state = StateData()
self._project_root: Path | None = None
self._end_command_args: dict = {}
def run(self) -> None:
"""Run the tool."""
# Make sure we can locate the project bacloud is being run from.
self._project_root = Path(sys.argv[0]).parents[1]
if not all(
Path(self._project_root, name).is_dir()
for name in ('tools', 'config', 'tests')
):
raise CleanError('Unable to locate project directory.')
# Also run project prereqs checks so we can hopefully inform the user
# of missing Python modules/etc. instead of just failing cryptically.
try:
subprocess.run(
['make', '--quiet', 'prereqs'],
check=True,
cwd=self._project_root,
)
except subprocess.CalledProcessError as exc:
raise CleanError(
'"make prereqs" check failed. '
'Install missing requirements and try again.'
) from exc
self._load_state()
# Simply pass all args to the server and let it do the thing.
self.run_interactive_command(sys.argv[1:])
self._save_state()
@property
def _state_dir(self) -> Path:
"""The full path to the state dir."""
assert self._project_root is not None
return Path(self._project_root, '.cache/bacloud')
@property
def _state_data_path(self) -> Path:
"""The full path to the state data file."""
return Path(self._state_dir, 'state')
def _load_state(self) -> None:
if not os.path.exists(self._state_data_path):
return
try:
with open(self._state_data_path, 'r', encoding='utf-8') as infile:
self._state = dataclass_from_json(StateData, infile.read())
except Exception:
print(
f'{Clr.RED}Error loading {TOOL_NAME} data;'
f' resetting to defaults.{Clr.RST}'
)
def _save_state(self) -> None:
if not self._state_dir.exists():
self._state_dir.mkdir(parents=True, exist_ok=True)
with open(self._state_data_path, 'w', encoding='utf-8') as outfile:
outfile.write(dataclass_to_json(self._state))
def _servercmd(
self, cmd: str, payload: dict, files: dict[str, IO] | None = None
) -> ResponseData:
"""Issue a command to the server and get a response."""
response_content: str | None = None
url = f'{BACLOUD_SERVER_URL}/bacloudcmd'
headers = {'User-Agent': f'bacloud/{BACLOUD_VERSION}'}
rdata = {
'v': BACLOUD_VERSION,
'r': dataclass_to_json(
RequestData(
command=cmd,
token=self._state.login_token,
payload=payload,
tzoffset=get_tz_offset_seconds(),
isatty=sys.stdout.isatty(),
)
),
}
# Trying urllib for comparison (note that this doesn't support
# files arg so not actually production ready)
if bool(False):
import urllib.request
import urllib.parse
with urllib.request.urlopen(
urllib.request.Request(
url, urllib.parse.urlencode(rdata).encode(), headers
)
) as raw_response:
if raw_response.getcode() != 200:
raise RuntimeError('Error talking to server')
response_content = raw_response.read().decode()
# Using requests module.
else:
with requests.post(
url,
headers=headers,
data=rdata,
files=files,
timeout=TIMEOUT_SECONDS,
) as response_raw:
response_raw.raise_for_status()
assert isinstance(response_raw.content, bytes)
response_content = response_raw.content.decode()
assert response_content is not None
response = dataclass_from_json(ResponseData, response_content)
# Handle a few things inline.
# (so this functionality is available even to recursive commands, etc.)
if response.message is not None:
print(response.message, end=response.message_end, flush=True)
if response.error is not None:
raise CleanError(response.error)
if response.delay_seconds > 0.0:
time.sleep(response.delay_seconds)
return response
def _upload_file(self, filename: str, call: str, args: dict) -> None:
import tempfile
print(f'Uploading {Clr.BLU}{filename}{Clr.RST}', flush=True)
with tempfile.TemporaryDirectory() as tempdir:
srcpath = Path(filename)
gzpath = Path(tempdir, 'file.gz')
subprocess.run(
f'gzip --stdout "{srcpath}" > "{gzpath}"',
shell=True,
check=True,
)
with open(gzpath, 'rb') as infile:
putfiles: dict = {'file': infile}
_response = self._servercmd(
call,
args,
files=putfiles,
)
def _handle_dir_manifest_response(self, dirmanifest: str) -> None:
from bacommon.transfer import DirectoryManifest
self._end_command_args['manifest'] = dataclass_to_dict(
DirectoryManifest.create_from_disk(Path(dirmanifest))
)
def _handle_uploads(self, uploads: tuple[list[str], str, dict]) -> None:
from concurrent.futures import ThreadPoolExecutor
assert len(uploads) == 3
filenames, uploadcmd, uploadargs = uploads
assert isinstance(filenames, list)
assert isinstance(uploadcmd, str)
assert isinstance(uploadargs, dict)
def _do_filename(filename: str) -> None:
self._upload_file(filename, uploadcmd, uploadargs)
# Here we can run uploads concurrently if that goes faster...
# (should keep an eye on this to make sure its thread safe
# and behaves itself)
with ThreadPoolExecutor(max_workers=4) as executor:
# Convert the generator to a list to trigger any
# exceptions that occurred.
list(executor.map(_do_filename, filenames))
def _handle_deletes(self, deletes: list[str]) -> None:
"""Handle file deletes."""
for fname in deletes:
# Server shouldn't be sending us dir paths here.
assert not os.path.isdir(fname)
os.unlink(fname)
def _handle_downloads_inline(
self,
downloads_inline: dict[str, str],
) -> None:
"""Handle inline file data to be saved to the client."""
import base64
for fname, fdata in downloads_inline.items():
# If there's a directory where we want our file to go, clear it
# out first. File deletes should have run before this so
# everything under it should be empty and thus killable via rmdir.
if os.path.isdir(fname):
for basename, dirnames, _fn in os.walk(fname, topdown=False):
for dirname in dirnames:
os.rmdir(os.path.join(basename, dirname))
os.rmdir(fname)
dirname = os.path.dirname(fname)
if dirname:
os.makedirs(dirname, exist_ok=True)
data_zipped = base64.b64decode(fdata)
data = zlib.decompress(data_zipped)
with open(fname, 'wb') as outfile:
outfile.write(data)
def _handle_dir_prune_empty(self, prunedir: str) -> None:
"""Handle pruning empty directories."""
# Walk the tree bottom-up so we can properly kill recursive empty dirs.
for basename, dirnames, filenames in os.walk(prunedir, topdown=False):
# It seems that child dirs we kill during the walk are still
# listed when the parent dir is visited, so lets make sure
# to only acknowledge still-existing ones.
dirnames = [
d for d in dirnames if os.path.exists(os.path.join(basename, d))
]
if not dirnames and not filenames and basename != prunedir:
os.rmdir(basename)
def _handle_uploads_inline(self, uploads_inline: list[str]) -> None:
"""Handle uploading files inline."""
import base64
files: dict[str, str] = {}
for filepath in uploads_inline:
if not os.path.exists(filepath):
raise CleanError(f'File not found: {filepath}')
with open(filepath, 'rb') as infile:
data = infile.read()
data_zipped = zlib.compress(data)
data_base64 = base64.b64encode(data_zipped).decode()
files[filepath] = data_base64
self._end_command_args['uploads_inline'] = files
def _handle_open_url(self, url: str) -> None:
import webbrowser
print(f'{Clr.CYN}(url: {url}){Clr.RST}')
webbrowser.open(url)
def _handle_input_prompt(self, prompt: str, as_password: bool) -> None:
if as_password:
from getpass import getpass
self._end_command_args['input'] = getpass(prompt=prompt)
else:
if prompt:
print(prompt, end='', flush=True)
self._end_command_args['input'] = input()
def run_interactive_command(self, args: list[str]) -> None:
"""Run a single user command to completion."""
# pylint: disable=too-many-branches
nextcall: tuple[str, dict] | None = ('_interactive', {'a': args})
# Now talk to the server in a loop until there's nothing left to do.
while nextcall is not None:
self._end_command_args = {}
response = self._servercmd(*nextcall)
nextcall = None
if response.login is not None:
self._state.login_token = response.login
if response.logout:
self._state.login_token = None
if response.dir_manifest is not None:
self._handle_dir_manifest_response(response.dir_manifest)
if response.uploads_inline is not None:
self._handle_uploads_inline(response.uploads_inline)
if response.uploads is not None:
self._handle_uploads(response.uploads)
# Note: we handle file deletes *before* downloads. This
# way our file-download code only has to worry about creating or
# removing directories and not files, and corner cases such as
# a file getting replaced with a directory should just work.
if response.deletes:
self._handle_deletes(response.deletes)
if response.downloads_inline:
self._handle_downloads_inline(response.downloads_inline)
if response.dir_prune_empty:
self._handle_dir_prune_empty(response.dir_prune_empty)
if response.open_url is not None:
self._handle_open_url(response.open_url)
if response.input_prompt is not None:
self._handle_input_prompt(
prompt=response.input_prompt[0],
as_password=response.input_prompt[1],
)
if response.end_message is not None:
print(
response.end_message,
end=response.end_message_end,
flush=True,
)
if response.end_command is not None:
nextcall = response.end_command
for key, val in self._end_command_args.items():
# noinspection PyUnresolvedReferences
nextcall[1][key] = val
if __name__ == '__main__':
try:
App().run()
except KeyboardInterrupt:
# Let's do a clean fail on keyboard interrupt.
# Can make this optional if a backtrace is ever useful.
sys.exit(1)
except CleanError as clean_exc:
clean_exc.pretty_print()
sys.exit(1)
try:
App().run()
except KeyboardInterrupt:
# Let's do a clean fail on keyboard interrupt.
# Can make this optional if a backtrace is ever useful.
sys.exit(1)
except CleanError as clean_exc:
clean_exc.pretty_print()
sys.exit(1)

383
tools/batools/bacloud.py Executable file
View File

@ -0,0 +1,383 @@
#!/usr/bin/env python3.11
# Released under the MIT License. See LICENSE for details.
#
"""A tool for interacting with ballistica's cloud services.
This facilitates workflows such as creating asset-packages, etc.
"""
from __future__ import annotations
import os
import sys
import zlib
import time
import datetime
import subprocess
from pathlib import Path
from typing import TYPE_CHECKING
from dataclasses import dataclass
import requests
from efro.terminal import Clr
from efro.error import CleanError
from efro.dataclassio import (
dataclass_from_json,
dataclass_to_dict,
dataclass_to_json,
ioprepped,
)
from bacommon.bacloud import RequestData, ResponseData, BACLOUD_VERSION
if TYPE_CHECKING:
from typing import IO
TOOL_NAME = 'bacloud'
TIMEOUT_SECONDS = 60 * 5
# Server we talk to (can override via env var).
BACLOUD_SERVER_URL = os.getenv('BACLOUD_SERVER_URL', 'https://ballistica.net')
@ioprepped
@dataclass
class StateData:
"""Persistent state data stored to disk."""
login_token: str | None = None
def get_tz_offset_seconds() -> float:
"""Return the offset between utc and local time in seconds."""
tval = time.time()
utc_offset = (
datetime.datetime.fromtimestamp(tval)
- datetime.datetime.utcfromtimestamp(tval)
).total_seconds()
return utc_offset
class App:
"""Context for a run of the tool."""
def __init__(self) -> None:
self._state = StateData()
self._project_root: Path | None = None
self._end_command_args: dict = {}
def run(self) -> None:
"""Run the tool."""
# Make sure we can locate the project bacloud is being run from.
self._project_root = Path(sys.argv[0]).parents[1]
if not all(
Path(self._project_root, name).is_dir()
for name in ('tools', 'config', 'tests')
):
raise CleanError('Unable to locate project directory.')
# Also run project prereqs checks so we can hopefully inform the user
# of missing Python modules/etc. instead of just failing cryptically.
try:
subprocess.run(
['make', '--quiet', 'prereqs'],
check=True,
cwd=self._project_root,
)
except subprocess.CalledProcessError as exc:
raise CleanError(
'"make prereqs" check failed. '
'Install missing requirements and try again.'
) from exc
self._load_state()
# Simply pass all args to the server and let it do the thing.
self.run_interactive_command(sys.argv[1:])
self._save_state()
@property
def _state_dir(self) -> Path:
"""The full path to the state dir."""
assert self._project_root is not None
return Path(self._project_root, '.cache/bacloud')
@property
def _state_data_path(self) -> Path:
"""The full path to the state data file."""
return Path(self._state_dir, 'state')
def _load_state(self) -> None:
if not os.path.exists(self._state_data_path):
return
try:
with open(self._state_data_path, 'r', encoding='utf-8') as infile:
self._state = dataclass_from_json(StateData, infile.read())
except Exception:
print(
f'{Clr.RED}Error loading {TOOL_NAME} data;'
f' resetting to defaults.{Clr.RST}'
)
def _save_state(self) -> None:
if not self._state_dir.exists():
self._state_dir.mkdir(parents=True, exist_ok=True)
with open(self._state_data_path, 'w', encoding='utf-8') as outfile:
outfile.write(dataclass_to_json(self._state))
def _servercmd(
self, cmd: str, payload: dict, files: dict[str, IO] | None = None
) -> ResponseData:
"""Issue a command to the server and get a response."""
response_content: str | None = None
url = f'{BACLOUD_SERVER_URL}/bacloudcmd'
headers = {'User-Agent': f'bacloud/{BACLOUD_VERSION}'}
rdata = {
'v': BACLOUD_VERSION,
'r': dataclass_to_json(
RequestData(
command=cmd,
token=self._state.login_token,
payload=payload,
tzoffset=get_tz_offset_seconds(),
isatty=sys.stdout.isatty(),
)
),
}
# Trying urllib for comparison (note that this doesn't support
# files arg so not actually production ready)
if bool(False):
import urllib.request
import urllib.parse
with urllib.request.urlopen(
urllib.request.Request(
url, urllib.parse.urlencode(rdata).encode(), headers
)
) as raw_response:
if raw_response.getcode() != 200:
raise RuntimeError('Error talking to server')
response_content = raw_response.read().decode()
# Using requests module.
else:
with requests.post(
url,
headers=headers,
data=rdata,
files=files,
timeout=TIMEOUT_SECONDS,
) as response_raw:
response_raw.raise_for_status()
assert isinstance(response_raw.content, bytes)
response_content = response_raw.content.decode()
assert response_content is not None
response = dataclass_from_json(ResponseData, response_content)
# Handle a few things inline.
# (so this functionality is available even to recursive commands, etc.)
if response.message is not None:
print(response.message, end=response.message_end, flush=True)
if response.error is not None:
raise CleanError(response.error)
if response.delay_seconds > 0.0:
time.sleep(response.delay_seconds)
return response
def _upload_file(self, filename: str, call: str, args: dict) -> None:
import tempfile
print(f'Uploading {Clr.BLU}{filename}{Clr.RST}', flush=True)
with tempfile.TemporaryDirectory() as tempdir:
srcpath = Path(filename)
gzpath = Path(tempdir, 'file.gz')
subprocess.run(
f'gzip --stdout "{srcpath}" > "{gzpath}"',
shell=True,
check=True,
)
with open(gzpath, 'rb') as infile:
putfiles: dict = {'file': infile}
_response = self._servercmd(
call,
args,
files=putfiles,
)
def _handle_dir_manifest_response(self, dirmanifest: str) -> None:
from bacommon.transfer import DirectoryManifest
self._end_command_args['manifest'] = dataclass_to_dict(
DirectoryManifest.create_from_disk(Path(dirmanifest))
)
def _handle_uploads(self, uploads: tuple[list[str], str, dict]) -> None:
from concurrent.futures import ThreadPoolExecutor
assert len(uploads) == 3
filenames, uploadcmd, uploadargs = uploads
assert isinstance(filenames, list)
assert isinstance(uploadcmd, str)
assert isinstance(uploadargs, dict)
def _do_filename(filename: str) -> None:
self._upload_file(filename, uploadcmd, uploadargs)
# Here we can run uploads concurrently if that goes faster...
# (should keep an eye on this to make sure its thread safe
# and behaves itself)
with ThreadPoolExecutor(max_workers=4) as executor:
# Convert the generator to a list to trigger any
# exceptions that occurred.
list(executor.map(_do_filename, filenames))
def _handle_deletes(self, deletes: list[str]) -> None:
"""Handle file deletes."""
for fname in deletes:
# Server shouldn't be sending us dir paths here.
assert not os.path.isdir(fname)
os.unlink(fname)
def _handle_downloads_inline(
self,
downloads_inline: dict[str, str],
) -> None:
"""Handle inline file data to be saved to the client."""
import base64
for fname, fdata in downloads_inline.items():
# If there's a directory where we want our file to go, clear it
# out first. File deletes should have run before this so
# everything under it should be empty and thus killable via rmdir.
if os.path.isdir(fname):
for basename, dirnames, _fn in os.walk(fname, topdown=False):
for dirname in dirnames:
os.rmdir(os.path.join(basename, dirname))
os.rmdir(fname)
dirname = os.path.dirname(fname)
if dirname:
os.makedirs(dirname, exist_ok=True)
data_zipped = base64.b64decode(fdata)
data = zlib.decompress(data_zipped)
with open(fname, 'wb') as outfile:
outfile.write(data)
def _handle_dir_prune_empty(self, prunedir: str) -> None:
"""Handle pruning empty directories."""
# Walk the tree bottom-up so we can properly kill recursive empty dirs.
for basename, dirnames, filenames in os.walk(prunedir, topdown=False):
# It seems that child dirs we kill during the walk are still
# listed when the parent dir is visited, so lets make sure
# to only acknowledge still-existing ones.
dirnames = [
d for d in dirnames if os.path.exists(os.path.join(basename, d))
]
if not dirnames and not filenames and basename != prunedir:
os.rmdir(basename)
def _handle_uploads_inline(self, uploads_inline: list[str]) -> None:
"""Handle uploading files inline."""
import base64
files: dict[str, str] = {}
for filepath in uploads_inline:
if not os.path.exists(filepath):
raise CleanError(f'File not found: {filepath}')
with open(filepath, 'rb') as infile:
data = infile.read()
data_zipped = zlib.compress(data)
data_base64 = base64.b64encode(data_zipped).decode()
files[filepath] = data_base64
self._end_command_args['uploads_inline'] = files
def _handle_open_url(self, url: str) -> None:
import webbrowser
print(f'{Clr.CYN}(url: {url}){Clr.RST}')
webbrowser.open(url)
def _handle_input_prompt(self, prompt: str, as_password: bool) -> None:
if as_password:
from getpass import getpass
self._end_command_args['input'] = getpass(prompt=prompt)
else:
if prompt:
print(prompt, end='', flush=True)
self._end_command_args['input'] = input()
def run_interactive_command(self, args: list[str]) -> None:
"""Run a single user command to completion."""
# pylint: disable=too-many-branches
nextcall: tuple[str, dict] | None = ('_interactive', {'a': args})
# Now talk to the server in a loop until there's nothing left to do.
while nextcall is not None:
self._end_command_args = {}
response = self._servercmd(*nextcall)
nextcall = None
if response.login is not None:
self._state.login_token = response.login
if response.logout:
self._state.login_token = None
if response.dir_manifest is not None:
self._handle_dir_manifest_response(response.dir_manifest)
if response.uploads_inline is not None:
self._handle_uploads_inline(response.uploads_inline)
if response.uploads is not None:
self._handle_uploads(response.uploads)
# Note: we handle file deletes *before* downloads. This
# way our file-download code only has to worry about creating or
# removing directories and not files, and corner cases such as
# a file getting replaced with a directory should just work.
if response.deletes:
self._handle_deletes(response.deletes)
if response.downloads_inline:
self._handle_downloads_inline(response.downloads_inline)
if response.dir_prune_empty:
self._handle_dir_prune_empty(response.dir_prune_empty)
if response.open_url is not None:
self._handle_open_url(response.open_url)
if response.input_prompt is not None:
self._handle_input_prompt(
prompt=response.input_prompt[0],
as_password=response.input_prompt[1],
)
if response.end_message is not None:
print(
response.end_message,
end=response.end_message_end,
flush=True,
)
if response.end_command is not None:
nextcall = response.end_command
for key, val in self._end_command_args.items():
# noinspection PyUnresolvedReferences
nextcall[1][key] = val
if __name__ == '__main__':
try:
App().run()
except KeyboardInterrupt:
# Let's do a clean fail on keyboard interrupt.
# Can make this optional if a backtrace is ever useful.
sys.exit(1)
except CleanError as clean_exc:
clean_exc.pretty_print()
sys.exit(1)