From 8fde3890019fab2342a268ab4c7931bedef75488 Mon Sep 17 00:00:00 2001 From: Eric Froemling Date: Fri, 1 Oct 2021 10:44:50 -0500 Subject: [PATCH] Replaced FieldStoragePathCapture with cleaner DataclassFieldLookup in dataclassio --- .idea/dictionaries/ericf.xml | 2 + .../.idea/dictionaries/ericf.xml | 2 + docs/ba_module.md | 2 +- tests/test_efro/test_dataclassio.py | 58 +++++++++---- tools/bacommon/assets.py | 31 ++++--- tools/efro/dataclassio/__init__.py | 20 +++-- tools/efro/dataclassio/_pathcapture.py | 85 +++++++++++++------ 7 files changed, 135 insertions(+), 65 deletions(-) diff --git a/.idea/dictionaries/ericf.xml b/.idea/dictionaries/ericf.xml index 844b8d2d..18caaff6 100644 --- a/.idea/dictionaries/ericf.xml +++ b/.idea/dictionaries/ericf.xml @@ -1139,6 +1139,7 @@ janktastic janky jascha + jdict jenkinsfile jexport jisx @@ -1623,6 +1624,7 @@ pathcapture pathlib pathnames + pathparts pathstonames pathtmp patsubst diff --git a/ballisticacore-cmake/.idea/dictionaries/ericf.xml b/ballisticacore-cmake/.idea/dictionaries/ericf.xml index 01985613..36dd41ab 100644 --- a/ballisticacore-cmake/.idea/dictionaries/ericf.xml +++ b/ballisticacore-cmake/.idea/dictionaries/ericf.xml @@ -540,6 +540,7 @@ janky jaxis jcjwf + jdict jmessage jnames keepalives @@ -771,6 +772,7 @@ parameteriv passcode pathcapture + pathparts pausable pcommands pdataclass diff --git a/docs/ba_module.md b/docs/ba_module.md index 101f2087..95ac7540 100644 --- a/docs/ba_module.md +++ b/docs/ba_module.md @@ -1,5 +1,5 @@ -

last updated on 2021-09-29 for Ballistica version 1.6.5 build 20393

+

last updated on 2021-10-01 for Ballistica version 1.6.5 build 20393

This page documents the Python classes and functions in the 'ba' module, which are the ones most relevant to modding in Ballistica. If you come across something you feel should be included here or could be better explained, please let me know. Happy modding!

diff --git a/tests/test_efro/test_dataclassio.py b/tests/test_efro/test_dataclassio.py index ef4ec9c2..c275b26b 100644 --- a/tests/test_efro/test_dataclassio.py +++ b/tests/test_efro/test_dataclassio.py @@ -16,7 +16,7 @@ import pytest from efro.util import utc_now from efro.dataclassio import (dataclass_validate, dataclass_from_dict, dataclass_to_dict, ioprepped, IOAttrs, Codec, - FieldStoragePathCapture) + DataclassFieldLookup) if TYPE_CHECKING: pass @@ -697,20 +697,6 @@ class _SPTestClass2: IOAttrs('s')] = field(default_factory=_SPTestClass1) -def test_field_storage_path_capture() -> None: - """Test FieldStoragePathCapture functionality.""" - - obj = _SPTestClass2() - - namecap = FieldStoragePathCapture(obj) - assert namecap.subc.barf == 'subc.barf' - assert namecap.subc2.barf == 's.barf' - assert namecap.subc2.barf2 == 's.b' - - with pytest.raises(AttributeError): - assert namecap.nonexistent.barf == 's.barf' - - def test_datetime_limits() -> None: """Test limiting datetime values in various ways.""" from efro.util import utc_today, utc_this_hour @@ -752,6 +738,48 @@ def test_datetime_limits() -> None: dataclass_from_dict(_TestClass2, out) +def test_field_paths() -> None: + """Test type-safe field path evaluations.""" + + # Define a few nested dataclass types, some of which + # have storage names differing from their field names. + @ioprepped + @dataclass + class _TestClass: + + @dataclass + class _TestSubClass: + val1: int = 0 + val2: Annotated[int, IOAttrs('v2')] = 0 + + sub1: _TestSubClass = field(default_factory=_TestSubClass) + sub2: Annotated[_TestSubClass, + IOAttrs('s2')] = field(default_factory=_TestSubClass) + + # Now let's lookup various storage paths. + lookup = DataclassFieldLookup(_TestClass) + + # Make sure lookups are returning correct storage paths. + assert lookup.path(lambda obj: obj.sub1) == 'sub1' + assert lookup.path(lambda obj: obj.sub1.val1) == 'sub1.val1' + assert lookup.path(lambda obj: obj.sub1.val2) == 'sub1.v2' + assert lookup.path(lambda obj: obj.sub2.val1) == 's2.val1' + assert lookup.path(lambda obj: obj.sub2.val2) == 's2.v2' + + # Attempting to return fields that aren't there should fail + # in both type-checking and runtime. + with pytest.raises(AttributeError): + lookup.path(lambda obj: obj.sub1.val3) # type: ignore + + # Returning non-field objects will fail at runtime + # even if type-checking evaluates them as valid values. + with pytest.raises(TypeError): + lookup.path(lambda obj: 1) + + with pytest.raises(TypeError): + lookup.path(lambda obj: obj.sub1.val1.real) + + def test_nested() -> None: """Test nesting dataclasses.""" diff --git a/tools/bacommon/assets.py b/tools/bacommon/assets.py index 60d1409c..2f5f25c5 100644 --- a/tools/bacommon/assets.py +++ b/tools/bacommon/assets.py @@ -4,10 +4,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, Optional, List from enum import Enum -from efro import entity +from typing_extensions import Annotated + +from efro.dataclassio import ioprepped, IOAttrs +# from efro import entity if TYPE_CHECKING: pass @@ -33,26 +37,27 @@ class AssetType(Enum): COLLISION_MESH = 'collision_mesh' -class AssetPackageFlavorManifestValue(entity.CompoundValue): +@ioprepped +@dataclass +class AssetPackageFlavorManifest: """A manifest of asset info for a specific flavor of an asset package.""" - assetfiles = entity.DictField('assetfiles', str, entity.StringValue()) + assetfiles: Annotated[Dict[str, str], + IOAttrs('assetfiles')] = field(default_factory=dict) -class AssetPackageFlavorManifest(entity.EntityMixin, - AssetPackageFlavorManifestValue): - """A self contained AssetPackageFlavorManifestValue.""" - - -class AssetPackageBuildState(entity.Entity): +@ioprepped +@dataclass +class AssetPackageBuildState: """Contains info about an in-progress asset cloud build.""" # Asset names still being built. - in_progress_builds = entity.ListField('b', entity.StringValue()) + in_progress_builds: Annotated[List[str], + IOAttrs('b')] = field(default_factory=list) # The initial number of assets needing to be built. - initial_build_count = entity.Field('c', entity.IntValue()) + initial_build_count: Annotated[int, IOAttrs('c')] = 0 # Build error string. If this is present, it should be presented # to the user and they should required to explicitly restart the build # in some way if desired. - error = entity.Field('e', entity.OptionalStringValue()) + error: Annotated[Optional[str], IOAttrs('e')] = None diff --git a/tools/efro/dataclassio/__init__.py b/tools/efro/dataclassio/__init__.py index db5bcb7f..f8eff835 100644 --- a/tools/efro/dataclassio/__init__.py +++ b/tools/efro/dataclassio/__init__.py @@ -16,14 +16,14 @@ from efro.dataclassio._outputter import _Outputter from efro.dataclassio._inputter import _Inputter from efro.dataclassio._base import Codec, IOAttrs from efro.dataclassio._prep import ioprep, ioprepped, is_ioprepped_dataclass -from efro.dataclassio._pathcapture import FieldStoragePathCapture +from efro.dataclassio._pathcapture import DataclassFieldLookup if TYPE_CHECKING: from typing import Any, Dict, Type, Tuple, Optional, List, Set __all__ = [ 'Codec', 'IOAttrs', 'ioprep', 'ioprepped', 'is_ioprepped_dataclass', - 'FieldStoragePathCapture', 'dataclass_to_dict', 'dataclass_to_json', + 'DataclassFieldLookup', 'dataclass_to_dict', 'dataclass_to_json', 'dataclass_from_dict', 'dataclass_from_json', 'dataclass_validate' ] @@ -57,18 +57,20 @@ def dataclass_to_dict(obj: Any, return out -def dataclass_to_json(obj: Any, coerce_to_float: bool = True) -> str: +def dataclass_to_json(obj: Any, + coerce_to_float: bool = True, + pretty: bool = False) -> str: """Utility function; return a json string from a dataclass instance. Basically json.dumps(dataclass_to_dict(...)). """ import json - return json.dumps( - dataclass_to_dict(obj=obj, - coerce_to_float=coerce_to_float, - codec=Codec.JSON), - separators=(',', ':'), - ) + jdict = dataclass_to_dict(obj=obj, + coerce_to_float=coerce_to_float, + codec=Codec.JSON) + if pretty: + return json.dumps(jdict, indent=2, sort_keys=True) + return json.dumps(jdict, separators=(',', ':')) def dataclass_from_dict(cls: Type[T], diff --git a/tools/efro/dataclassio/_pathcapture.py b/tools/efro/dataclassio/_pathcapture.py index 72b56a40..52e5ca0f 100644 --- a/tools/efro/dataclassio/_pathcapture.py +++ b/tools/efro/dataclassio/_pathcapture.py @@ -5,39 +5,35 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar, Generic from efro.dataclassio._base import _parse_annotated, _get_origin from efro.dataclassio._prep import PrepSession if TYPE_CHECKING: - from typing import Any, Dict, Type, Tuple, Optional, List, Set + from typing import Any, Dict, Type, Tuple, Optional, List, Set, Callable + +T = TypeVar('T') -class FieldStoragePathCapture: - """Utility for obtaining dataclass storage paths in a type safe way. +class _PathCapture: + """Utility for obtaining dataclass storage paths in a type safe way.""" - Given dataclass instance foo, FieldStoragePathCapture(foo).bar.eep - will return 'bar.eep' (or something like 'b.e' if storagenames are - overridden). This can be combined with type-checking tricks that - return foo in the type-checker's eyes while returning - FieldStoragePathCapture(foo) at runtime in order to grant a measure - of type safety to specifying field paths for things such as db - queries. Be aware, however, that the type-checker will incorrectly - think these lookups are returning actual attr values when they - are actually returning strings. - """ - - def __init__(self, obj: Any, path: List[str] = None): - if path is None: - path = [] - if not dataclasses.is_dataclass(obj): - raise TypeError(f'Expected a dataclass type/instance;' - f' got {type(obj)}.') + def __init__(self, obj: Any, pathparts: List[str] = None): + self._is_dataclass = dataclasses.is_dataclass(obj) + if pathparts is None: + pathparts = [] self._cls = obj if isinstance(obj, type) else type(obj) - self._path = path + self._pathparts = pathparts + + def __getattr__(self, name: str) -> _PathCapture: + + # We only allow diving into sub-objects if we are a dataclass. + if not self._is_dataclass: + raise TypeError( + f"Field path cannot include attribute '{name}' " + f'under parent {self._cls}; parent types must be dataclasses.') - def __getattr__(self, name: str) -> Any: prep = PrepSession(explicit=False).prep_dataclass(self._cls, recursion_level=0) try: @@ -48,8 +44,43 @@ class FieldStoragePathCapture: storagename = (name if (ioattrs is None or ioattrs.storagename is None) else ioattrs.storagename) origin = _get_origin(anntype) - path = self._path + [storagename] + return _PathCapture(origin, pathparts=self._pathparts + [storagename]) - if dataclasses.is_dataclass(origin): - return FieldStoragePathCapture(origin, path=path) - return '.'.join(path) + @property + def path(self) -> str: + """The final output path.""" + return '.'.join(self._pathparts) + + +class DataclassFieldLookup(Generic[T]): + """Get info about nested dataclass fields in type-safe way.""" + + def __init__(self, cls: T) -> None: + self.cls = cls + + def path(self, callback: Callable[[T], Any]) -> str: + """Look up a path on child dataclass fields. + + example: + DataclassFieldLookup(MyType).path(lambda obj: obj.foo.bar) + + The above example will return the string 'foo.bar' or something + like 'f.b' if the dataclasses have custom storage names set. + It will also be static-type-checked, triggering an error if + MyType.foo.bar is not a valid path. Note, however, that the + callback technically allows any return value but only nested + dataclasses and their fields will succeed. + """ + + # We tell the type system that we are returning an instance + # of our class, which allows it to perform type checking on + # member lookups. In reality, however, we are providing a + # special object which captures path lookups so we can build + # a string from them. + if not TYPE_CHECKING: + out = callback(_PathCapture(self.cls)) + if not isinstance(out, _PathCapture): + raise TypeError(f'Expected a valid path under' + f' the provided object; got a {type(out)}.') + return out.path + return ''